Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(reporting): stop validate from repeating its header in short terminals (#368)

Christopher Kevine40abf2cd200
diff --git a/CHANGELOG.md b/CHANGELOG.md
index bbfb8107..1ad3292f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -249,6 +249,12 @@ All notable changes to SkillEvaluator are documented in this file.
a `static_parse_limit` row on the manifest, so that scan is recorded as
incomplete in the same way.
+- `validate` no longer repeats its header box in short terminal windows or
+ when a log warning prints mid-run. Only the tier that is still running is
+ redrawn; the header and finished tiers print once, and log records print
+ through the live view's console. The progress line now prints once, above
+ the final verdict.
+
## 0.9.0 - 2026-11-13
### Fixed
diff --git a/src/skillevaluator/logging_config.py b/src/skillevaluator/logging_config.py
index af692c40..4c67d1aa 100644
--- a/src/skillevaluator/logging_config.py
+++ b/src/skillevaluator/logging_config.py
@@ -18,9 +18,28 @@ console = Console()
_LOG_MARKUP = False
+# While a live view owns the terminal, log records print through its console.
+# Rich writes every other console past Live's stdout proxy, which leaves stale
+# frame lines (a duplicated header) on screen.
+_live_console: list[Console] = []
+
+
+def route_logs_to(target: Console | None) -> None:
+ """Print log records through *target* (a live view's console) until called with ``None``."""
+ _live_console[:] = [target] if target is not None else []
+
+
class _LiteralRichHandler(RichHandler):
"""Rich handler that also drops terminal escape sequences from log messages."""
+ def emit(self, record: logging.LogRecord) -> None:
+ own = self.console
+ self.console = _live_console[0] if _live_console else own
+ try:
+ super().emit(record)
+ finally:
+ self.console = own
+
def render_message(self, record: logging.LogRecord, message: str) -> ConsoleRenderable:
# Imported here: skillevaluator.utils imports this module.
from skillevaluator.utils.rich_markup import strip_terminal_controls
diff --git a/src/skillevaluator/reporting/console_ui.py b/src/skillevaluator/reporting/console_ui.py
index 3ddcadef..4f022aa0 100644
--- a/src/skillevaluator/reporting/console_ui.py
+++ b/src/skillevaluator/reporting/console_ui.py
@@ -47,6 +47,7 @@ from rich.panel import Panel
from rich.text import Text
from skillevaluator.constants import CONTENT_DEDUP_LLM_REFUSED_CHECK
+from skillevaluator.logging_config import route_logs_to
from skillevaluator.reporting.base import (
get_skip_reason,
source_validation_errors,
@@ -574,13 +575,13 @@ class ValidateView:
if not self.enabled:
return
if self._live_capable():
- self._live = Live(
- self,
- console=self.console,
- refresh_per_second=8,
- vertical_overflow="visible",
- )
+ self.console.print(self._header())
+ self.console.print()
+ # Only unfinished tiers stay live; finished ones print once above. A live
+ # region taller than the window can't be erased, so each redraw would repeat it.
+ self._live = Live(get_renderable=self._live_region, console=self.console, refresh_per_second=8)
self._live.start()
+ route_logs_to(self.console)
else:
from skillevaluator.tier3.harbor.progress import PlainProgressReporter
@@ -625,8 +626,8 @@ class ValidateView:
block.duration = time.time() - block.started_at if block.started_at else None
block.rows = rows or []
self._complete_progress(index, "failed" if failed else "complete")
- self._refresh()
self._flush_static()
+ self._refresh()
def tier_skip(self, index: int, reason: str) -> None:
if not self.enabled:
@@ -637,34 +638,20 @@ class ValidateView:
# Multi-line reasons (setup instructions) continue on unlabeled rows.
lines = reason.split("\n") or [""]
block.rows = [TierRow("skipped" if index == 0 else "", [(line, MUTED)]) for index, line in enumerate(lines)]
- self._refresh()
self._flush_static()
+ self._refresh()
def finish(self, verdict: Verdict, links: list[tuple[str, str]]) -> None:
if not self.enabled:
return
- if self.progress_reporter is not None:
- self.progress_reporter.close()
self.verdict = verdict
- # Keep artifacts out of Live's refresh/stop buffers. After its render
- # hook is removed, normal printing can distinguish TTY from capture.
- self.links = []
- if self._live is not None:
- live = self._live
- try:
- live.update(self)
- finally:
- try:
- live.stop()
- finally:
- self._live = None
- else:
- if len(self.blocks) >= 2:
- self.console.print(self._gate())
- self.console.print()
- self.console.print(self._verdict_panel(verdict))
- self.console.print()
self.links = links
+ self.stop()
+ if len(self.blocks) >= 2:
+ self.console.print(self._gate())
+ self.console.print()
+ self.console.print(self._verdict_panel(verdict))
+ self.console.print()
for line in self._footer_lines():
self.console.print(line)
@@ -673,6 +660,7 @@ class ValidateView:
if self.progress_reporter is not None:
self.progress_reporter.close()
if self._live is not None:
+ route_logs_to(None)
self._live.stop()
self._live = None
@@ -852,17 +840,22 @@ class ValidateView:
def _refresh(self) -> None:
if self._live is not None:
- self._live.update(self)
+ self._live.refresh()
+
+ def _live_region(self) -> Group:
+ return Group(*(self._block_panel(b) for b in self.blocks[self._static_printed :] if b.status != PENDING))
def _flush_static(self) -> None:
- """Non-tty mode: print blocks once, as they complete."""
- if self._live is not None or self._live_capable():
- return
+ """Print blocks once, in order, as they complete."""
+ done = []
for block in self.blocks[self._static_printed :]:
if block.status in (PENDING, RUNNING):
break
+ done.append(block)
+ # Advance first so the live region drops these blocks as they print above it.
+ self._static_printed += len(done)
+ for block in done:
self.console.print(self._block_panel(block))
- self._static_printed += 1
def check_ticker_row(lineup: list[str], done: list[str], current: str | None) -> TierRow:
diff --git a/tests/reporting/test_tier3_validate_cases.py b/tests/reporting/test_tier3_validate_cases.py
index 4d87f4c5..abb89ae6 100644
--- a/tests/reporting/test_tier3_validate_cases.py
+++ b/tests/reporting/test_tier3_validate_cases.py
@@ -1,6 +1,7 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
+import io
import json
from pathlib import Path
from types import SimpleNamespace
@@ -16,6 +17,24 @@ def _no_skillspector_provider(monkeypatch):
monkeypatch.delenv("SKILLSPECTOR_PROVIDER", raising=False)
+class _TtyStream(io.StringIO):
+ """A stream that reports itself as a terminal, which the live view requires."""
+
+ def isatty(self):
+ return True
+
+
+def _live_console(out, **kwargs):
+ """A Rich console the live view will draw on.
+
+ ``color_system=None`` keeps the test from caching a colour code on Rich's shared Style
+ objects, which later tests (the banner colour test) would then read instead of their own.
+ """
+ from rich.console import Console
+
+ return Console(file=out, force_terminal=True, color_system=None, **kwargs)
+
+
def _text(rows):
return "\n".join(f"{r.label:<12}" + "".join(chunk for chunk, _ in r.segments) for r in rows)
@@ -328,6 +347,21 @@ def test_llm_fix_without_a_provider_gives_every_export(monkeypatch):
assert item.actions[0] == "export SKILL_EVAL_LLM_PROVIDER=nv_build NVIDIA_API_KEY='your-api-key'"
+def test_live_view_prints_log_records_through_its_own_console(monkeypatch):
+ from skillevaluator.logging_config import get_logger
+ from skillevaluator.reporting.console_ui import ValidateView
+
+ monkeypatch.setenv("TERM", "xterm-256color")
+ out = _TtyStream()
+ view = ValidateView("skill: x", [(1, "Static", "static")], console=_live_console(out, width=100))
+ view.start()
+ get_logger("skillevaluator.test_live_view").warning("LLM call failed")
+ view.stop()
+ get_logger("skillevaluator.test_live_view").warning("after the view")
+ assert "LLM call failed" in out.getvalue()
+ assert "after the view" not in out.getvalue()
+
+
def test_llm_fix_points_at_doctor_when_the_provider_call_fails(monkeypatch):
from skillevaluator import provider_config
from skillevaluator.reporting.console_ui import _llm_fix
@@ -522,6 +556,29 @@ def test_tier1_summary_counts_validators_and_their_sub_checks():
assert "2/4 checks passed (13 sub-checks) · 1 failed · 1 didn't run" in _text(rows)
+def test_live_view_draws_the_header_and_finished_tiers_once(monkeypatch):
+ import re
+
+ from skillevaluator.reporting.console_ui import TierRow, ValidateView, Verdict
+
+ monkeypatch.setenv("TERM", "xterm-256color")
+ out = _TtyStream()
+ console = _live_console(out, width=100, height=10)
+ view = ValidateView("x", [(1, "Static", "static"), (2, "Dedup", "dedup")], console=console)
+ view.start()
+ for index in range(2):
+ view.tier_start(index)
+ for step in range(5):
+ view.tier_progress(index, [TierRow("checks", [(f"step {step}", "")])])
+ view._live.refresh()
+ view.tier_done(index, rows=[TierRow("summary", [(f"tier {index} done", "")])])
+ view.finish(Verdict(passed=True, headline="all passed"), [])
+ text = re.sub(r"\x1b\[[0-9;?]*[A-Za-z]", "", out.getvalue())
+ assert text.count("skillevaluator validate") == 1
+ assert text.count("tier 0 done") == 1
+ assert text.count("tier 1 done") == 1
+
+
def test_tier1_summary_row_agrees_with_the_verdict_line():
from skillevaluator.models.result import ValidationResult
from skillevaluator.reporting.console_ui import summarize_tier1