Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(reporting): escape untrusted text in Rich CLI output (#173)
Narendran Raghavanb234e481b882
diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.pyindex 25567b21..7a7d76e6 100644--- a/src/skillevaluator/cli.py+++ b/src/skillevaluator/cli.py@@ -57,6 +57,7 @@ from skillevaluator.tier1.commands import ()from skillevaluator.tier3_environments import HARBOR_ENVIRONMENTSfrom skillevaluator.tier_group import TierGroup+from skillevaluator.utils.rich_markup import escape_markupfrom skillevaluator.utils.tier2_paths import (is_link_or_reparse,paths_refer_to_same_location,@@ -1279,11 +1280,11 @@ def _print_run_banner(target_path: Path, content_type: str, profile: str | None)run up front instead of opening straight on the Tier 1 section."""console.print(f"\n[bold]SkillEvaluator {content_type.title()} Validation[/bold]")- console.print(f"Target: {target_path}")+ console.print(f"Target: {escape_markup(str(target_path))}")console.print(f"Type: {content_type}")if profile:profile_color = "cyan"- console.print(f"Profile: [{profile_color}]{profile}[/{profile_color}]")+ console.print(f"Profile: [{profile_color}]{escape_markup(str(profile))}[/{profile_color}]")@cli.command(epilog=_VALIDATE_EPILOG)diff --git a/src/skillevaluator/logging_config.py b/src/skillevaluator/logging_config.pyindex 5e25f55e..af692c40 100644--- a/src/skillevaluator/logging_config.py+++ b/src/skillevaluator/logging_config.py@@ -5,12 +5,28 @@import logging-from rich.console import Console+from rich.console import Console, ConsoleRenderablefrom rich.logging import RichHandler# Global console instance for rich outputconsole = Console()+# Log messages interpolate skill paths, provider errors, and tool output, so the+# handlers must not parse them as Rich markup: "[/x]" in a message would raise+# rich.errors.MarkupError out of the logging call itself, and "[link=...]"+# could spoof the log line.+_LOG_MARKUP = False+++class _LiteralRichHandler(RichHandler):+ """Rich handler that also drops terminal escape sequences from log messages."""++ def render_message(self, record: logging.LogRecord, message: str) -> ConsoleRenderable:+ # Imported here: skillevaluator.utils imports this module.+ from skillevaluator.utils.rich_markup import strip_terminal_controls++ return super().render_message(record, strip_terminal_controls(message))+def get_logger(name: str, level: int = logging.INFO) -> logging.Logger:"""Get a configured logger instance.@@ -29,11 +45,11 @@ def get_logger(name: str, level: int = logging.INFO) -> logging.Logger:logger.setLevel(level)# Rich handler for console output- handler = RichHandler(+ handler = _LiteralRichHandler(console=console,show_time=False,show_path=False,- markup=True,+ markup=_LOG_MARKUP,rich_tracebacks=True,)handler.setLevel(level)@@ -67,11 +83,11 @@ def setup_logging(verbose: bool = False) -> None:root_logger.removeHandler(handler)# Add Rich handler- handler = RichHandler(+ handler = _LiteralRichHandler(console=console,show_time=False,show_path=False,- markup=True,+ markup=_LOG_MARKUP,rich_tracebacks=True,)handler.setLevel(level)diff --git a/src/skillevaluator/reporting/cli.py b/src/skillevaluator/reporting/cli.pyindex 29f61433..549ee2b7 100644--- a/src/skillevaluator/reporting/cli.py+++ b/src/skillevaluator/reporting/cli.py@@ -11,6 +11,13 @@ Features:- Summary table for multiple validators- Tree-structured findings display- Progress indicators and spinners++Text that SkillEvaluator does not author (skill and plugin content, paths,+LLM judge output, external tool messages) goes through ``escape_markup`` before+it is interpolated into Rich markup: ``[/x]`` would otherwise raise+``rich.errors.MarkupError`` and ``[link=...]`` could restyle or spoof output.+Consoles are created with ``emoji=False`` so ``:name:`` codes in that text+(``root:x:0:0:``) print literally."""from __future__ import annotations@@ -19,7 +26,6 @@ from io import StringIOfrom typing import TYPE_CHECKINGfrom rich.console import Console-from rich.markup import escape as rich_escapefrom rich.panel import Panelfrom rich.table import Table@@ -33,6 +39,7 @@ from skillevaluator.reporting.harbor_viewer import (normalize_harbor_viewer_for_display,safe_url,)+from skillevaluator.utils.rich_markup import escape_markupif TYPE_CHECKING:from skillevaluator.models import Finding, ValidationResult@@ -70,7 +77,7 @@ class CLIReporter(ReporterBase):def console(self) -> Console:"""Get or create the Rich console instance."""if self._console is None:- self._console = Console()+ self._console = Console(emoji=False)return self._console@property@@ -84,14 +91,14 @@ class CLIReporter(ReporterBase):def render(self, result: ValidationResult) -> str:"""Render single result to string (captures console output)."""string_io = StringIO()- temp_console = Console(file=string_io, force_terminal=True)+ temp_console = Console(file=string_io, force_terminal=True, emoji=False)self.render_result(result, temp_console)return string_io.getvalue()def render_all(self, results: list[ValidationResult]) -> str:"""Render all results to string with summary table."""string_io = StringIO()- temp_console = Console(file=string_io, force_terminal=True)+ temp_console = Console(file=string_io, force_terminal=True, emoji=False)self._render_all_results(results, temp_console)return string_io.getvalue()@@ -164,9 +171,9 @@ class CLIReporter(ReporterBase):def render_result(self, result: ValidationResult, console: Console) -> None:"""Render a single validation result."""# Header- console.print(f"\n[bold][{result.validator_name}][/bold]")+ console.print(f"\n[bold]{escape_markup(f'[{result.validator_name}]')}[/bold]")if result.validator_description:- console.print(f"[dim]{result.validator_description}[/dim]")+ console.print(f"[dim]{escape_markup(str(result.validator_description))}[/dim]")# Quality score display (unified table for single and multi-skill)qs = result.metadata.get("quality_scores") if result.metadata else None@@ -191,7 +198,7 @@ class CLIReporter(ReporterBase):self._print_summary_stats(result, console)elif result.is_incomplete:tools = ", ".join(result.incomplete_scans)- console.print(f"[yellow][INCOMPLETE] {tools} did not complete[/yellow]\n")+ console.print(f"[yellow][INCOMPLETE] {escape_markup(tools)} did not complete[/yellow]\n")self._print_summary_stats(result, console)self._print_findings(result, console)elif result.passed:@@ -225,13 +232,15 @@ class CLIReporter(ReporterBase):avg_score = qs.get("overall_score", 0)avg_grade = qs.get("grade", "?")gc = grade_colors.get(avg_grade, "white")+ avg_grade_text = escape_markup(str(avg_grade))if multi:- console.print(f"\n [{gc}]Average: {avg_score:.1f}/100 (Grade: {avg_grade})[/{gc}]")- console.print(f" [dim]Skills analyzed:[/dim] {qs.get('skill_count', len(skills_list))}\n")+ console.print(f"\n [{gc}]Average: {avg_score:.1f}/100 (Grade: {avg_grade_text})[/{gc}]")+ skill_count = escape_markup(str(qs.get("skill_count", len(skills_list))))+ console.print(f" [dim]Skills analyzed:[/dim] {skill_count}\n")else:- stype = skills_list[0].get("skill_type", "unknown")- console.print(f"\n [{gc}]Overall: {avg_score:.1f}/100 (Grade: {avg_grade})[/{gc}]")+ stype = escape_markup(str(skills_list[0].get("skill_type", "unknown")))+ console.print(f"\n [{gc}]Overall: {avg_score:.1f}/100 (Grade: {avg_grade_text})[/{gc}]")console.print(f" [dim]Skill Type:[/dim] {stype}\n")table = Table(title="Quality Scores by Skill", border_style="cyan", show_header=True)@@ -250,8 +259,8 @@ class CLIReporter(ReporterBase):sgc = grade_colors.get(sgrade, "white")dims = skill_qs.get("dimensions", {})table.add_row(- sname,- f"[{sgc}]{sgrade}[/{sgc}]",+ escape_markup(str(sname)),+ f"[{sgc}]{escape_markup(str(sgrade))}[/{sgc}]",f"[{sgc}]{sscore:.1f}[/{sgc}]",CLIReporter._score_cell(dims, "correctness"),CLIReporter._score_cell(dims, "discoverability"),@@ -265,7 +274,7 @@ class CLIReporter(ReporterBase):avg_dims = qs.get("dimensions", {})table.add_row("[bold]Average[/bold]",- f"[{agc}]{avg_grade}[/{agc}]",+ f"[{agc}]{avg_grade_text}[/{agc}]",f"[{agc}]{avg_score:.1f}[/{agc}]",CLIReporter._score_cell(avg_dims, "correctness"),CLIReporter._score_cell(avg_dims, "discoverability"),@@ -284,7 +293,7 @@ class CLIReporter(ReporterBase):console.print(f"\n [{color}]LLM Rubric Score: {score}/100[/{color}]")summary = rubric.get("summary", "")if summary:- console.print(f" [dim]{summary}[/dim]")+ console.print(f" [dim]{escape_markup(str(summary))}[/dim]")console.print()table = Table(title="Rubric Evaluation", border_style="cyan", show_header=True)@@ -298,10 +307,10 @@ class CLIReporter(ReporterBase):cc = "green" if cs >= 7 else ("yellow" if cs >= 5 else "red")passed = "[green]Yes[/green]" if check.get("pass") else "[red]No[/red]"table.add_row(- check.get("id", "?").replace("_", " ").title(),+ escape_markup(str(check.get("id", "?")).replace("_", " ").title()),f"[{cc}]{cs}/10[/{cc}]",passed,- check.get("notes", ""),+ escape_markup(str(check.get("notes") or "")),)console.print(table)@@ -316,17 +325,21 @@ class CLIReporter(ReporterBase):vc = "green" if verdict == "pass" else ("red" if verdict == "fail" else "yellow")composite_text = f"{composite:+.2f}" if isinstance(composite, int | float) else "N/A"- console.print(f"\n [{vc}]Verdict: {verdict.upper()} (composite lift = {composite_text})[/{vc}]")+ verdict_text = escape_markup(str(verdict).upper())+ console.print(f"\n [{vc}]Verdict: {verdict_text} (composite lift = {composite_text})[/{vc}]")if runtime:console.print(f" [dim]Runtime: {runtime:.1f}s[/dim]")harbor_viewer = normalize_harbor_viewer_for_display(agent_eval)if harbor_viewer.get("job_url") or harbor_viewer.get("analysis_url"):console.print(" [dim]Harbor artifacts:[/dim]")if harbor_viewer.get("job_url"):- console.print(f" [dim]Harbor logs:[/dim] [cyan]{harbor_viewer['job_url']}[/cyan]", soft_wrap=True)+ console.print(+ f" [dim]Harbor logs:[/dim] [cyan]{escape_markup(harbor_viewer['job_url'])}[/cyan]",+ soft_wrap=True,+ )if harbor_viewer.get("analysis_url"):console.print(- f" [dim]Harbor analysis:[/dim] [cyan]{harbor_viewer['analysis_url']}[/cyan]",+ f" [dim]Harbor analysis:[/dim] [cyan]{escape_markup(harbor_viewer['analysis_url'])}[/cyan]",soft_wrap=True,)console.print()@@ -349,7 +362,7 @@ class CLIReporter(ReporterBase):lift = scores.get("lift", 0.0)lc = "green" if lift > 0.01 else ("red" if lift < -0.01 else "dim")table.add_row(- name.replace("_", " ").title(),+ escape_markup(str(name).replace("_", " ").title()),f"{ws:.2f}",f"{bl:.2f}",f"[{lc}]{lift:+.2f}[/{lc}]",@@ -370,14 +383,13 @@ class CLIReporter(ReporterBase):if not printed:console.print("[bold]Recommendations[/bold]")printed = True- console.print(f" • {message}", soft_wrap=True)+ console.print(f" • {escape_markup(message)}", soft_wrap=True)evidence = recommendation.get("evidence")if isinstance(evidence, dict):url = safe_url(evidence.get("url"))if url:- console.print(- f" [dim]{harbor_evidence_link_text(evidence)}:[/dim] [cyan]{url}[/cyan]", soft_wrap=True- )+ link_text = escape_markup(harbor_evidence_link_text(evidence))+ console.print(f" [dim]{link_text}:[/dim] [cyan]{escape_markup(url)}[/cyan]", soft_wrap=True)if printed:console.print()@@ -399,7 +411,7 @@ class CLIReporter(ReporterBase):explanation = info.get("explanation", "")if isinstance(score, str):sc = "green" if score.upper() == "PASS" else "red"- score_str = f"[{sc}]{score}[/{sc}]"+ score_str = f"[{sc}]{escape_markup(score)}[/{sc}]"else:sc = ("green"@@ -407,7 +419,12 @@ class CLIReporter(ReporterBase):else ("yellow" if score >= DIMENSION_VERDICT_NEUTRAL_THRESHOLD else "red"))score_str = f"[{sc}]{score:.2f}[/{sc}]"- table.add_row(dim.title(), score_str, explanation[:80])+ # Truncate before escaping so the cut cannot split an escape sequence.+ table.add_row(+ escape_markup(str(dim).title()),+ score_str,+ escape_markup(str(explanation or "")[:80]),+ )console.print(table)console.print()@@ -441,7 +458,7 @@ class CLIReporter(ReporterBase):if result.messages:console.print("[dim]Details:[/dim]")for msg in result.messages:- console.print(f" {rich_escape(str(msg))}")+ console.print(f" {escape_markup(str(msg))}")returnconsole.print("[dim]Details:[/dim]")@@ -453,8 +470,8 @@ class CLIReporter(ReporterBase):meta_str = f" ({', '.join(meta_parts)})"console.print(" [green][OK][/green] "- f"{rich_escape(str(detail.check_name))}: "- f"{rich_escape(str(detail.message))}{rich_escape(meta_str)}"+ f"{escape_markup(str(detail.check_name))}: "+ f"{escape_markup(str(detail.message))}{escape_markup(meta_str)}")def _print_findings(self, result: ValidationResult, console: Console) -> None:@@ -464,13 +481,13 @@ class CLIReporter(ReporterBase):label = "Execution errors" if result.is_incomplete else "Errors"console.print(f"[dim]{label}:[/dim]")for error in errors:- console.print(f" [red]•[/red] {rich_escape(str(error))}")+ console.print(f" [red]•[/red] {escape_markup(str(error))}")if not result.findings:# Preserve legacy warnings when there are no structured findings.if result.warnings:console.print("[dim]Warnings:[/dim]")for warning in result.warnings:- console.print(f" [yellow][WARN][/yellow] {rich_escape(str(warning))}")+ console.print(f" [yellow][WARN][/yellow] {escape_markup(str(warning))}")returnif errors:@@ -488,25 +505,25 @@ class CLIReporter(ReporterBase):"""severity_color = finding.severity.colorconsole.print(- f" {index}. [{severity_color}]{rich_escape(finding.tag)}[/{severity_color}] {rich_escape(finding.message)}"+ f" {index}. [{severity_color}]{escape_markup(finding.tag)}[/{severity_color}] {escape_markup(finding.message)}")- console.print(f" [dim]File:[/dim] {rich_escape(str(finding.location))}")- console.print(f" [dim]Check:[/dim] {rich_escape(str(finding.check_name))}")+ console.print(f" [dim]File:[/dim] {escape_markup(str(finding.location))}")+ console.print(f" [dim]Check:[/dim] {escape_markup(str(finding.check_name))}")related_paths = _related_paths(finding)if related_paths:- console.print(f" [dim]Related paths:[/dim] {rich_escape(' <-> '.join(related_paths))}")+ console.print(f" [dim]Related paths:[/dim] {escape_markup(' <-> '.join(related_paths))}")if finding.line_content:content = finding.line_content.strip()if len(content) > 70:content = content[:67] + "..."- console.print(f" [dim]Content:[/dim] [italic]{rich_escape(content)}[/italic]")+ console.print(f" [dim]Content:[/dim] [italic]{escape_markup(content)}[/italic]")if finding.suggestion:label = ("Recommendation" if finding.category in ("WHOLE_DUPLICATE", "PARTIAL_OVERLAP", "DUPLICATE") else "Fix")- console.print(f" [dim]{label}:[/dim] {rich_escape(finding.suggestion)}")+ console.print(f" [dim]{label}:[/dim] {escape_markup(finding.suggestion)}")console.print()def _print_summary_table(self, results: list[ValidationResult], console: Console) -> None:@@ -539,9 +556,10 @@ class CLIReporter(ReporterBase):if advisory_skip:agent_eval = result.metadata.get("agent_eval", {})provenance = agent_eval.get("provenance", {}) if isinstance(agent_eval, dict) else {}- details = str(provenance.get("message") or "Live evaluation did not run")+ details = escape_markup(str(provenance.get("message") or "Live evaluation did not run"))elif result.is_incomplete:- details = f"[bold yellow]{', '.join(result.incomplete_scans)} did not complete[/bold yellow]"+ scanners = escape_markup(", ".join(result.incomplete_scans))+ details = f"[bold yellow]{scanners} did not complete[/bold yellow]"counts = []if s.errors:counts.append(f"{s.errors} errors")@@ -553,7 +571,7 @@ class CLIReporter(ReporterBase):if result.metadata.get("skipped"):details = "Skipped (see warnings)"elif static_test_evidence:- details = rich_escape(static_test_evidence)+ details = escape_markup(static_test_evidence)elif s.checks_performed > 0:details = f"{s.checks_performed} checks passed"else:@@ -566,7 +584,7 @@ class CLIReporter(ReporterBase):parts.append(f"{s.warnings} warnings")details = ", ".join(parts) if parts else "Failed"- table.add_row(result.validator_name, status, details)+ table.add_row(escape_markup(str(result.validator_name)), status, details)console.print(table)diff --git a/src/skillevaluator/tier1/commands.py b/src/skillevaluator/tier1/commands.pyindex d0822fa0..95ecaee8 100644--- a/src/skillevaluator/tier1/commands.py+++ b/src/skillevaluator/tier1/commands.py@@ -22,6 +22,7 @@ from skillevaluator.models.result import ValidationResultfrom skillevaluator.reporting import CLIReporter, HTMLReporter, JSONReporter, MarkdownReporter, SARIFReporterfrom skillevaluator.reporting.html import is_tier2_validator_namefrom skillevaluator.reporting.naming import DEFAULT_REPORT_BASENAME+from skillevaluator.utils.rich_markup import escape_markupfrom skillevaluator.validators.base import continue_on_failure_scopefrom skillevaluator.validators.code_risk import CodeRiskValidatorfrom skillevaluator.validators.dependencies import DependencySecurityValidator@@ -40,7 +41,8 @@ from skillevaluator.validators.unicode_smuggle import UnicodeSmuggleValidatorfrom skillevaluator.validators.version import VersionValidatorfrom skillevaluator.validators.workflows_schema import WorkflowsSchemaValidator-console = Console()+# emoji=False: ":name:" codes in untrusted text (such as "root:x:0:0:") must print literally.+console = Console(emoji=False)# Per-check progress goes to stderr so piped stdout (reports, JSON) stays# clean; without it, slow targets print nothing for minutes and look hung.@@ -369,6 +371,6 @@ def emit_reports(output_path = output_dir / f"{basename}{reporter.get_file_extension()}"reporter.save(results, output_path)if announce_paths:- console.print(f"[dim]{fmt} report:[/dim] [cyan]{output_path}[/cyan]")+ console.print(f"[dim]{fmt} report:[/dim] [cyan]{escape_markup(str(output_path))}[/cyan]")return all(result.passed for result in results)diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.pyindex b134919f..ae8c24a5 100644--- a/src/skillevaluator/tier3/commands.py+++ b/src/skillevaluator/tier3/commands.py@@ -62,8 +62,10 @@ from skillevaluator.tier3.results_location import (resolve_results_root,)from skillevaluator.tier3.toml_utils import toml_quote+from skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controls-console = Console()+# emoji=False: ":name:" codes in untrusted text must print literally.+console = Console(emoji=False)_HARBOR_RESERVED_CASE_NAMES = frozenset({"dataset.toml", "readme.md", "metric.py", "results"})_DEFAULT_AGENT_BY_PROVIDER = {@@ -461,7 +463,10 @@ def init_custom_grader(existing = [path for path in (grader_py, grader_sh) if path.exists()]if existing and not force:names = ", ".join(str(evals_dir / path.name) for path in existing)- console.print(f"[red]Error:[/red] custom grader already exists: {names}. Re-run with --force to overwrite.")+ console.print(+ f"[red]Error:[/red] custom grader already exists: {escape_markup(names)}. "+ "Re-run with --force to overwrite."+ )return 1if force:for existing_path in existing:@@ -476,7 +481,7 @@ def init_custom_grader(copytree_secure(private_evals, evals_dir, replace_existing=True, allowed_root=private_root)grader_path = evals_dir / private_grader.name- console.print(f"Created SkillEvaluator custom grader starter at [cyan]{grader_path}[/cyan]")+ console.print(f"Created SkillEvaluator custom grader starter at [cyan]{escape_markup(str(grader_path))}[/cyan]")return 0@@ -535,7 +540,10 @@ def init_harbor_task(public_case = evals_dir / "harbor" / case_idif os.path.lexists(private_case):if not force:- console.print(f"[red]Error:[/red] {public_case} already exists. Re-run with --force to overwrite.")+ console.print(+ f"[red]Error:[/red] {escape_markup(str(public_case))} already exists. "+ "Re-run with --force to overwrite."+ )return 1if not private_case.is_dir() or private_case.is_symlink():raise ValueError(f"existing Harbor case must be a real directory: {public_case}")@@ -614,7 +622,7 @@ Keep the result contract stable:copytree_secure(private_evals, evals_dir, replace_existing=True, allowed_root=private_root)- console.print(f"Created Harbor BYOT starter at [cyan]{evals_dir / 'harbor'}[/cyan]")+ console.print(f"Created Harbor BYOT starter at [cyan]{escape_markup(str(evals_dir / 'harbor'))}[/cyan]")return 0@@ -857,7 +865,7 @@ def doctor(status_styles = {"pass": "green", "warn": "yellow", "fail": "red"}for name, status, detail in rows:style = status_styles[status]- table.add_row(name, f"[{style}]{status}[/{style}]", detail)+ table.add_row(escape_markup(str(name)), f"[{style}]{status}[/{style}]", escape_markup(str(detail)))console.print(table)return 1 if any(row[1] == "fail" for row in rows) else 0@@ -923,8 +931,8 @@ def _print_validate_results(skill_path: Path, results: list[Any]) -> None:icon, style = status_icon.get(result.status, ("[ ? ]", "dim"))body.append(f" {icon:<10s}", style=style)path_style = "bold" if result.status in ("error", "missing") else "white"- body.append(f"{result.path:<38s}", style=path_style)- body.append(f"{result.message}\n", style="dim")+ body.append(f"{strip_terminal_controls(str(result.path)):<38s}", style=path_style)+ body.append(f"{strip_terminal_controls(str(result.message))}\n", style="dim")n_ok = sum(1 for result in results if result.status == "ok")n_warn = sum(1 for result in results if result.status == "warning")@@ -943,7 +951,8 @@ def _print_validate_results(skill_path: Path, results: list[Any]) -> None:console.print(Panel(body,- title=f"[bold]Validate: {skill_path.name}/evals/[/bold]",+ # Panel parses a str title with emoji enabled whatever the console says.+ title=Text.from_markup(f"[bold]Validate: {escape_markup(skill_path.name)}/evals/[/bold]", emoji=False),border_style="cyan" if not n_err else "red",padding=(1, 1),)@@ -968,7 +977,7 @@ def view_results(skill_path: Path, *, results_dir: Path | None = None) -> Path:report_path = target / "report.html"if not report_path.exists():report_path = render_agent_eval_html_report(skill_path, target)- console.print(f"Opening: [cyan]{report_path}[/cyan]")+ console.print(f"Opening: [cyan]{escape_markup(str(report_path))}[/cyan]")webbrowser.open(report_path.as_uri())return report_path@@ -991,7 +1000,7 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> intcandidate_roots = iter_candidate_results_roots(skill_path, results_dir)if not any(root.exists() for root in candidate_roots):searched = ", ".join(str(p) for p in candidate_roots)- console.print(f"[red]Error: No results found. Searched: {searched}[/red]")+ console.print(f"[red]Error: No results found. Searched: {escape_markup(searched)}[/red]")return 1agent_with: dict[str, dict[str, float]] = {}@@ -1058,12 +1067,12 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> inttable = Table(show_header=True, header_style="bold dim", box=SIMPLE, padding=(0, 1), show_edge=False, expand=True)table.add_column("Evaluator", style="white", min_width=18, no_wrap=True)for agent in agents:- table.add_column(f"{agent}\nscore", justify="right", min_width=7)+ table.add_column(f"{escape_markup(agent)}\nscore", justify="right", min_width=7)if agent in agent_without:table.add_column("\nlift", justify="right", min_width=7)for metric in display_metrics:- row: list[str | Text] = [Text(metric)]+ row: list[str | Text] = [Text(strip_terminal_controls(metric))]for agent in agents:with_score = _safe_score(agent_with[agent], metric)row.append(Text(f"{with_score:.2f}", style=f"bold {_score_style(with_score)}"))@@ -1095,12 +1104,19 @@ def compare_results(skill_path: Path, *, results_dir: Path | None = None) -> intconsole.print()console.print(- Panel(table, title=f"[bold]Skill Evaluation - {skill_path.name}[/bold]", border_style="cyan", padding=(1, 1))+ Panel(+ table,+ title=Text.from_markup(f"[bold]Skill Evaluation - {escape_markup(skill_path.name)}[/bold]", emoji=False),+ border_style="cyan",+ padding=(1, 1),+ ))console.print()for agent in agents:meta = agent_meta[agent]- console.print(f" [dim]{agent:<16s} {meta['timestamp']} (Harbor, {meta.get('num_trials', '?')} trials)[/dim]")+ # Pad before escaping so escape backslashes do not count toward the column width.+ run_line = f"{agent:<16s} {meta['timestamp']} (Harbor, {meta.get('num_trials', '?')} trials)"+ console.print(f" [dim]{escape_markup(run_line)}[/dim]")console.print()return 0diff --git a/src/skillevaluator/tier3/harbor/progress.py b/src/skillevaluator/tier3/harbor/progress.pyindex fa2e65f1..87be35ff 100644--- a/src/skillevaluator/tier3/harbor/progress.py+++ b/src/skillevaluator/tier3/harbor/progress.py@@ -20,6 +20,7 @@ from time import monotonicfrom typing import Literal, Protocol, TextIO, runtime_checkablefrom skillevaluator.tier3.harbor.secret_redaction import redact_secrets_in_log_line+from skillevaluator.utils.rich_markup import strip_terminal_controlsProgressMode = Literal["auto", "rich", "plain", "off"]logger = logging.getLogger(__name__)@@ -32,9 +33,6 @@ _SECRET_ASSIGNMENT_RE = re.compile(r"(?:\"[^\"]*\"|'[^']*'|[^\s,;]+)")_SECRET_ENV_NAME_RE = re.compile(r"(?i)(?:api[_-]?key|access[_-]?key|auth|credential|password|secret|token)")-_ANSI_ESCAPE_RE = re.compile(r"\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")-_OSC_ESCAPE_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")-_TERMINAL_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")@dataclass(frozen=True, slots=True)@@ -99,10 +97,7 @@ class ProgressReporter(Protocol):def redact_progress_detail(detail: object, *, secret_values: set[str] | None = None) -> str:"""Return a single-line diagnostic safe enough for a progress surface."""- text = _OSC_ESCAPE_RE.sub("", str(detail))- text = _ANSI_ESCAPE_RE.sub("", text)- text = _TERMINAL_CONTROL_RE.sub("", text)- text = " ".join(text.split())+ text = " ".join(strip_terminal_controls(str(detail)).split())for secret in sorted(secret_values or (), key=len, reverse=True):if len(secret) >= 4:text = text.replace(secret, "<redacted>")diff --git a/src/skillevaluator/tier3/harbor/report.py b/src/skillevaluator/tier3/harbor/report.pyindex 57c98e47..f886867d 100644--- a/src/skillevaluator/tier3/harbor/report.py+++ b/src/skillevaluator/tier3/harbor/report.py@@ -270,6 +270,8 @@ def _render_findings_body(findings: list[dict[str, Any]]) -> Any:"""Render findings list into a Rich Text body (icon/label/score/question/reasons/evidence)."""from rich.text import Text+ from skillevaluator.utils.rich_markup import strip_terminal_controls+body = Text()for finding in findings:if finding["severity"] == "critical":@@ -283,7 +285,7 @@ def _render_findings_body(findings: list[dict[str, Any]]) -> Any:style = "bold green"body.append(f" {icon}: ", style=style)- body.append(f"{finding['label']} ", style="bold white")+ body.append(f"{strip_terminal_controls(str(finding['label']))} ", style="bold white")body.append(f"{finding['score']:.2f}\n", style=style)question = METRIC_QUESTIONS.get(finding["metric"])if question:@@ -291,13 +293,11 @@ def _render_findings_body(findings: list[dict[str, Any]]) -> Any:reason_style = "dim green" if finding["severity"] == "ok" else "dim"for reason in finding["reasons"]:- body.append(f" → {reason}\n", style=reason_style)+ body.append(f" → {strip_terminal_controls(reason)}\n", style=reason_style)for ref in (finding.get("evidence_refs") or [])[:3]:- if isinstance(ref, str):- body.append(f" evidence: {ref}\n", style="dim")- else:- body.append(f" evidence: {_compact_evidence_ref(ref)}\n", style="dim")+ ref_text = ref if isinstance(ref, str) else _compact_evidence_ref(ref)+ body.append(f" evidence: {strip_terminal_controls(ref_text)}\n", style="dim")body.append("\n")return body@@ -835,8 +835,11 @@ def display_findings_report("""from rich.console import Consolefrom rich.panel import Panel+ from rich.text import Text++ from skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controls- console = Console()+ console = Console(emoji=False)agents_data = harbor_result.get("agents", {})report_agents = list(dict.fromkeys([*harbor_agents, *agents_data.keys()]))@@ -849,7 +852,7 @@ def display_findings_report(best_agent_label = _agent_model_label(best_agent, harbor_result, agents_data)console.print(f" [dim]Findings from best performing agent/model combination for your skill:[/dim] "- f"[bold cyan]{best_agent_label}[/bold cyan]"+ f"[bold cyan]{escape_markup(best_agent_label)}[/bold cyan]")console.print()else:@@ -914,14 +917,14 @@ def display_findings_report(suggestions = add_evidence_links_to_suggestions(suggestions, rewards)body.append(" \U0001f4a1 SUGGESTIONS\n", style="bold cyan")for i, suggestion in enumerate(suggestions, 1):- body.append(f" {i}. {suggestion}\n", style="white")+ body.append(f" {i}. {strip_terminal_controls(str(suggestion))}\n", style="white")else:body.append(" \U0001f4a1 NEXT STEPS\n", style="bold cyan")suggestions = _passing_skill_suggestions(findings, rewards)rendered_messages.update(str(suggestion).strip() for suggestion in suggestions if str(suggestion).strip())suggestions = add_evidence_links_to_suggestions(suggestions, rewards)for i, suggestion in enumerate(suggestions, 1):- body.append(f" {i}. {suggestion}\n", style="white")+ body.append(f" {i}. {strip_terminal_controls(str(suggestion))}\n", style="white")rendered_messages.update(str(suggestion).strip() for suggestion in suggestions if str(suggestion).strip())for agent, (agent_findings, _agent_rewards) in agent_reports.items():@@ -943,7 +946,11 @@ def display_findings_report(console.print(Panel(body,- title=f"[bold]{skill_name} / {panel_agent_label} \u2014 Findings[/bold]",+ # Panel parses a str title with emoji enabled whatever the console says.+ title=Text.from_markup(+ f"[bold]{escape_markup(f'{skill_name} / {panel_agent_label}')} \u2014 Findings[/bold]",+ emoji=False,+ ),border_style="cyan",padding=(1, 1),)diff --git a/src/skillevaluator/utils/rich_markup.py b/src/skillevaluator/utils/rich_markup.pynew file mode 100644index 00000000..7bf4f96b--- /dev/null+++ b/src/skillevaluator/utils/rich_markup.py@@ -0,0 +1,66 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Make untrusted text safe to print through Rich."""++from __future__ import annotations++import re++# OSC strings (hyperlinks, window titles) go first so their payload goes with them.+_OSC_ESCAPE_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")+# CSI sequences, then every other ECMA-48 escape (such as "ESC c", a full terminal reset).+_ANSI_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|[ -/]*[0-~])")+# C0 controls except tab, line feed and carriage return, then DEL and C1 controls.+_TERMINAL_CONTROL_RE = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f-\x9f]")++# Every "[" with the whole backslash run before it (the lookbehind and possessive+# quantifier keep long runs linear). The optional group is set when the bracket+# opens something Rich parses as a tag (same shape as ``rich.markup.RE_TAGS``).+_OPEN_BRACKET = re.compile(r"(?<!\\)(\\*+)\[(?=([a-z#/@][^\[\]]*\])?)")+# Rich halves a backslash run only when a tag follows it, so an escaped trailing+# run is always followed by this empty tag pair.+_EMPTY_TAG = "[bold][/bold]"+++def strip_terminal_controls(text: str) -> str:+ """Remove terminal escape sequences and control characters other than tab, LF and CR.++ Rich only strips BEL, BS, VT, FF and CR, so without this untrusted text can+ move the cursor, erase lines, reset the terminal or emit OSC 8 hyperlinks.+ """+ text = _OSC_ESCAPE_RE.sub("", text)+ text = _ANSI_ESCAPE_RE.sub("", text)+ return _TERMINAL_CONTROL_RE.sub("", text)+++def _escape_bracket(match: re.Match[str]) -> str:+ backslashes, tag = match.group(1), match.group(2)+ if tag is not None:+ # Tag-shaped: literal backslashes are doubled, then the tag is escaped.+ return f"{backslashes}{backslashes}\\["+ # Rich renders a plain-text "\[" as "[", so one extra backslash keeps the rest.+ return f"{backslashes}\\["+++def escape_markup(text: str) -> str:+ """Return ``text`` made safe to interpolate into Rich markup.++ Terminal escape sequences and control characters other than tab and newline+ are removed first (see ``strip_terminal_controls``). Every bracket is then+ escaped: ``rich.markup.escape`` only escapes tag-shaped brackets, but Rich+ also turns every plain-text ``\\[`` into ``[``, so Windows paths such as+ ``C:\\skills\\[x]`` would lose a separator. A trailing backslash run is+ doubled and closed with an empty tag, so it renders literally whatever+ follows it in the markup (a tag, plain text, or nothing).++ Rich also replaces ``:name:`` emoji codes, which markup cannot escape.+ Consoles that print untrusted text must be created with ``emoji=False``, and+ Panel titles and Status text, which Rich parses with emoji enabled whatever+ the console says, must be passed as ``Text.from_markup(..., emoji=False)``.+ """+ text = strip_terminal_controls(text).replace("\r", "")+ escaped = _OPEN_BRACKET.sub(_escape_bracket, text)+ body = escaped.rstrip("\\")+ trailing = len(escaped) - len(body)+ return body + "\\" * (2 * trailing) + _EMPTY_TAG if trailing else escapeddiff --git a/src/skillevaluator/validators/rubric_eval.py b/src/skillevaluator/validators/rubric_eval.pyindex 3176fc87..ca734b2c 100644--- a/src/skillevaluator/validators/rubric_eval.py+++ b/src/skillevaluator/validators/rubric_eval.py@@ -515,12 +515,15 @@ class RubricEvalValidator(ValidatorBase):from rich.console import Consolefrom rich.status import Status+ from rich.text import Text- with Status(- f"[bold cyan]Evaluating {skill_path.name} with LLM judge...[/bold cyan]",- console=Console(),- spinner="dots",- ):+ from skillevaluator.utils.rich_markup import escape_markup++ # Status parses a str with emoji enabled whatever the console says.+ status_text = Text.from_markup(+ f"[bold cyan]Evaluating {escape_markup(skill_path.name)} with LLM judge...[/bold cyan]", emoji=False+ )+ with Status(status_text, console=Console(emoji=False), spinner="dots"):report = self._judge.process(skill_name=skill_path.name,skill_content=content,diff --git a/tests/reporting/test_rich_markup_escaping.py b/tests/reporting/test_rich_markup_escaping.pynew file mode 100644index 00000000..be780b69--- /dev/null+++ b/tests/reporting/test_rich_markup_escaping.py@@ -0,0 +1,629 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Untrusted text must render literally in Rich console output.++Skill names, descriptions, paths, finding messages, LLM rubric notes, judge+reasoning, and external tool output are interpolated into Rich markup strings.+Unescaped, ``[/x]`` raises ``rich.errors.MarkupError`` and aborts the report,+while ``[bold]...[/bold]`` or ``[link=...]...[/link]`` restyles or spoofs it.+"""++from __future__ import annotations++import io+import json+import logging+import re+import sys+from pathlib import Path, PurePosixPath, PureWindowsPath+from types import SimpleNamespace+from unittest.mock import patch++import pytest+from rich.console import Console+from rich.logging import RichHandler+from rich.text import Text++from skillevaluator.models import Finding, Severity, ValidationResult+from skillevaluator.reporting import CLIReporter++PAYLOADS = ("[/x]", "[bold]evil[/bold]", "[link=http://x]y[/link]")+# pathlib collapses "//", so path payloads avoid it; a payload with "/" spans+# directory components ("[" and "x]" for "[/x]").+PATH_PAYLOADS = ("[/x]", "[bold]evil[/bold]")+# A single path component (a skill or agent directory name) cannot contain "/",+# and on Windows it cannot contain ":" either, so the emoji name is POSIX-only there.+_WINDOWS = sys.platform == "win32"+_EMOJI_NAME_SUFFIX = "" if _WINDOWS else ":x:"+NAME_PAYLOADS = (+ "[bold]evil",+ "[green]PASS",+ pytest.param("demo:x:", marks=pytest.mark.skipif(_WINDOWS, reason="':' is not valid in Windows file names")),+)+# Rich turns ":x:", ":key:" and ":white_check_mark:" into emoji unless the console,+# Panel title or Status text disables it.+EMOJI_TEXT = "grep root:x:0:0:root:/root:/bin/bash :key: :white_check_mark:"+EMOJI_GLYPHS = ("\u274c", "\U0001f511", "\u2705")++# Terminal control sequences untrusted text could carry, and what stays visible.+CONTROL_PAYLOADS = {+ "\x1b[1A\x1b[2K": "", # cursor up one line, erase it+ "\x1b[2J": "", # clear the screen+ "\x1b]8;;https://evil.test/\x1b\\see docs\x1b]8;;\x1b\\": "see docs", # OSC 8 hyperlink+ "\x1bc": "", # RIS: full terminal reset+ "\x9b2J": "2J", # 8-bit CSI+}+_SGR = re.compile(r"\x1b\[[0-9;]*m")+_CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f-\x9f]")+++def _recording_console(width: int = 200) -> Console:+ # Tests that print absolute tmp paths pass a wider width so long paths do not wrap.+ return Console(file=io.StringIO(), record=True, width=width)+++def _terminal_console() -> Console:+ return Console(file=io.StringIO(), force_terminal=True, color_system="truecolor", width=200)+++def _terminal_text(output: str) -> str:+ """Drop Rich's own SGR styling and fail if any other control character is left."""+ text = _SGR.sub("", output)+ assert _CONTROL.search(text) is None, repr(text)+ return text+++def _quality_result(payload: str) -> ValidationResult:+ result = ValidationResult(validator_name="QUALITY", validator_description=f"Quality of {payload}")+ dimensions = {name: {"score": 90.0} for name in ("correctness", "discoverability", "reliability", "efficiency")}+ result.metadata["quality_scores"] = {+ "overall_score": 88.0,+ "grade": "B",+ "skill_count": 2,+ "dimensions": dimensions,+ }+ result.metadata["quality_scores_all"] = [+ {"skill_name": f"alpha {payload}", "grade": "A", "overall_score": 91.0, "dimensions": dimensions},+ {"skill_name": f"beta {payload}", "grade": "B", "overall_score": 85.0, "dimensions": dimensions},+ ]+ return result+++def _rubric_result(payload: str) -> ValidationResult:+ result = ValidationResult(validator_name="RUBRIC_EVAL", validator_description="LLM rubric evaluation")+ result.metadata["rubric_eval"] = {+ "overall_score": 64.0,+ "summary": f"Judge summary {payload}",+ "checks": [{"id": "scope_definition", "score": 6, "pass": False, "notes": f"Judge notes {payload}"}],+ }+ result.add_error(f"Rubric error {payload}")+ return result+++def _finding_result(payload: str) -> ValidationResult:+ result = ValidationResult(validator_name="SECURITY", validator_description="Security checks")+ result.add_finding(+ Finding(+ category="SECURITY",+ severity=Severity.HIGH,+ check_name="mcp-least-privilege",+ message=f"Finding message {payload}",+ file_path=f"skills/{payload}/SKILL.md",+ line_number=3,+ line_content=f"content {payload}",+ suggestion=f"Suggested fix {payload}",+ )+ )+ return result+++def _agent_eval_result(payload: str) -> ValidationResult:+ result = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live agent evaluation")+ result.metadata["agent_eval"] = {+ "verdict": "pass",+ "composite_lift": 0.2,+ "runtime_seconds": 3.0,+ "harbor_viewer": {+ "job_url": f"https://harbor.example.test/jobs/{payload}",+ "analysis_url": f"https://harbor.example.test/analysis/{payload}",+ },+ "evaluators": {f"custom {payload}": {"with_skill": 0.9, "baseline": 0.5, "lift": 0.4}},+ "recommendations": [+ {+ "message": f"Recommendation {payload}",+ "evidence": {"url": f"https://harbor.example.test/trials/{payload}", "label": f"trial {payload}"},+ }+ ],+ "insights": {f"dim {payload}": {"score": f"PASS {payload}", "explanation": f"Judge reasoning {payload}"}},+ }+ return result+++def _advisory_skip_result(payload: str) -> ValidationResult:+ result = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live agent evaluation")+ result.metadata["agent_eval"] = {+ "provenance": {"advisory": True, "reason": "skipped", "message": f"Skipped because {payload}"}+ }+ return result+++def _incomplete_result(payload: str) -> ValidationResult:+ result = ValidationResult(validator_name=f"Scanner {payload}", validator_description="External scanner")+ result.mark_scan_incomplete(f"tool {payload}")+ return result++[email protected]("payload", PAYLOADS)+def test_cli_report_renders_untrusted_text_literally(payload: str) -> None:+ console = _recording_console()+ reporter = CLIReporter(console=console)+ results = [+ _quality_result(payload),+ _rubric_result(payload),+ _finding_result(payload),+ _agent_eval_result(payload),+ _advisory_skip_result(payload),+ _incomplete_result(payload),+ ]++ for result in results:+ reporter.print(result) # every per-result section+ reporter.print_all(results) # summary table, failure details, overall verdict++ html = console.export_html(clear=False)+ text = console.export_text()+ expected = [+ f"alpha {payload}", # quality table skill names+ f"beta {payload}",+ f"Quality of {payload}", # validator description+ f"Judge summary {payload}", # rubric summary (LLM)+ f"Judge notes {payload}", # rubric notes (LLM)+ f"Rubric error {payload}",+ f"Finding message {payload}",+ f"skills/{payload}/SKILL.md",+ f"content {payload}",+ f"Suggested fix {payload}",+ f"https://harbor.example.test/jobs/{payload}",+ f"https://harbor.example.test/analysis/{payload}",+ f"custom {payload}".title(), # evaluator (custom grader metric) name+ f"Recommendation {payload}",+ f"View trial {payload}", # evidence link text+ f"https://harbor.example.test/trials/{payload}",+ f"dim {payload}".title(), # judge insight dimension+ f"PASS {payload}", # judge insight verdict string+ f"Judge reasoning {payload}",+ f"Skipped because {payload}", # advisory skip provenance+ f"[Scanner {payload}]", # result header+ f"tool {payload} did not complete", # incomplete scanners+ ]+ assert [snippet for snippet in expected if snippet not in text] == []+ assert 'href="http://x"' not in html+++def test_escaping_does_not_disturb_trusted_status_markup() -> None:+ console = _recording_console()+ reporter = CLIReporter(console=console)+ passing = ValidationResult(validator_name="SCHEMA", validator_description="Schema checks")+ failing = _finding_result("[/x]")++ reporter.print_all([passing, failing])++ text = console.export_text()+ assert "[SCHEMA]" not in text # passing results only appear in the summary table+ assert "[SECURITY]" in text+ assert "[FAIL] Validation failed" in text+ assert "\\[" not in text # no escape backslashes leak into the output++[email protected](("payload", "visible"), CONTROL_PAYLOADS.items())+def test_cli_report_strips_terminal_control_sequences(payload: str, visible: str) -> None:+ console = _terminal_console()+ reporter = CLIReporter(console=console)+ results = [+ _rubric_result(payload),+ _finding_result(payload),+ _agent_eval_result(payload),+ _advisory_skip_result(payload),+ ]++ for result in results:+ reporter.print(result)+ reporter.print_all(results)++ text = _terminal_text(console.file.getvalue())+ for label in ("Judge notes", "Finding message", "Judge reasoning", "Skipped because"):+ assert f"{label} {visible}".rstrip() in text+++def test_trailing_backslashes_render_literally(monkeypatch: pytest.MonkeyPatch) -> None:+ from skillevaluator import cli as cli_module++ console = _recording_console(width=250)+ monkeypatch.setattr(cli_module, "console", console)+ reporter = CLIReporter(console=console)+ errored = ValidationResult(validator_name="SCHEMA", validator_description="Schema checks")+ errored.add_error("Output directory is not writable: D:\\reports\\")+ # The judge explanation is cut at 80 characters, here right after a backslash.+ prefix = "Agent wrote its report to C:\\Users\\runner\\AppData\\Local\\Temp\\"+ explanation = prefix + "r" * (79 - len(prefix)) + "\\ and then exited without writing result.json"+ judged = ValidationResult(validator_name="AGENT_EVAL", validator_description="Live agent evaluation")+ judged.metadata["agent_eval"] = {+ "verdict": "pass",+ "composite_lift": 0.1,+ "insights": {"goal": {"score": "PASS", "explanation": explanation}},+ }++ cli_module._print_run_banner(PureWindowsPath("\\\\fileserver\\skills\\"), "skill", None)+ reporter.print(errored)+ reporter.print(judged)++ lines = [line.rstrip(" │") for line in console.export_text().splitlines()]+ assert "Target: \\\\fileserver\\skills\\" in lines+ assert " • Output directory is not writable: D:\\reports\\" in lines+ assert any(line.endswith(f" {explanation[:80]}") for line in lines)+++def test_cli_reporter_consoles_print_emoji_codes_literally(+ capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setenv("COLUMNS", "250")+ results = [+ _rubric_result(EMOJI_TEXT),+ _finding_result(EMOJI_TEXT),+ _agent_eval_result(EMOJI_TEXT),+ _advisory_skip_result(EMOJI_TEXT),+ ]+ reporter = CLIReporter() # the reporter's own default console++ for result in results:+ reporter.print(result)+ reporter.print_all(results)+ outputs = [capsys.readouterr().out, reporter.render(results[1]), reporter.render_all(results)]++ for output in outputs:+ text = _terminal_text(output)+ assert [glyph for glyph in EMOJI_GLYPHS if glyph in text] == []+ assert f"Finding message {EMOJI_TEXT}" in text+ assert f"Suggested fix {EMOJI_TEXT}" in text+ assert f"Judge notes {EMOJI_TEXT}" in outputs[0]+ assert f"Skipped because {EMOJI_TEXT}" in outputs[0]+++def test_command_consoles_print_emoji_codes_literally(+ capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator import cli as cli_module+ from skillevaluator.tier3 import commands as tier3_commands++ monkeypatch.setenv("COLUMNS", "200")++ # The run banner prints through the tier1 command console.+ cli_module._print_run_banner(PurePosixPath("/work/root:x:0:0:/skill"), "skill", "custom :key:")+ tier3_commands._print_validate_results(+ PurePosixPath("/work/demo:x:"),+ [SimpleNamespace(status="ok", path="evals/evals.json", message="found :white_check_mark:")],+ )++ text = _terminal_text(capsys.readouterr().out)+ assert [glyph for glyph in EMOJI_GLYPHS if glyph in text] == []+ assert "Target: /work/root:x:0:0:/skill" in text+ assert "Profile: custom :key:" in text+ assert "Validate: demo:x:/evals/" in text+ assert "found :white_check_mark:" in text+++def test_log_handlers_render_untrusted_text_literally(monkeypatch: pytest.MonkeyPatch) -> None:+ from skillevaluator import logging_config++ console = _recording_console()+ monkeypatch.setattr(logging_config, "console", console)+ root = logging.getLogger()+ # setup_logging() rewires the root logger; keep pytest's handlers and levels intact.+ monkeypatch.setattr(root, "handlers", list(root.handlers))+ monkeypatch.setattr(root, "level", root.level)+ for name in ("httpx", "urllib3"):+ noisy = logging.getLogger(name)+ monkeypatch.setattr(noisy, "level", noisy.level)+ named = logging.getLogger("skillevaluator.tests.rich_markup_escaping")+ monkeypatch.setattr(named, "handlers", [])+ monkeypatch.setattr(named, "propagate", named.propagate)+ monkeypatch.setattr(named, "level", named.level)++ logging_config.get_logger(named.name)+ logging_config.setup_logging()+ handlers = [*named.handlers, *(h for h in root.handlers if isinstance(h, RichHandler))]+ assert len(handlers) == 2++ for handler in handlers:+ for payload in PAYLOADS:+ record = logging.LogRecord(+ named.name, logging.WARNING, __file__, 1, "Could not read %s", (f"skills/{payload}/SKILL.md",), None+ )+ handler.handle(record)++ for index, payload in enumerate(CONTROL_PAYLOADS):+ record = logging.LogRecord(+ named.name, logging.WARNING, __file__, 1, "Tool %d said: %s", (index, f"done{payload}"), None+ )+ handler.handle(record)++ text = console.export_text()+ for payload in PAYLOADS:+ assert text.count(f"Could not read skills/{payload}/SKILL.md") == 2+ lines = [line.rstrip() for line in text.splitlines()]+ for index, visible in enumerate(CONTROL_PAYLOADS.values()):+ assert sum(line.endswith(f"Tool {index} said: done{visible}") for line in lines) == 2+ assert _CONTROL.search(text) is None++[email protected]("payload", PATH_PAYLOADS)+def test_validate_run_banner_renders_target_and_profile_literally(+ payload: str, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator import cli as cli_module++ console = _recording_console()+ monkeypatch.setattr(cli_module, "console", console)+ target = Path("/work/skills") / payload++ cli_module._print_run_banner(target, "skill", f"custom {payload}")++ text = console.export_text()+ assert f"Target: {target}" in text+ assert f"Profile: custom {payload}" in text++[email protected]("payload", PATH_PAYLOADS)+def test_report_path_announcement_renders_literally(+ payload: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator.tier1 import commands as tier1_commands++ console = _recording_console(width=1000)+ monkeypatch.setattr(tier1_commands, "console", console)+ output_dir = tmp_path / payload+ output_dir.mkdir(parents=True)+ result = ValidationResult(validator_name="SCHEMA", validator_description="Schema checks")++ tier1_commands.emit_reports([result], report_formats=("json",), output_dir=output_dir)++ assert f"json report: {output_dir / 'skillevaluator-output.json'}" in console.export_text()++[email protected]("name", NAME_PAYLOADS)+def test_tier3_validate_panel_renders_skill_name_literally(+ name: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator.tier3 import commands as tier3_commands++ console = _recording_console()+ monkeypatch.setattr(tier3_commands, "console", console)++ tier3_commands._print_validate_results(+ tmp_path / name, [SimpleNamespace(status="ok", path="evals/evals.json", message="found")]+ )++ assert f"Validate: {name}/evals/" in console.export_text()+++def test_tier3_validate_panel_strips_terminal_control_sequences(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator.tier3 import commands as tier3_commands++ console = _terminal_console()+ monkeypatch.setattr(tier3_commands, "console", console)++ tier3_commands._print_validate_results(+ tmp_path / "demo",+ [SimpleNamespace(status="error", path="evals/\x1b[2Jevals.json", message="bad\x1bc value")],+ )++ text = _terminal_text(console.file.getvalue())+ assert "evals/evals.json" in text+ assert "bad value" in text++[email protected]("payload", PATH_PAYLOADS)+def test_tier3_grader_starter_renders_created_path_literally(+ payload: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator.tier3 import commands as tier3_commands++ console = _recording_console(width=1000)+ monkeypatch.setattr(tier3_commands, "console", console)+ skill_path = tmp_path / payload+ skill_path.mkdir(parents=True)++ assert (+ tier3_commands.init_custom_grader(+ skill_path, language="python", mode="custom_only", force=False, no_config=True+ )+ == 0+ )+ assert (+ tier3_commands.init_custom_grader(+ skill_path, language="python", mode="custom_only", force=False, no_config=True+ )+ == 1+ )++ text = console.export_text()+ grader_path = skill_path.resolve() / "evals" / "grader.py"+ assert f"custom grader starter at {grader_path}" in text+ assert f"custom grader already exists: {grader_path}." in text+++def test_tier3_compare_renders_agent_and_summary_values_literally(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator.tier3 import commands as tier3_commands++ console = _recording_console()+ monkeypatch.setattr(tier3_commands, "console", console)+ skill_name, agent = f"[green]PASS{_EMOJI_NAME_SUFFIX}", "[bold]evil"+ skill_path = tmp_path / skill_name+ skill_path.mkdir()+ results_root = tmp_path / "results"+ run_dir = results_root / skill_name / "20260709_010000"+ summary_dir = run_dir / agent / "with-skill"+ summary_dir.mkdir(parents=True)+ (summary_dir / "summary.json").write_text(+ json.dumps(+ {+ "execution_status": "succeeded",+ "scores": {"security": 0.9, "custom: grader\x1b[2J": 0.5},+ "num_trials": "[/x]",+ }+ ),+ encoding="utf-8",+ )+ (run_dir / "run_config.json").write_text("{}", encoding="utf-8")+ (run_dir / "result.json").write_text(json.dumps({"run_id": run_dir.name, "agents": {}}), encoding="utf-8")++ assert tier3_commands.compare_results(skill_path, results_dir=results_root) == 0++ text = console.export_text()+ assert f"Skill Evaluation - {skill_name}" in text+ assert f"{agent:<16s} 20260709_010000 (Harbor, [/x] trials)" in text+ assert "custom: grader" in text+ assert "\x1b" not in text+++def test_tier3_doctor_renders_provider_and_prerequisite_errors_literally(monkeypatch: pytest.MonkeyPatch) -> None:+ from skillevaluator.provider_config import ProviderConfigurationError+ from skillevaluator.tier3 import commands as tier3_commands++ console = _recording_console()+ monkeypatch.setattr(tier3_commands, "console", console)++ def _unconfigured_provider():+ raise ProviderConfigurationError("provider [/x] is not configured")++ monkeypatch.setattr(tier3_commands, "resolve_llm_provider", _unconfigured_provider)+ monkeypatch.setattr(tier3_commands, "_check_prerequisites", lambda **_kwargs: ["docker: [bold]evil[/bold]"])++ assert tier3_commands.doctor(agents=None, env_mode="local") == 1++ text = console.export_text()+ assert "provider [/x] is not configured" in text+ assert "docker: [bold]evil[/bold]" in text+++def test_tier3_doctor_strips_terminal_control_sequences(monkeypatch: pytest.MonkeyPatch) -> None:+ from skillevaluator.provider_config import ProviderConfigurationError+ from skillevaluator.tier3 import commands as tier3_commands++ console = _terminal_console()+ monkeypatch.setattr(tier3_commands, "console", console)++ def _unconfigured_provider():+ raise ProviderConfigurationError("provider\x1b[1A\x1b[2K is not configured\x1bc")++ monkeypatch.setattr(tier3_commands, "resolve_llm_provider", _unconfigured_provider)+ monkeypatch.setattr(+ tier3_commands,+ "_check_prerequisites",+ lambda **_kwargs: ["docker: \x1b]8;;https://evil.test/\x1b\\see docs\x1b]8;;\x1b\\"],+ )++ assert tier3_commands.doctor(agents=None, env_mode="local") == 1++ text = _terminal_text(console.file.getvalue())+ assert "provider is not configured" in text+ assert "docker: see docs" in text+++def test_harbor_findings_report_renders_skill_and_model_literally(+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]+) -> None:+ from skillevaluator.tier3.harbor import report++ monkeypatch.setenv("COLUMNS", "200")+ scores = {+ "security": 1.0,+ "skill_execution": 1.0,+ "skill_efficiency": 1.0,+ "accuracy": 1.0,+ "goal_accuracy": 0.1,+ "behavior_check": 1.0,+ }+ condition_dir = tmp_path / "codex" / "with-skill"+ trial_dir = condition_dir / "trials" / "case-001"+ trial_dir.mkdir(parents=True)+ (condition_dir / "summary.json").write_text(+ json.dumps(+ {+ "agent": "codex",+ "scores": scores,+ "execution_status": "succeeded",+ "execution_errors": [],+ "expected_attempts": 1,+ "scored_attempts": 1,+ }+ ),+ encoding="utf-8",+ )+ reward = {+ "entry_id": "case-001",+ **scores,+ "details": {"goal_accuracy": {"reason": "no result [/x]\x1b]8;;https://evil.test/\x1b\\"}},+ }+ (trial_dir / "reward.json").write_text(json.dumps(reward), encoding="utf-8")+ monkeypatch.setattr(+ report,+ "_generate_suggestions_structured",+ lambda _skill, _findings, _rewards: [+ {"suggestion": "Write the result file [/x]\x1bc", "dimension": "goal_accuracy", "evidence_refs": []}+ ],+ )+ skill_name = "skill [bold]evil[/bold] :x:"++ report.display_findings_report(+ {+ "agents": {+ "codex": {"execution_status": "succeeded", "model": "gpt-[/x]:key:", "with_skill": scores},+ "opencode": {"execution_status": "succeeded", "model": "m", "with_skill": {"security": 0.1}},+ }+ },+ skill_name,+ ["codex", "opencode"],+ tmp_path,+ )++ output = _terminal_text(capsys.readouterr().out)+ assert "combination for your skill: codex / gpt-[/x]:key:" in output+ assert f"{skill_name} / codex / gpt-[/x]:key: — Findings" in output+ assert "no result [/x]" in output+ assert "Write the result file [/x]" in output++[email protected]("name", NAME_PAYLOADS)+def test_rubric_progress_status_renders_skill_name_literally(+ name: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch+) -> None:+ from skillevaluator.validators.rubric_eval import RubricEvalValidator, RubricJudge++ statuses: list[str] = []++ class _RecordingStatus:+ def __init__(self, status: str, **_kwargs: object) -> None:+ statuses.append(status)++ def __enter__(self) -> _RecordingStatus:+ return self++ def __exit__(self, *_exc: object) -> None:+ return None++ monkeypatch.setattr("rich.status.Status", _RecordingStatus)+ skill_dir = tmp_path / name+ skill_dir.mkdir()+ (skill_dir / "SKILL.md").write_text("---\nname: demo\ndescription: test\n---\n# Demo\n", encoding="utf-8")++ with patch.object(RubricJudge, "process", return_value={}):+ RubricEvalValidator().validate(skill_dir)++ assert all(isinstance(status, Text) for status in statuses)+ assert [status.plain for status in statuses] == [f"Evaluating {name} with LLM judge..."]diff --git a/tests/utils/test_rich_markup.py b/tests/utils/test_rich_markup.pynew file mode 100644index 00000000..3d580b3d--- /dev/null+++ b/tests/utils/test_rich_markup.py@@ -0,0 +1,117 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""``escape_markup`` must make any text render literally inside Rich markup."""++from __future__ import annotations++import io+import itertools+import time++import pytest+from rich.console import Console++from skillevaluator.utils.rich_markup import escape_markup++# Places escaped text can sit in markup, with the expected render. A trailing+# backslash renders differently depending on what follows it.+CONTEXTS = {+ "between tags": ("[green]<[/green]{}[green]>[/green]", "<{}>"),+ "end of markup": ("pre {}", "pre {}"),+ "before plain text": ("pre {} post", "pre {} post"),+ "before a closing tag": ("[dim]{}[/dim]", "{}"),+ "before an implicit close": ("[dim]{}[/]", "{}"),+ "inside a style": ("[bold]<{}>[/bold]", "<{}>"),+ "before a non-tag bracket": ("{}[INCOMPLETE]", "{}[INCOMPLETE]"),+}+++def _render(markup: str) -> str:+ console = Console(file=io.StringIO(), record=True, width=10_000, highlight=False)+ console.print(markup, soft_wrap=True)+ return console.export_text()[:-1]+++ "text",+ [+ "[/x]",+ "[bold]evil[/bold]",+ "[link=http://x]y[/link]",+ "[#fff]x",+ "[@click]y",+ # Windows paths: Rich turns a plain-text "\[" into "[", dropping the separator.+ r"\work\skills\[\x]",+ r"C:\Users\runner\Temp\[abc]\report.json",+ r"\\server\share\[a]",+ r"\[/x]",+ r"[a\[b]",+ "[a[b]",+ "[[b]]",+ "[",+ "]",+ "x[",+ "[]",+ "plain",+ # Trailing backslashes must stay literal and must not escape the closing tag.+ "C:\\dir\\",+ "a\\\\",+ "a\\\\\\",+ ],+)+def test_escaped_text_renders_literally_between_tags(text: str) -> None:+ assert _render(f"[green]<[/green]{escape_markup(text)}[green]>[/green]") == f"<{text}>"+++def test_escaped_text_renders_literally_between_plain_text() -> None:+ text = r"C:\a\[b]\c [/x] \\[bold]"+ assert _render(f"pre {escape_markup(text)} post") == f"pre {text} post"++[email protected]("text", ["C:\\", "D:\\reports\\", "\\\\fileserver\\skills\\", "a\\\\\\"])[email protected]("context", CONTEXTS)+def test_trailing_backslashes_render_literally(context: str, text: str) -> None:+ template, expected = CONTEXTS[context]+ assert _render(template.replace("{}", escape_markup(text))) == expected.replace("{}", text)++[email protected]("context", CONTEXTS)+def test_every_short_string_renders_literally(context: str) -> None:+ template, expected = CONTEXTS[context]+ alphabet = ["[", "]", "\\", "/", "a", "#", "="]+ for length in range(6):+ for chars in itertools.product(alphabet, repeat=length):+ text = "".join(chars)+ assert _render(template.replace("{}", escape_markup(text))) == expected.replace("{}", text), text+++ ("text", "expected"),+ [+ ("clear\x1b[2Jscreen", "clearscreen"),+ ("up\x1b[1A\x1b[2Kerase", "uperase"),+ ("\x1b]8;;https://evil.test/\x1b\\see docs\x1b]8;;\x1b\\", "see docs"),+ ("title\x1b]0;forged\x07 set", "title set"),+ ("bad\x1bc thing", "bad thing"),+ ("charset\x1b(B reset", "charset reset"),+ ("eight-bit\x9b2J csi", "eight-bit2J csi"),+ ("nul\x00 bel\x07 del\x7f", "nul bel del"),+ ("crlf\r\nline", "crlf\nline"),+ ("tab\tand\nnewline", "tab\tand\nnewline"),+ ],+)+def test_terminal_controls_are_stripped(text: str, expected: str) -> None:+ assert escape_markup(text) == expected+++def test_controls_are_stripped_before_brackets_are_escaped() -> None:+ text = "[\x1b[0m/x]"+ assert _render(f"[green]<[/green]{escape_markup(text)}[green]>[/green]") == "<[/x]>"++[email protected]("unit", ["\\", "[a", "\\[", "[a]", "\x1b[", "\x1b]8;;"])+def test_long_input_is_escaped_in_linear_time(unit: str) -> None:+ started = time.perf_counter()+ escape_markup(unit * 200_000)+ assert time.perf_counter() - started < 1.0