Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
Fix CLI output on legacy Windows code pages (#449)
Efeca22a918f21c
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex c8f17392..ffcc9dee 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -410,6 +410,7 @@ jobs:tests/validators/test_rubric_eval.pytests/reportingtests/test_cli.py+ tests/test_cli_stdout_encoding.pytests/test_harbor_output_provenance.pytests/test_harbor_runner_status.pytests/test_harbor_runtime_skill_isolation.pydiff --git a/CHANGELOG.md b/CHANGELOG.mdindex 66910748..9b12b807 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -454,6 +454,11 @@ All notable changes to SkillEvaluator are documented in this file.### Fixed+- Configure CLI stdout to replace characters that its encoding cannot+ represent, so reports and root help complete on legacy Windows code pages.+ JSON dataset previews and validation output preserve Unicode values, using+ readable characters when the output encoding supports them and escapes otherwise.+- `skillevaluator view` no longer asks the LLM again when it regenerates amissing `report.html` for a Tier 3 run that saved its report feedback in`result.json`. The regenerated report shows the conclusions,diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.pyindex 3727f80b..4ede3925 100644--- a/src/skillevaluator/cli.py+++ b/src/skillevaluator/cli.py@@ -9,6 +9,7 @@ import copyimport jsonimport loggingimport math+import sysfrom concurrent.futures import ProcessPoolExecutor, as_completedfrom contextvars import ContextVarfrom datetime import UTC, datetime@@ -63,6 +64,7 @@ from skillevaluator.tier1.commands import ()from skillevaluator.tier3_environments import all_env_modesfrom skillevaluator.tier_group import TierGroup+from skillevaluator.utils.json_output import json_needs_asciifrom skillevaluator.utils.path_security import resolve_repo_context_rootfrom skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controlsfrom skillevaluator.utils.tier2_paths import (@@ -262,8 +264,17 @@ _TOP_LEVEL_COMMAND_HELP_GROUPS = ()+class _EncodingSafeGroup(RichGroup):+ """Configure output before Click parses eager options such as --help."""++ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:+ if hasattr(sys.stdout, "reconfigure"):+ sys.stdout.reconfigure(errors="replace")+ return super().parse_args(ctx, args)++@click.group(- cls=RichGroup,+ cls=_EncodingSafeGroup,context_settings=CONTEXT_SETTINGS,help_command_groups=_TOP_LEVEL_COMMAND_HELP_GROUPS,show_banner=True,@@ -4035,7 +4046,8 @@ def tier3_migrate_dataset(source: Path, skill_name: str | None, destination: Patexcept (OSError, ValueError, TypeError) as exc:raise click.ClickException(str(exc)) from excif destination is None:- click.echo(json.dumps(result.dataset, indent=2, ensure_ascii=False))+ ensure_ascii = json_needs_ascii(result.dataset, getattr(sys.stdout, "encoding", None))+ click.echo(json.dumps(result.dataset, indent=2, ensure_ascii=ensure_ascii))else:click.echo(f"Wrote {len(result.dataset['evals'])} cases to {destination}; source preserved.")diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.pyindex 4423e527..d82d5113 100644--- a/src/skillevaluator/tier3/commands.py+++ b/src/skillevaluator/tier3/commands.py@@ -80,6 +80,7 @@ from skillevaluator.tier3.results_location import (resolve_results_root,)from skillevaluator.tier3.toml_utils import toml_quote+from skillevaluator.utils.json_output import json_needs_asciifrom skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controls# emoji=False: ":name:" codes in untrusted text must print literally.@@ -993,12 +994,11 @@ def validate_evals(skill_path: Path, *, as_json: bool, strict: bool, harbor_contcoverage = dataset_coverage_result(skill_path)if as_json:+ data = [{"path": r.path, "status": r.status, "message": r.message} for r in results]+ data += [coverage] if coverage is not None else []console.print_json(- json.dumps(- [{"path": r.path, "status": r.status, "message": r.message} for r in results]- + ([coverage] if coverage is not None else []),- indent=2,- )+ data=data,+ ensure_ascii=json_needs_ascii(data, getattr(console.file, "encoding", None)),)else:_print_validate_results(skill_path, results)diff --git a/src/skillevaluator/utils/json_output.py b/src/skillevaluator/utils/json_output.pynew file mode 100644index 00000000..abaed4f1--- /dev/null+++ b/src/skillevaluator/utils/json_output.py@@ -0,0 +1,17 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""Keep JSON readable when the output encoding can represent its values."""++from __future__ import annotations++import json+++def json_needs_ascii(data: object, encoding: str | None) -> bool:+ """Escape JSON only when strict encoding would otherwise lose characters."""+ try:+ json.dumps(data, ensure_ascii=False).encode(encoding or "utf-8", errors="strict")+ except UnicodeEncodeError:+ return True+ return Falsediff --git a/tests/test_cli_stdout_encoding.py b/tests/test_cli_stdout_encoding.pynew file mode 100644index 00000000..6db0d25c--- /dev/null+++ b/tests/test_cli_stdout_encoding.py@@ -0,0 +1,136 @@+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.+# SPDX-License-Identifier: Apache-2.0++"""CLI output remains available without corrupting machine-readable values."""++from __future__ import annotations++import json+import os+import subprocess+import sys+from io import BytesIO, StringIO, TextIOWrapper+from types import SimpleNamespace++import pytest+from rich.console import Console++from skillevaluator import cli as cli_module+from skillevaluator.utils.json_output import json_needs_ascii++_ENCODINGS = ("cp936", "cp1252", "ascii", "utf-8")+_UNICODE = "Résumé 中文 • 😀"+_JSON_CASES = [(encoding, _UNICODE, encoding == "utf-8") for encoding in _ENCODINGS] + [+ ("cp936", "中文", True),+ ("cp1252", "Résumé café", True),+ ("ascii", "plain ASCII", True),+]++[email protected]("newline", (None, "\n", "\r\n"))[email protected]("encoding", _ENCODINGS)+def test_cli_replaces_characters_not_supported_by_stdout_encoding(monkeypatch, encoding, newline) -> None:+ output = BytesIO()+ stdout = TextIOWrapper(output, encoding=encoding, errors="strict", newline=newline)+ monkeypatch.setattr(cli_module.sys, "stdout", stdout)++ with cli_module.cli.make_context("skillevaluator", ["tier3"]):+ Console(file=stdout, force_terminal=False).print(f" {_UNICODE} Errors: 1")+ stdout.flush()++ assert stdout.errors == "replace"+ assert output.getvalue().decode(encoding).replace("\r\n", "\n") == f" {_UNICODE} Errors: 1\n".encode(+ encoding, errors="replace"+ ).decode(encoding)++[email protected]("encoding", _ENCODINGS)+def test_root_help_before_callback_supports_code_pages(encoding) -> None:+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ "from skillevaluator.cli import cli; import sys; "+ "cli.main(standalone_mode=False); assert sys.stdout.errors == 'replace'",+ "--help",+ ],+ env={**os.environ, "PYTHONIOENCODING": f"{encoding}:strict", "NO_COLOR": "1"},+ capture_output=True,+ check=False,+ )+ assert result.returncode == 0, result.stderr+ assert b"Usage:" in result.stdout+++def test_stdout_without_reconfigure_is_supported(monkeypatch) -> None:+ stdout = StringIO()+ monkeypatch.setattr(cli_module.sys, "stdout", stdout)+ cli_module.cli.main(["--help"], standalone_mode=False)+ assert "Usage:" in stdout.getvalue()++[email protected]("encoding,text,readable", _JSON_CASES)+def test_migration_preview_preserves_unicode(encoding, text, readable, tmp_path) -> None:+ source = tmp_path / "legacy.json"+ source.write_text(+ json.dumps([{"id": 1, "question": text, "ground_truth": text, "expected_skill": "demo"}]),+ encoding="utf-8",+ )+ original = source.read_bytes()+ result = subprocess.run(+ [+ sys.executable,+ "-c",+ "from skillevaluator.cli import cli; cli()",+ "tier3",+ "migrate-dataset",+ str(source),+ "--skill-name",+ "demo",+ ],+ env={**os.environ, "PYTHONIOENCODING": f"{encoding}:strict", "NO_COLOR": "1"},+ capture_output=True,+ check=False,+ )+ assert result.returncode == 0, result.stderr+ rendered = result.stdout.decode(encoding)+ case = json.loads(rendered)["evals"][0]+ assert case["prompt"] == text+ assert case["expected_output"] == text+ if readable:+ assert text in rendered+ assert source.read_bytes() == original++[email protected]("encoding,text,readable", _JSON_CASES)+def test_validate_json_preserves_unicode(monkeypatch, encoding, text, readable, tmp_path) -> None:+ from skillevaluator.tier3 import commands++ output = BytesIO()+ stdout = TextIOWrapper(output, encoding=encoding, errors="strict")+ monkeypatch.setattr(cli_module.sys, "stdout", stdout)+ monkeypatch.setattr(commands, "console", Console(file=stdout, force_terminal=False, width=120))+ monkeypatch.setattr(+ commands,+ "validate_skillevaluators",+ lambda _: [+ SimpleNamespace(path=text, status="warning", message=text),+ ],+ )+ monkeypatch.setattr(commands, "dataset_coverage_result", lambda _: None)+ monkeypatch.setattr(cli_module, "setup_logging", lambda **_kwargs: None)++ with pytest.raises(SystemExit) as exc:+ cli_module.cli.main(["tier3", "validate", str(tmp_path), "--json"], standalone_mode=False)+ assert exc.value.code == 0+ stdout.flush()+ rendered = output.getvalue().decode(encoding)+ assert json.loads(rendered) == [+ {"path": text, "status": "warning", "message": text},+ ]+ if readable:+ assert text in rendered+++def test_json_without_stream_encoding_remains_readable() -> None:+ assert json_needs_ascii({"value": _UNICODE}, None) is False