Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

Fix CLI output on legacy Windows code pages (#449)

Efeca22a918f21c
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c8f17392..ffcc9dee 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -410,6 +410,7 @@ jobs:
tests/validators/test_rubric_eval.py
tests/reporting
tests/test_cli.py
+ tests/test_cli_stdout_encoding.py
tests/test_harbor_output_provenance.py
tests/test_harbor_runner_status.py
tests/test_harbor_runtime_skill_isolation.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 66910748..9b12b807 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -454,6 +454,11 @@ All notable changes to SkillEvaluator are documented in this file.
### Fixed
+- Configure CLI stdout to replace characters that its encoding cannot
+ represent, so reports and root help complete on legacy Windows code pages.
+ JSON dataset previews and validation output preserve Unicode values, using
+ readable characters when the output encoding supports them and escapes otherwise.
+
- `skillevaluator view` no longer asks the LLM again when it regenerates a
missing `report.html` for a Tier 3 run that saved its report feedback in
`result.json`. The regenerated report shows the conclusions,
diff --git a/src/skillevaluator/cli.py b/src/skillevaluator/cli.py
index 3727f80b..4ede3925 100644
--- a/src/skillevaluator/cli.py
+++ b/src/skillevaluator/cli.py
@@ -9,6 +9,7 @@ import copy
import json
import logging
import math
+import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from contextvars import ContextVar
from datetime import UTC, datetime
@@ -63,6 +64,7 @@ from skillevaluator.tier1.commands import (
)
from skillevaluator.tier3_environments import all_env_modes
from skillevaluator.tier_group import TierGroup
+from skillevaluator.utils.json_output import json_needs_ascii
from skillevaluator.utils.path_security import resolve_repo_context_root
from skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controls
from skillevaluator.utils.tier2_paths import (
@@ -262,8 +264,17 @@ _TOP_LEVEL_COMMAND_HELP_GROUPS = (
)
+class _EncodingSafeGroup(RichGroup):
+ """Configure output before Click parses eager options such as --help."""
+
+ def parse_args(self, ctx: click.Context, args: list[str]) -> list[str]:
+ if hasattr(sys.stdout, "reconfigure"):
+ sys.stdout.reconfigure(errors="replace")
+ return super().parse_args(ctx, args)
+
+
@click.group(
- cls=RichGroup,
+ cls=_EncodingSafeGroup,
context_settings=CONTEXT_SETTINGS,
help_command_groups=_TOP_LEVEL_COMMAND_HELP_GROUPS,
show_banner=True,
@@ -4035,7 +4046,8 @@ def tier3_migrate_dataset(source: Path, skill_name: str | None, destination: Pat
except (OSError, ValueError, TypeError) as exc:
raise click.ClickException(str(exc)) from exc
if destination is None:
- click.echo(json.dumps(result.dataset, indent=2, ensure_ascii=False))
+ ensure_ascii = json_needs_ascii(result.dataset, getattr(sys.stdout, "encoding", None))
+ click.echo(json.dumps(result.dataset, indent=2, ensure_ascii=ensure_ascii))
else:
click.echo(f"Wrote {len(result.dataset['evals'])} cases to {destination}; source preserved.")
diff --git a/src/skillevaluator/tier3/commands.py b/src/skillevaluator/tier3/commands.py
index 4423e527..d82d5113 100644
--- a/src/skillevaluator/tier3/commands.py
+++ b/src/skillevaluator/tier3/commands.py
@@ -80,6 +80,7 @@ from skillevaluator.tier3.results_location import (
resolve_results_root,
)
from skillevaluator.tier3.toml_utils import toml_quote
+from skillevaluator.utils.json_output import json_needs_ascii
from skillevaluator.utils.rich_markup import escape_markup, strip_terminal_controls
# emoji=False: ":name:" codes in untrusted text must print literally.
@@ -993,12 +994,11 @@ def validate_evals(skill_path: Path, *, as_json: bool, strict: bool, harbor_cont
coverage = dataset_coverage_result(skill_path)
if as_json:
+ data = [{"path": r.path, "status": r.status, "message": r.message} for r in results]
+ data += [coverage] if coverage is not None else []
console.print_json(
- json.dumps(
- [{"path": r.path, "status": r.status, "message": r.message} for r in results]
- + ([coverage] if coverage is not None else []),
- indent=2,
- )
+ data=data,
+ ensure_ascii=json_needs_ascii(data, getattr(console.file, "encoding", None)),
)
else:
_print_validate_results(skill_path, results)
diff --git a/src/skillevaluator/utils/json_output.py b/src/skillevaluator/utils/json_output.py
new file mode 100644
index 00000000..abaed4f1
--- /dev/null
+++ b/src/skillevaluator/utils/json_output.py
@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Keep JSON readable when the output encoding can represent its values."""
+
+from __future__ import annotations
+
+import json
+
+
+def json_needs_ascii(data: object, encoding: str | None) -> bool:
+ """Escape JSON only when strict encoding would otherwise lose characters."""
+ try:
+ json.dumps(data, ensure_ascii=False).encode(encoding or "utf-8", errors="strict")
+ except UnicodeEncodeError:
+ return True
+ return False
diff --git a/tests/test_cli_stdout_encoding.py b/tests/test_cli_stdout_encoding.py
new file mode 100644
index 00000000..6db0d25c
--- /dev/null
+++ b/tests/test_cli_stdout_encoding.py
@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""CLI output remains available without corrupting machine-readable values."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+from io import BytesIO, StringIO, TextIOWrapper
+from types import SimpleNamespace
+
+import pytest
+from rich.console import Console
+
+from skillevaluator import cli as cli_module
+from skillevaluator.utils.json_output import json_needs_ascii
+
+_ENCODINGS = ("cp936", "cp1252", "ascii", "utf-8")
+_UNICODE = "Résumé 中文 • 😀"
+_JSON_CASES = [(encoding, _UNICODE, encoding == "utf-8") for encoding in _ENCODINGS] + [
+ ("cp936", "中文", True),
+ ("cp1252", "Résumé café", True),
+ ("ascii", "plain ASCII", True),
+]
+
+
[email protected]("newline", (None, "\n", "\r\n"))
[email protected]("encoding", _ENCODINGS)
+def test_cli_replaces_characters_not_supported_by_stdout_encoding(monkeypatch, encoding, newline) -> None:
+ output = BytesIO()
+ stdout = TextIOWrapper(output, encoding=encoding, errors="strict", newline=newline)
+ monkeypatch.setattr(cli_module.sys, "stdout", stdout)
+
+ with cli_module.cli.make_context("skillevaluator", ["tier3"]):
+ Console(file=stdout, force_terminal=False).print(f" {_UNICODE} Errors: 1")
+ stdout.flush()
+
+ assert stdout.errors == "replace"
+ assert output.getvalue().decode(encoding).replace("\r\n", "\n") == f" {_UNICODE} Errors: 1\n".encode(
+ encoding, errors="replace"
+ ).decode(encoding)
+
+
[email protected]("encoding", _ENCODINGS)
+def test_root_help_before_callback_supports_code_pages(encoding) -> None:
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "from skillevaluator.cli import cli; import sys; "
+ "cli.main(standalone_mode=False); assert sys.stdout.errors == 'replace'",
+ "--help",
+ ],
+ env={**os.environ, "PYTHONIOENCODING": f"{encoding}:strict", "NO_COLOR": "1"},
+ capture_output=True,
+ check=False,
+ )
+ assert result.returncode == 0, result.stderr
+ assert b"Usage:" in result.stdout
+
+
+def test_stdout_without_reconfigure_is_supported(monkeypatch) -> None:
+ stdout = StringIO()
+ monkeypatch.setattr(cli_module.sys, "stdout", stdout)
+ cli_module.cli.main(["--help"], standalone_mode=False)
+ assert "Usage:" in stdout.getvalue()
+
+
[email protected]("encoding,text,readable", _JSON_CASES)
+def test_migration_preview_preserves_unicode(encoding, text, readable, tmp_path) -> None:
+ source = tmp_path / "legacy.json"
+ source.write_text(
+ json.dumps([{"id": 1, "question": text, "ground_truth": text, "expected_skill": "demo"}]),
+ encoding="utf-8",
+ )
+ original = source.read_bytes()
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "from skillevaluator.cli import cli; cli()",
+ "tier3",
+ "migrate-dataset",
+ str(source),
+ "--skill-name",
+ "demo",
+ ],
+ env={**os.environ, "PYTHONIOENCODING": f"{encoding}:strict", "NO_COLOR": "1"},
+ capture_output=True,
+ check=False,
+ )
+ assert result.returncode == 0, result.stderr
+ rendered = result.stdout.decode(encoding)
+ case = json.loads(rendered)["evals"][0]
+ assert case["prompt"] == text
+ assert case["expected_output"] == text
+ if readable:
+ assert text in rendered
+ assert source.read_bytes() == original
+
+
[email protected]("encoding,text,readable", _JSON_CASES)
+def test_validate_json_preserves_unicode(monkeypatch, encoding, text, readable, tmp_path) -> None:
+ from skillevaluator.tier3 import commands
+
+ output = BytesIO()
+ stdout = TextIOWrapper(output, encoding=encoding, errors="strict")
+ monkeypatch.setattr(cli_module.sys, "stdout", stdout)
+ monkeypatch.setattr(commands, "console", Console(file=stdout, force_terminal=False, width=120))
+ monkeypatch.setattr(
+ commands,
+ "validate_skillevaluators",
+ lambda _: [
+ SimpleNamespace(path=text, status="warning", message=text),
+ ],
+ )
+ monkeypatch.setattr(commands, "dataset_coverage_result", lambda _: None)
+ monkeypatch.setattr(cli_module, "setup_logging", lambda **_kwargs: None)
+
+ with pytest.raises(SystemExit) as exc:
+ cli_module.cli.main(["tier3", "validate", str(tmp_path), "--json"], standalone_mode=False)
+ assert exc.value.code == 0
+ stdout.flush()
+ rendered = output.getvalue().decode(encoding)
+ assert json.loads(rendered) == [
+ {"path": text, "status": "warning", "message": text},
+ ]
+ if readable:
+ assert text in rendered
+
+
+def test_json_without_stream_encoding_remains_readable() -> None:
+ assert json_needs_ascii({"value": _UNICODE}, None) is False