Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

Pass Windows program dirs and Python UTF-8 mode to Harbor (#220)

Godfred Arhinful9ffee8fcf0ba
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 313c12ba..d4395b0e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,10 @@ All notable changes to SkillEvaluator are documented in this file.
- Changing finding severities through a policy no longer clears an independent
quality-score threshold failure. A score below `--min-score` still fails,
and its reason appears in the terminal, HTML, benchmark, and JSON reports.
+- Tier 3 on Windows passes `PROGRAMFILES`, `PROGRAMW6432` and `PROGRAMDATA` to
+ Harbor, so Docker Desktop finds its `compose` plugin, and passes
+ `PYTHONUTF8` and `PYTHONIOENCODING` so Python UTF-8 mode reaches Harbor on
+ non-UTF-8 locales.
## 0.6.0 - 2026-10-20
diff --git a/src/skillevaluator/tier3/harbor/runner.py b/src/skillevaluator/tier3/harbor/runner.py
index 1bbf4343..f02f2a34 100644
--- a/src/skillevaluator/tier3/harbor/runner.py
+++ b/src/skillevaluator/tier3/harbor/runner.py
@@ -850,6 +850,11 @@ _HARBOR_BASE_ENV_VARS = _TRUSTED_NETWORK_HOST_ENV_VARS | {
"LC_CTYPE",
"PATH",
"PATHEXT",
+ "PROGRAMDATA",
+ "PROGRAMFILES",
+ "PROGRAMW6432",
+ "PYTHONIOENCODING",
+ "PYTHONUTF8",
"SKILLEVALUATOR_ALLOWED_MCP_HOSTS",
"SKILLEVALUATOR_ALLOWED_MCP_SECRETS",
"SYSTEMROOT",
diff --git a/src/skillevaluator/tier3/harbor/secure_docker_environment.py b/src/skillevaluator/tier3/harbor/secure_docker_environment.py
index a06e7657..de640a13 100644
--- a/src/skillevaluator/tier3/harbor/secure_docker_environment.py
+++ b/src/skillevaluator/tier3/harbor/secure_docker_environment.py
@@ -3005,6 +3005,7 @@ class SkillEvaluatorDockerEnvironment(DockerEnvironment):
"APPDATA",
"LOCALAPPDATA",
"PROGRAMDATA",
+ "PROGRAMFILES",
"SSH_AUTH_SOCK",
"SSH_AGENT_PID",
"DBUS_SESSION_BUS_ADDRESS",
diff --git a/tests/test_harbor_runner_environment.py b/tests/test_harbor_runner_environment.py
index 627dddb6..4f05a78a 100644
--- a/tests/test_harbor_runner_environment.py
+++ b/tests/test_harbor_runner_environment.py
@@ -55,6 +55,31 @@ def test_docker_opencode_nv_build_uses_operator_key_without_skill_runtime_env(
assert "OPENAI_API_KEY" not in plan.subprocess_env
+def test_harbor_environment_keeps_windows_program_dirs_and_python_utf8_mode(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Docker Desktop finds its CLI plugins under %ProgramFiles%, and Harbor needs the
+ # UTF-8 mode variables on non-UTF-8 Windows locales.
+ windows_env = {
+ "PROGRAMDATA": r"C:\ProgramData",
+ "PROGRAMFILES": r"C:\Program Files",
+ "PROGRAMW6432": r"C:\Program Files",
+ "PYTHONIOENCODING": "utf-8",
+ "PYTHONUTF8": "1",
+ }
+ monkeypatch.setattr(runner.os, "environ", {"PATH": r"C:\Windows", "UNRELATED_TOKEN": "secret", **windows_env})
+
+ env = runner._harbor_subprocess_environment(
+ env_mode="docker",
+ provider=_provider("nv_build"),
+ configured_runtime_env={},
+ provider_env={},
+ )
+
+ assert {name: env.get(name) for name in windows_env} == windows_env
+ assert "UNRELATED_TOKEN" not in env
+
+
@pytest.mark.parametrize(
("provider_name", "runtime_model", "catalog_model", "litellm_prefix"),
[
diff --git a/tests/test_harbor_secure_docker_environment.py b/tests/test_harbor_secure_docker_environment.py
index 15cda13e..6358941b 100644
--- a/tests/test_harbor_secure_docker_environment.py
+++ b/tests/test_harbor_secure_docker_environment.py
@@ -2589,6 +2589,40 @@ sys.stdin.buffer.read()
)
+def test_secure_handoff_keeps_program_files_for_every_compose_client(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ # Docker Desktop's CLI looks for its compose plugin under %ProgramFiles% on Windows.
+ audit_path = tmp_path / "program-files-audit.jsonl"
+ bin_dir = tmp_path / "trusted-bin"
+ bin_dir.mkdir()
+ docker_path = bin_dir / "docker"
+ docker_path.write_text(
+ f"""#!{sys.executable}
+import json
+import os
+import sys
+
+with open({str(audit_path)!r}, "a", encoding="utf-8") as audit:
+ audit.write(json.dumps(os.environ.get("PROGRAMFILES")) + "\\n")
+sys.stdin.buffer.read()
+""",
+ encoding="utf-8",
+ )
+ docker_path.chmod(0o700)
+ monkeypatch.setenv("PATH", str(bin_dir))
+ monkeypatch.setenv("PROGRAMFILES", r"C:\Program Files")
+ environment = _initialized_secure_docker_environment(tmp_path)
+
+ result = asyncio.run(environment.exec("true", env={"OPENAI_API_KEY": "sk-test-key"}))
+ audits = [json.loads(line) for line in audit_path.read_text(encoding="utf-8").splitlines()]
+
+ assert result.return_code == 0
+ assert len(audits) == 4
+ assert audits == [r"C:\Program Files"] * 4
+
+
def test_sidecar_filter_preserves_harbor_infra_over_user_name_collision(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,