Skip to content

Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.

fix(tier3): stop short host flags from blocking Compose sidecar values (#598)

Christopher Kevinbefb63dccb86
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 614fe73a..4a66fb4c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -451,6 +451,16 @@ All notable changes to SkillEvaluator are documented in this file.
A RAGAS `replied` entry for a reply with no text and no tool call, such as a
refusal or a content-filter stop, now saves the whole reply body instead of an
empty reply.
+- Tier 3 tasks with Compose sidecars no longer fail with `Docker Compose
+ interpolation variable ... requires protected execution state` because the
+ shell exports a short flag with a sensitive-looking name. A host variable such
+ as `XDG_SESSION_ID=3` or `MY_AUTH=1` made the sidecar check refuse every
+ Compose value containing that character, for example `python:3.13-slim` or
+ `127.0.0.1`. Host values shorter than eight characters now match a Compose
+ value only when the two are identical, as the trusted Compose client check
+ already does. Host values of eight characters or more, and sensitive-named
+ values a task declares for its main container, are still refused anywhere
+ inside a Compose value.
- Code Risk Analysis reports Semgrep file-level parse errors as one partial-scan
error that names up to 20 unparsed files, relative to the skill and with the
line of the first error, instead of the first three truncated messages. The
diff --git a/src/skillevaluator/tier3/harbor/secure_docker_environment.py b/src/skillevaluator/tier3/harbor/secure_docker_environment.py
index de640a13..bf7773b6 100644
--- a/src/skillevaluator/tier3/harbor/secure_docker_environment.py
+++ b/src/skillevaluator/tier3/harbor/secure_docker_environment.py
@@ -2300,9 +2300,23 @@ class SkillEvaluatorDockerEnvironment(DockerEnvironment):
raise RuntimeError(f"Docker Compose interpolation variable {name!r} requires protected execution state")
if self._is_compose_client_operational_name(name):
raise RuntimeError(f"Docker Compose interpolation variable {name!r} cannot use host client controls")
+ # Host values come from whatever the operator's shell exports, so a
+ # sensitive-looking flag such as SOME_SESSION=1 must not turn every
+ # Compose value containing "1" into protected state. Below the
+ # exact-secret floor they match by equality only, as in
+ # _trusted_compose_client_environment. Sensitive-named values the
+ # task itself declares for main still match as substrings at any
+ # length through _other_main_environment_values.
if any(
+ value == protected_value
+ or (
+ len(protected_value) >= _MIN_EXACT_SECRET_LENGTH
+ and _value_contains_protected_value(value, protected_value)
+ )
+ for protected_value in protected_values
+ ) or any(
_value_contains_protected_value(value, protected_value)
- for protected_value in (protected_values | self._other_main_environment_values(name, value))
+ for protected_value in self._other_main_environment_values(name, value)
if protected_value
):
raise RuntimeError(f"Docker Compose interpolation variable {name!r} requires protected execution state")
diff --git a/tests/test_harbor_secure_docker_environment.py b/tests/test_harbor_secure_docker_environment.py
index 6b42f827..11e7fef6 100644
--- a/tests/test_harbor_secure_docker_environment.py
+++ b/tests/test_harbor_secure_docker_environment.py
@@ -3005,6 +3005,83 @@ def test_sidecar_retains_structurally_required_nonsecret_host_compose_value(
assert captured_environments[0]["CUSTOM_HOST_IP"] == "127.0.0.1"
+ ("required_name", "required_value"),
+ [
+ ("HELPER_IMAGE", "python:3.13-slim"),
+ ("CUSTOM_HOST_IP", "127.0.0.1"),
+ ],
+)
+def test_sidecar_retains_compose_value_merely_containing_short_sensitive_host_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ required_name: str,
+ required_value: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ f"services:\n helper:\n image: ${{{required_name}:?required}}\n",
+ encoding="utf-8",
+ )
+ # A flag such as ``MY_AUTH=1`` has a sensitive-looking name but a value far
+ # below the exact-secret floor; sharing one character with it proves nothing.
+ monkeypatch.setenv("MY_AUTH", "1")
+ monkeypatch.setenv(required_name, required_value)
+ captured_environments: list[dict[str, str]] = []
+
+ async def create_subprocess(*_args: object, **kwargs: object) -> _BufferedComposeProcess:
+ captured_environments.append(dict(kwargs["env"]))
+ return _BufferedComposeProcess(stdout=b"ok")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ result = asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert result.stdout == "ok"
+ assert len(captured_environments) == 1
+ assert captured_environments[0][required_name] == required_value
+ assert "MY_AUTH" not in captured_environments[0]
+
+
+ ("host_value", "required_value"),
+ [
+ # Below the floor: still refused when the Compose value is exactly the host value.
+ ("pin4821", "pin4821"),
+ # At or above the floor: still refused anywhere inside the Compose value.
+ ("8charval", "registry.invalid/helper:8charval"),
+ ("host-only-sensitive-value", "prefix:host-only-sensitive-value:suffix"),
+ ],
+)
+def test_sidecar_rejects_compose_value_carrying_sensitive_host_value(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ host_value: str,
+ required_value: str,
+) -> None:
+ environment = _initialized_secure_docker_environment(tmp_path)
+ (environment.environment_dir / "docker-compose.yaml").write_text(
+ "services:\n helper:\n image: ${HELPER_IMAGE:?required}\n",
+ encoding="utf-8",
+ )
+ monkeypatch.setenv("MY_AUTH", host_value)
+ monkeypatch.setenv("HELPER_IMAGE", required_value)
+ spawned = False
+
+ async def create_subprocess(*_args: object, **_kwargs: object) -> object:
+ nonlocal spawned
+ spawned = True
+ raise AssertionError("sensitive host value must fail before spawn")
+
+ monkeypatch.setattr(asyncio, "create_subprocess_exec", create_subprocess)
+
+ with pytest.raises(RuntimeError, match="requires protected execution state") as caught:
+ asyncio.run(environment.service_exec("true", service="helper"))
+
+ assert host_value not in str(caught.value)
+ assert spawned is False
+
+
@pytest.mark.parametrize(
("required_name", "required_value"),
[