Multi-tier framework for evaluating AI agent skills and plugins with quality gates, security scanning, harness compatibility checks, semantic overlap detection, synthetic evaluation dataset generation, and live agent evaluation that measures how skills affect agent behavior. Ships reusable CI templates that bring every check to any repository.
Python agent-evaluationagent-pluginsagent-securityagent-skillsagentic-aibenchmarkci-templatesclaude-codecodexevaluateevaluationharnesspluginplugin-evaluationsecurity-scannerskill-evalskill-evaluationskill-evaluatorskill-gap-analysisskills
fix(tier3): launch the Bubblewrap binary that local mode detected (#278)
Narendran Raghavan1836cd5dcd6e
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex da376c84..a45ec6d0 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -345,6 +345,17 @@ All notable changes to SkillEvaluator are documented in this file.unchanged, so a large skill still fails when its score falls below theminimum.+### Security++- Linux local mode now launches Bubblewrap by the absolute path that passed its+ smoke test instead of a bare `bwrap`. The bare name was looked up on the host+ `PATH` for every command, from that command's working directory (the+ writable trial workspace by default), so a relative `PATH` entry such as `.`,+ an empty entry, or `./node_modules/.bin` let a `bwrap` file created by an+ earlier sandboxed command run outside the sandbox. A `bwrap` symlink on+ `PATH` is resolved to its target before the smoke test, and the local-mode+ isolation message names that resolved path.+## 0.6.0 - 2026-10-20### Addeddiff --git a/docs/agents-and-sandboxes.mdx b/docs/agents-and-sandboxes.mdxindex fb99d1b5..174626be 100644--- a/docs/agents-and-sandboxes.mdx+++ b/docs/agents-and-sandboxes.mdx@@ -584,7 +584,9 @@ on your machine runs the trials, wrapped in an OS-level sandbox. On Linux thatis Bubblewrap — kernel namespace isolation, a read-only view of the system,writes confined to the run directories, and a network namespace that isisolated when the run is airgapped and shared for model egress when network-is allowed. On macOS it is Seatbelt, which confines reads and writes and —+is allowed. SkillEvaluator resolves `bwrap` on `PATH` to an absolute path once,+smoke-tests that binary, and launches the same binary for every command. On+macOS it is Seatbelt, which confines reads and writes and —even when network is allowed — denies inbound connections, socket binding,and Unix sockets (except the `mDNSResponder` DNS socket); it cannot providefull process isolation, so it is the weaker of the two. Native Windows isdiff --git a/src/skillevaluator/tier3/harbor/local_sandbox.py b/src/skillevaluator/tier3/harbor/local_sandbox.pyindex 18013636..b071424f 100644--- a/src/skillevaluator/tier3/harbor/local_sandbox.py+++ b/src/skillevaluator/tier3/harbor/local_sandbox.py@@ -155,6 +155,7 @@ class SandboxPlan:backend: str # "bubblewrap" | "seatbelt" | "none"strength: str # "kernel" | "kernel-macos" | "advisory-only"reason: str # human-readable, surfaced in run summaries+ executable: str | None = None # absolute Linux launcher that passed detect()'s smoke testdef require_supported_platform() -> str:@@ -194,6 +195,7 @@ class Sandbox:if self.plan.backend == "bubblewrap":return _bwrap_argv(argv,+ bwrap=self.plan.executable or "bwrap",workdir=workdir,write_roots=write_roots,home=home,@@ -266,8 +268,14 @@ def detect(mode: str = "require") -> Sandbox:if system == "Linux":bwrap = shutil.which("bwrap")+ if bwrap:+ # Launch the absolute binary that passed the smoke test. A bare+ # "bwrap" is looked up again on the host PATH at every exec, with+ # cwd set to the writable workspace, so a relative PATH entry could+ # pick up a launcher planted there by an earlier sandboxed command.+ bwrap = str(Path(bwrap).resolve())if bwrap and _userns_enabled() and _bwrap_smoke_test(bwrap):- return Sandbox(SandboxPlan("bubblewrap", "kernel", f"bubblewrap at {bwrap}"))+ return Sandbox(SandboxPlan("bubblewrap", "kernel", f"bubblewrap at {bwrap}", bwrap))if not bwrap:reason = "bubblewrap not installed (install the bubblewrap package)"elif not _userns_enabled():@@ -480,6 +488,7 @@ def _seatbelt_smoke_test() -> bool:def _bwrap_argv(argv: list[str],*,+ bwrap: str = "bwrap",workdir: Path,write_roots: list[Path] | tuple[Path, ...],home: Path,@@ -497,7 +506,7 @@ def _bwrap_argv(the subprocess, and ``--clearenv`` would discard it."""wrapper = [- "bwrap",+ bwrap,"--unshare-all", # user, ipc, pid, uts, cgroup, AND net (loopback only)"--die-with-parent","--new-session", # blocks TIOCSTI terminal-injection escapesdiff --git a/tests/test_harbor_local_sandbox.py b/tests/test_harbor_local_sandbox.pyindex 71f3e714..8d6955e0 100644--- a/tests/test_harbor_local_sandbox.py+++ b/tests/test_harbor_local_sandbox.py@@ -130,6 +130,100 @@ class TestDetect:for flag in ("--die-with-parent", "--new-session", "--proc", "--dev", "--tmpfs"):assert flag in argv+ def test_linux_pins_resolved_bubblewrap_that_passed_smoke_test(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ target = tmp_path / "pkg" / "bwrap"+ target.parent.mkdir()+ target.write_text("#!/bin/sh\n", encoding="utf-8")+ link = tmp_path / "bin" / "bwrap"+ link.parent.mkdir()+ link.symlink_to(target)+ probes: list[str] = []+ monkeypatch.setattr(local_sandbox.platform, "system", lambda: "Linux")+ monkeypatch.setattr(local_sandbox.shutil, "which", lambda _name: str(link))+ monkeypatch.setattr(local_sandbox, "_userns_enabled", lambda: True)+ monkeypatch.setattr(local_sandbox, "_bwrap_smoke_test", lambda bwrap: probes.append(bwrap) is None)++ sandbox = local_sandbox.detect("require")+ # A later PATH lookup must not change which launcher runs.+ monkeypatch.setattr(local_sandbox.shutil, "which", lambda _name: "/different/bwrap")+ argv = sandbox.wrap(+ ["true"],+ workdir=tmp_path,+ write_roots=[tmp_path],+ home=tmp_path,+ tmp=tmp_path,+ allow_net=False,+ extra_ro=[],+ )++ resolved = str(target.resolve())+ assert sandbox.plan.executable == resolved+ assert probes == [resolved]+ assert argv[0] == resolved++ def test_linux_relative_which_result_is_pinned_absolute(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ (tmp_path / "bin").mkdir()+ (tmp_path / "bin" / "bwrap").write_text("#!/bin/sh\n", encoding="utf-8")+ monkeypatch.chdir(tmp_path)+ monkeypatch.setattr(local_sandbox.platform, "system", lambda: "Linux")+ monkeypatch.setattr(local_sandbox.shutil, "which", lambda _name: "bin/bwrap")+ monkeypatch.setattr(local_sandbox, "_userns_enabled", lambda: True)+ monkeypatch.setattr(local_sandbox, "_bwrap_smoke_test", lambda _bwrap: True)++ sandbox = local_sandbox.detect("require")++ assert sandbox.plan.executable == str((tmp_path / "bin" / "bwrap").resolve())+ assert Path(sandbox.plan.executable).is_absolute()++ @pytest.mark.skipif(os.name != "posix", reason="executes POSIX shell launchers")+ def test_relative_path_entry_cannot_substitute_launcher_from_workdir(+ self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path+ ) -> None:+ # Regression: local mode launches the wrapped argv with cwd set to the+ # writable workspace and the host PATH. A bare "bwrap" let a relative+ # PATH entry such as ./node_modules/.bin resolve against that workspace,+ # so a file planted by an earlier sandboxed command ran unconfined.+ markers = tmp_path / "markers"+ markers.mkdir()+ real_dir = tmp_path / "system-bin"+ real_dir.mkdir()+ real = real_dir / "bwrap"+ real.write_text(f"#!/bin/sh\n: > '{markers / 'real'}'\n", encoding="utf-8")+ real.chmod(0o755)+ workdir = tmp_path / "run" / "workspace"+ planted = workdir / "node_modules" / ".bin" / "bwrap"+ planted.parent.mkdir(parents=True)+ planted.write_text(f"#!/bin/sh\n: > '{markers / 'planted'}'\n", encoding="utf-8")+ planted.chmod(0o755)+ for name in ("home", "tmp"):+ (tmp_path / "run" / name).mkdir()+ host_path = f"./node_modules/.bin{os.pathsep}{real_dir}"+ monkeypatch.chdir(tmp_path)+ monkeypatch.setenv("PATH", host_path)+ monkeypatch.setattr(local_sandbox.platform, "system", lambda: "Linux")+ monkeypatch.setattr(local_sandbox, "_userns_enabled", lambda: True)+ monkeypatch.setattr(local_sandbox, "_bwrap_smoke_test", lambda _bwrap: True)++ sandbox = local_sandbox.detect("require")+ argv = sandbox.wrap(+ ["true"],+ workdir=workdir,+ write_roots=[tmp_path / "run"],+ home=tmp_path / "run" / "home",+ tmp=tmp_path / "run" / "tmp",+ allow_net=False,+ extra_ro=[],+ )+ subprocess.run(argv, cwd=str(workdir), env={"PATH": host_path}, check=True, timeout=10)++ assert Path(argv[0]).is_absolute()+ assert (markers / "real").exists()+ assert not (markers / "planted").exists()+@pytest.mark.skipif(platform.system() != "Windows", reason="requires a native Windows runner")@pytest.mark.parametrize("mode", local_sandbox.SANDBOX_MODES)@@ -707,6 +801,29 @@ class TestSandboxWrap:)assert wrapped[0] == "bwrap"+ def test_bubblewrap_backend_launches_detected_executable(self, tmp_path: Path) -> None:+ plan = local_sandbox.SandboxPlan("bubblewrap", "kernel", "test", "/opt/bubblewrap/bin/bwrap")+ wrapped = local_sandbox.Sandbox(plan).wrap(+ ["bash", "-c", "true"],+ workdir=tmp_path,+ write_roots=[tmp_path],+ home=tmp_path,+ tmp=tmp_path,+ allow_net=False,+ extra_ro=[],+ )+ unpinned = local_sandbox._bwrap_argv(+ ["bash", "-c", "true"],+ workdir=tmp_path,+ write_roots=[tmp_path],+ home=tmp_path,+ tmp=tmp_path,+ allow_net=False,+ extra_ro=[],+ )+ assert wrapped[0] == "/opt/bubblewrap/bin/bwrap"+ assert wrapped[1:] == unpinned[1:]+def test_strict_reads_rejects_root_that_contains_host_home(self,tmp_path: Path,