Skip to content

Secure, Fast, and Extensible Sandbox runtime for AI agents.

26.0 KB
Raw Blame
# Copyright 2026 The OpenSandbox Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tests for bundled skill install/list/show/uninstall flows."""

from __future__ import annotations

import importlib.resources
import json
from pathlib import Path
from types import SimpleNamespace
from typing import cast

import pytest
from click import Command, Group, Option
from click.testing import CliRunner

from opensandbox_cli.commands.skills import _TARGETS, TargetConfig, skills_group
from opensandbox_cli.main import cli
from opensandbox_cli.output import OutputFormatter
from opensandbox_cli.skill_registry import list_builtin_skills


def _rebase(path: Path, root: Path) -> Path:
    """Rebase a target path (absolute home-based or relative) under root."""
    relative = path.relative_to(Path.home()) if path.is_absolute() else path
    return root / relative


@pytest.fixture()
def isolated_skill_targets(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
    """Point project scope at tmp_path and global scope at tmp_path/home."""
    patched: dict[str, object] = {}
    for name, cfg in _TARGETS.items():
        scopes = {}
        for scope, scope_cfg in cfg["scopes"].items():
            root = tmp_path if scope == "project" else tmp_path / "home"
            new_cfg = dict(scope_cfg)
            if "dest_dir" in new_cfg:
                new_cfg["dest_dir"] = _rebase(new_cfg["dest_dir"], root)
            if "dest_file" in new_cfg:
                new_cfg["dest_file"] = _rebase(new_cfg["dest_file"], root)
            scopes[scope] = new_cfg
        patched[name] = {**cfg, "scopes": scopes}
    monkeypatch.setattr(
        "opensandbox_cli.commands.skills._TARGETS",
        cast(dict[str, TargetConfig], patched),
    )


class TestSkillsCommands:
    def test_list_supports_json_output(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["list"],
            obj=SimpleNamespace(output=OutputFormatter("json", color=False)),
        )

        assert result.exit_code == 0
        data = json.loads(result.output)
        assert "skills" in data
        assert "targets" in data
        assert any(skill["slug"] == "sandbox-troubleshooting" for skill in data["skills"])
        assert any(skill["slug"] == "sandbox-lifecycle" for skill in data["skills"])

    def test_show_supports_json_output(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["show", "sandbox-lifecycle"],
            obj=SimpleNamespace(output=OutputFormatter("json", color=False)),
        )

        assert result.exit_code == 0
        data = json.loads(result.output)
        assert data["skill"] == "sandbox-lifecycle"
        assert data["title"] == "OpenSandbox Sandbox Lifecycle"
        assert '"defaultAction": "deny"' in data["json_shapes"]

    def test_install_without_args_prints_guidance_and_does_not_install(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(skills_group, ["install"])

        assert result.exit_code != 0
        assert "Install guidance:" in result.output
        assert "osb skills install <skill-name> --target <tool> --scope <scope>" in result.output
        assert not (
            tmp_path / ".claude" / "skills" / "sandbox-troubleshooting" / "SKILL.md"
        ).exists()

    def test_install_with_skill_but_without_target_prints_guidance(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(skills_group, ["install", "sandbox-troubleshooting"])

        assert result.exit_code != 0
        assert "Install guidance:" in result.output
        assert "Missing required option '--target'" in result.output

    def test_install_with_all_builtins_but_without_target_prints_guidance(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(skills_group, ["install", "--all-builtins"])

        assert result.exit_code != 0
        assert "Install guidance:" in result.output
        assert "Missing required option '--target'" in result.output

    def test_install_copy_target_creates_named_skill_file(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "claude", "--scope", "project"],
        )

        assert result.exit_code == 0
        dest = tmp_path / ".claude" / "skills" / "sandbox-troubleshooting" / "SKILL.md"
        assert dest.exists()
        assert not (tmp_path / ".claude" / "skills" / "sandbox-troubleshooting.md").exists()
        content = dest.read_text(encoding="utf-8")
        assert content.startswith("---\nname: sandbox-troubleshooting")

    def test_install_codex_creates_skill_directory_with_frontmatter(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "codex", "--scope", "project"],
        )

        assert result.exit_code == 0
        dest = tmp_path / ".codex" / "skills" / "sandbox-troubleshooting" / "SKILL.md"
        content = dest.read_text(encoding="utf-8")
        assert content.startswith("---\nname: sandbox-troubleshooting")
        assert "# OpenSandbox Sandbox Troubleshooting" in content

    def test_install_reports_already_present_without_prompt(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        first = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "codex", "--scope", "project"],
        )
        assert first.exit_code == 0

        second = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "codex", "--scope", "project"],
        )

        assert second.exit_code == 0
        assert "already_present" in second.output
        dest = tmp_path / ".codex" / "skills" / "sandbox-troubleshooting" / "SKILL.md"
        assert dest.exists()

    def test_install_all_builtins_to_codex_creates_skill_directories(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "--all-builtins", "--target", "codex", "--scope", "project"],
        )

        assert result.exit_code == 0
        assert "Install plan:" in result.output
        assert "install one file per skill" in result.output
        for skill in list_builtin_skills():
            dest = tmp_path / ".codex" / "skills" / skill.slug / "SKILL.md"
            assert dest.exists()

    def test_install_supports_json_output(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "network-egress", "--target", "codex", "--scope", "project"],
            obj=SimpleNamespace(output=OutputFormatter("json", color=False)),
        )

        assert result.exit_code == 0
        data = json.loads(result.output)
        assert data["requires_restart"] is True
        assert data["operations"][0]["skill"] == "network-egress"
        assert data["operations"][0]["status"] == "installed"

    def test_install_rejects_skill_name_and_all_builtins_together(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--all-builtins"],
        )

        assert result.exit_code != 0
        assert "either a skill name or --all-builtins" in result.output

    def test_install_to_global_codex_uses_global_path(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "codex", "--scope", "global"],
        )

        assert result.exit_code == 0
        assert (tmp_path / "home" / ".codex" / "skills" / "sandbox-troubleshooting" / "SKILL.md").exists()

    def test_install_to_global_cline_uses_documented_rules_directory(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        user_rule = tmp_path / "home" / ".cline" / "rules" / "network-egress.md"
        user_rule.parent.mkdir(parents=True, exist_ok=True)
        user_rule.write_text("user-owned egress rule\n", encoding="utf-8")

        result = runner.invoke(
            skills_group,
            ["install", "network-egress", "--target", "cline", "--scope", "global"],
        )

        assert result.exit_code == 0
        dest = tmp_path / "home" / ".cline" / "rules" / "opensandbox-network-egress.md"
        assert dest.exists()
        assert dest.read_text(encoding="utf-8").startswith("# OpenSandbox Network Egress")
        assert user_rule.read_text(encoding="utf-8") == "user-owned egress rule\n"
        assert not (tmp_path / "home" / ".clinerules").exists()

        uninstall = runner.invoke(
            skills_group,
            ["uninstall", "network-egress", "--target", "cline", "--scope", "global"],
        )

        assert uninstall.exit_code == 0
        assert not dest.exists()
        assert user_rule.read_text(encoding="utf-8") == "user-owned egress rule\n"

    def test_install_to_global_copilot_uses_documented_user_file(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "network-egress", "--target", "copilot", "--scope", "global"],
        )

        assert result.exit_code == 0
        dest = tmp_path / "home" / ".copilot" / "copilot-instructions.md"
        assert dest.exists()
        assert "# OpenSandbox Network Egress" in dest.read_text(encoding="utf-8")
        assert not (tmp_path / "home" / ".github" / "copilot-instructions.md").exists()

    def test_install_to_project_opencode_creates_skill_directory(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "network-egress", "--target", "opencode", "--scope", "project"],
        )

        assert result.exit_code == 0
        dest = tmp_path / ".agents" / "skills" / "network-egress" / "SKILL.md"
        assert dest.exists()
        assert dest.read_text(encoding="utf-8").startswith("---\nname: network-egress")

    def test_show_prints_skill_metadata_and_content(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(skills_group, ["show", "file-operations"])

        assert result.exit_code == 0
        assert "Skill: file-operations" in result.output
        assert "Title: OpenSandbox File Operations" in result.output
        assert "When To Use:" in result.output
        assert "Quick Start:" in result.output
        assert "Minimal Closed Loops:" in result.output
        assert "Full Skill:" in result.output
        assert "osb file cat" in result.output

    def test_show_prints_network_and_vault_skills(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        network = runner.invoke(skills_group, ["show", "network-egress"])
        assert network.exit_code == 0
        assert "Skill: network-egress" in network.output
        assert "Quick Start:" in network.output
        assert "osb egress patch" in network.output

        vault = runner.invoke(skills_group, ["show", "credential-vault"])
        assert vault.exit_code == 0
        assert "Skill: credential-vault" in vault.output
        assert "osb credential-vault create" in vault.output

    def test_show_surfaces_json_shapes_for_lifecycle_skill(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(skills_group, ["show", "sandbox-lifecycle"])

        assert result.exit_code == 0
        assert "JSON Shapes:" in result.output
        assert '"defaultAction": "deny"' in result.output
        assert '"mountPath": "/workspace/data"' in result.output

    def test_list_reports_all_builtins(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        result = runner.invoke(skills_group, ["list"])

        assert result.exit_code == 0
        assert "aggregate into one instructions file" in result.output
        assert "install one file per skill" in result.output
        for skill in list_builtin_skills():
            assert skill.slug in result.output

    def test_list_reports_not_installed_when_append_target_file_exists_without_marker(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        dest = tmp_path / ".github" / "copilot-instructions.md"
        dest.parent.mkdir(parents=True, exist_ok=True)
        dest.write_text("user custom instructions\n", encoding="utf-8")

        result = runner.invoke(skills_group, ["list"])

        assert result.exit_code == 0
        assert "copilot" in result.output
        assert "not installed" in result.output

    def test_uninstall_append_target_preserves_non_skill_content(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        install_result = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "copilot", "--scope", "project"],
        )
        assert install_result.exit_code == 0

        dest = tmp_path / ".github" / "copilot-instructions.md"
        original = dest.read_text(encoding="utf-8")
        dest.write_text("team rules\n\n" + original, encoding="utf-8")

        uninstall_result = runner.invoke(
            skills_group,
            ["uninstall", "sandbox-troubleshooting", "--target", "copilot", "--scope", "project"],
        )

        assert uninstall_result.exit_code == 0
        assert dest.read_text(encoding="utf-8") == "team rules\n"

    def test_uninstall_supports_json_output(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
    ) -> None:
        install_result = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "codex", "--scope", "project"],
        )
        assert install_result.exit_code == 0

        uninstall_result = runner.invoke(
            skills_group,
            ["uninstall", "sandbox-troubleshooting", "--target", "codex", "--scope", "project"],
            obj=SimpleNamespace(output=OutputFormatter("json", color=False)),
        )

        assert uninstall_result.exit_code == 0
        data = json.loads(uninstall_result.output)
        assert data["operations"][0]["status"] == "removed"

    def test_reinstall_append_target_does_not_duplicate_skill_block(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        first = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "copilot", "--scope", "project"],
        )
        assert first.exit_code == 0

        second = runner.invoke(
            skills_group,
            ["install", "sandbox-troubleshooting", "--target", "copilot", "--scope", "project", "--force"],
        )

        assert second.exit_code == 0
        dest = tmp_path / ".github" / "copilot-instructions.md"
        content = dest.read_text(encoding="utf-8")
        assert content.count("<!-- BEGIN opensandbox-sandbox-troubleshooting -->") == 1

    def test_install_all_builtins_to_copy_target_creates_new_skill_files(
        self,
        runner: CliRunner,
        isolated_skill_targets: None,
        tmp_path: Path,
    ) -> None:
        result = runner.invoke(
            skills_group,
            ["install", "--all-builtins", "--target", "claude", "--scope", "project"],
        )

        assert result.exit_code == 0
        assert "Install plan:" in result.output
        assert "install one file per skill" in result.output
        assert (tmp_path / ".claude" / "skills" / "network-egress" / "SKILL.md").exists()
        assert (tmp_path / ".claude" / "skills" / "sandbox-troubleshooting" / "SKILL.md").exists()


def _read_builtin_skill(package_file: str) -> str:
    resource = importlib.resources.files("opensandbox_cli") / "skills" / package_file
    return Path(str(resource)).read_text(encoding="utf-8")


def _command(path: list[str]) -> Command:
    current: Command = cli
    for part in path:
        assert isinstance(current, Group), f"{current.name} is not a command group"
        current = current.commands[part]
    return current


def _option_names(command: Command) -> set[str]:
    names: set[str] = set()
    for param in command.params:
        if isinstance(param, Option):
            names.update(param.opts)
            names.update(param.secondary_opts)
    return names


class TestSkillContentQuality:
    def test_sandbox_troubleshooting_keeps_triage_and_diagnostics_contract(self) -> None:
        content = _read_builtin_skill("opensandbox-sandbox-troubleshooting.md")

        assert "## Triage Order" in content
        assert "osb sandbox get <sandbox-id> -o json" in content
        assert "osb diagnostics events <sandbox-id> --scope lifecycle" not in content
        assert "osb diagnostics events <sandbox-id> --scope runtime -o raw" in content
        assert "osb diagnostics events <sandbox-id> --scope all -o raw" in content
        assert "osb diagnostics logs <sandbox-id> --scope container -o raw" in content
        assert "osb diagnostics logs <sandbox-id> --scope all -o raw" in content
        assert "osb diagnostics logs <sandbox-id> --scope lifecycle" not in content
        assert "events:lifecycle" not in content
        assert "## Diagnostics Streams" in content
        assert "## Evidence Semantics" in content
        assert "## URL Delivery" in content
        assert "truncated: true" in content
        assert "osb devops" not in content
        assert "## Symptom To Command Mapping" in content

    def test_builtin_skills_do_not_reference_legacy_devops_diagnostics(self) -> None:
        skills_dir = Path("src/opensandbox_cli/skills")
        forbidden = (
            "osb devops inspect",
            "osb devops summary",
            "devops inspect",
            "devops summary",
        )

        violations: list[str] = []
        for skill_path in sorted(skills_dir.glob("*.md")):
            content = skill_path.read_text(encoding="utf-8")
            for term in forbidden:
                if term in content:
                    violations.append(f"{skill_path.name}: {term}")

        assert violations == []

    def test_lifecycle_skill_keeps_json_shapes_and_health_guidance(self) -> None:
        content = _read_builtin_skill("opensandbox-sandbox-lifecycle.md")

        assert "`kill` returns `deletion_requested` when accepted" in content
        assert "Pool `Noop` preserves processes" in content
        assert "## JSON Shapes" in content
        assert '"defaultAction": "deny"' in content
        assert '"mountPath": "/workspace/data"' in content
        assert "Prefer `health` over assuming readiness from `create` output alone" in content

    def test_sandbox_troubleshooting_keeps_cli_first_and_http_fallback_guidance(self) -> None:
        content = _read_builtin_skill("opensandbox-sandbox-troubleshooting.md")

        assert "## Operating Rules" in content
        assert "use CLI commands when `osb` is available" in content
        assert "use HTTP only when the CLI is unavailable" in content
        assert "Use raw HTTP only after domain, protocol, and API key expectations are explicit." in content
        assert "## Symptom To Command Mapping" in content


class TestSkillCliAlignment:
    def test_command_execution_skill_matches_command_cli(self) -> None:
        run_cmd = _command(["command", "run"])
        session_run_cmd = _command(["command", "session", "run"])
        logs_cmd = _command(["command", "logs"])
        session_delete_cmd = _command(["command", "session", "delete"])

        assert {"-d", "--background", "-w", "--workdir", "-t", "--timeout", "-o", "--output"} <= _option_names(run_cmd)
        assert {"-w", "--workdir", "-t", "--timeout", "-o", "--output"} <= _option_names(session_run_cmd)
        assert {"--cursor", "-o", "--output"} <= _option_names(logs_cmd)
        assert {"-o", "--output"} <= _option_names(session_delete_cmd)

    def test_sandbox_lifecycle_skill_matches_sandbox_cli(self) -> None:
        create_cmd = _command(["sandbox", "create"])
        resume_cmd = _command(["sandbox", "resume"])
        endpoint_cmd = _command(["sandbox", "endpoint"])
        metrics_cmd = _command(["sandbox", "metrics"])
        template_group = _command(["template"])
        snapshot_group = _command(["snapshot"])

        assert {
            "-i", "--image", "-t", "--timeout", "--entrypoint", "--network-policy-file",
            "--credential-proxy", "--volumes-file", "--skip-health-check",
            "--ready-timeout", "--template", "--snapshot-id", "-o", "--output",
        } <= _option_names(create_cmd)
        assert {"--skip-health-check", "--resume-timeout", "-o", "--output"} <= _option_names(resume_cmd)
        assert {"-p", "--port", "-o", "--output"} <= _option_names(endpoint_cmd)
        assert {"--watch", "-o", "--output"} <= _option_names(metrics_cmd)
        assert isinstance(template_group, Group)
        assert {"create", "get", "list", "delete"} <= set(template_group.commands)
        assert isinstance(snapshot_group, Group)
        assert {"create", "get", "list", "delete"} <= set(snapshot_group.commands)

    def test_file_operations_skill_matches_file_cli(self) -> None:
        expected_subcommands = {
            "cat", "write", "upload", "download", "rm", "mv", "mkdir",
            "rmdir", "search", "info", "chmod", "replace",
        }
        file_group = _command(["file"])
        assert isinstance(file_group, Group)
        assert expected_subcommands <= set(file_group.commands)

        assert {"-c", "--content", "--encoding", "--mode", "--owner", "--group", "-o", "--output"} <= _option_names(
            _command(["file", "write"])
        )
        assert {"-p", "--pattern", "-o", "--output"} <= _option_names(_command(["file", "search"]))
        assert {"--mode", "--owner", "--group", "-o", "--output"} <= _option_names(_command(["file", "chmod"]))
        assert {"--old", "--new", "-o", "--output"} <= _option_names(_command(["file", "replace"]))

    def test_network_egress_diagnostics_and_devops_skills_match_cli(self) -> None:
        patch_cmd = _command(["egress", "patch"])
        diag_logs_cmd = _command(["diagnostics", "logs"])
        diag_events_cmd = _command(["diagnostics", "events"])
        logs_cmd = _command(["devops", "logs"])
        events_cmd = _command(["devops", "events"])
        summary_cmd = _command(["devops", "summary"])

        assert {"--rule", "-o", "--output"} <= _option_names(patch_cmd)
        assert {"--scope", "-s", "-o", "--output"} <= _option_names(diag_logs_cmd)
        assert {"--scope", "-s", "-o", "--output"} <= _option_names(diag_events_cmd)
        assert {"--tail", "-n", "--since", "-s", "-o", "--output"} <= _option_names(logs_cmd)
        assert {"--limit", "-l", "-o", "--output"} <= _option_names(events_cmd)
        assert {"--tail", "-n", "--event-limit", "-o", "--output"} <= _option_names(summary_cmd)

    def test_credential_vault_skill_matches_cli(self) -> None:
        vault_group = _command(["credential-vault"])
        assert isinstance(vault_group, Group)
        assert {"create", "get", "patch", "delete", "credential", "binding"} <= set(vault_group.commands)

        assert {"--file", "-o", "--output"} <= _option_names(_command(["credential-vault", "create"]))
        assert {"--file", "-o", "--output"} <= _option_names(_command(["credential-vault", "patch"]))
        assert {"-o", "--output"} <= _option_names(_command(["credential-vault", "get"]))
        assert {"-o", "--output"} <= _option_names(_command(["credential-vault", "delete"]))
        assert {"-o", "--output"} <= _option_names(_command(["credential-vault", "credential", "list"]))
        assert {"-o", "--output"} <= _option_names(_command(["credential-vault", "binding", "get"]))

    def test_skill_osb_examples_use_explicit_output_formats(self) -> None:
        allowed_without_output = {
            "osb --version",
        }
        skills_dir = Path("src/opensandbox_cli/skills")

        missing_output: list[str] = []
        for skill_path in sorted(skills_dir.glob("*.md")):
            in_block = False
            for line in skill_path.read_text(encoding="utf-8").splitlines():
                stripped = line.strip()
                if stripped.startswith("```bash"):
                    in_block = True
                    continue
                if in_block and stripped == "```":
                    in_block = False
                    continue
                if not in_block or not stripped.startswith("osb "):
                    continue
                if stripped in allowed_without_output:
                    continue
                if " -o " not in stripped:
                    missing_output.append(f"{skill_path.name}: {stripped}")

        assert missing_output == []