Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

18.7 KB
Raw Blame
"""Unit tests for the ``run-skill`` Brain-callable tool.

Instruction-skill model (2026-06-09 rebuild, AD-S1/S2/S5): the tool resolves
a skill by name, enforces DRAFT/DISABLED/block-tier rejection, and returns the
rendered skill body as instructions for the brain to follow — it never
macro-executes. Optional ``resource`` argument serves bundled files
(progressive disclosure L3). Tests use Fakes (no ``unittest.mock``) per
``AGENTS.md`` testing-conventions.
"""
from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Any
from uuid import uuid4

import pytest

from dodaios.core.protocols import ExecutionContext, ToolResult
from dodaios.plugins.tool.run_skill import RunSkillTool
from dodaios.skills.schema import (
    Skill,
    SkillFrontmatter,
    SkillLifecycleState,
    SkillRiskPolicy,
)
from dodaios.skills.skill_context import SkillContext, set_skill_context

# ----------------------------------------------------------------------
# Fakes
# ----------------------------------------------------------------------


class _FakeRegistry:
    """Tiny registry whose ``get`` raises ``KeyError`` like the real one."""

    def __init__(self, skills: dict[str, Skill] | None = None) -> None:
        self._skills = skills or {}

    def get(self, name: str) -> Skill:
        if name not in self._skills:
            raise KeyError(f"Skill '{name}' not in registry")
        return self._skills[name]

    def list(self) -> list[Skill]:
        return list(self._skills.values())


@dataclass
class _RenderCall:
    skill: Skill
    args: dict[str, Any]


class _FakeRunner:
    """Records ``render_instructions`` calls, returns a scripted body."""

    def __init__(self, scripted_body: str = "# Demo\nDo the thing.") -> None:
        self.calls: list[_RenderCall] = []
        self.scripted_body = scripted_body

    def render_instructions(
        self, skill: Skill, *, args: dict[str, Any] | None = None
    ) -> str:
        self.calls.append(_RenderCall(skill=skill, args=dict(args or {})))
        return self.scripted_body


class _ExplodingRunner:
    """Runner that raises to verify the tool catches inner exceptions."""

    def __init__(self, exc: Exception) -> None:
        self.exc = exc
        self.calls: list[_RenderCall] = []

    def render_instructions(
        self, skill: Skill, *, args: dict[str, Any] | None = None
    ) -> str:
        self.calls.append(_RenderCall(skill=skill, args=dict(args or {})))
        raise self.exc


class _FakeBus:
    def __init__(self) -> None:
        self.published: list[Any] = []

    async def publish(self, event: Any) -> None:
        self.published.append(event)


# ----------------------------------------------------------------------
# Helpers
# ----------------------------------------------------------------------


def _make_skill(
    name: str = "demo_skill",
    *,
    state: SkillLifecycleState = SkillLifecycleState.ACTIVE,
    default_tier: str = "monitor",
    frontmatter: bool = True,
    execution: str = "inline",
    path: Path | None = None,
    resources: dict[str, tuple[str, ...]] | None = None,
) -> Skill:
    fm: SkillFrontmatter | None
    if frontmatter:
        fm = SkillFrontmatter(
            schema_version="1",
            name=name,
            description="fake skill",
            risk_policy=SkillRiskPolicy(default_tier=default_tier),  # type: ignore[arg-type]
            execution=execution,  # type: ignore[arg-type]
        )
    else:
        fm = None
    kwargs: dict[str, Any] = {}
    if resources is not None:
        kwargs["resources"] = resources
    return Skill(
        path=path or (Path("nonexistent") / name / "SKILL.md"),
        frontmatter=fm,
        body="dummy",
        state=state,
        body_hash="deadbeef",
        error=None,
        **kwargs,
    )


def _ctx() -> ExecutionContext:
    return ExecutionContext(
        trace_id=uuid4(),
        user_utterance="run the demo skill",
        config={},
        memory_read=None,
        approved_by="auto",
    )


def _wire(skill: Skill, runner: Any | None = None) -> Any:
    runner = runner or _FakeRunner()
    registry = _FakeRegistry({skill.name: skill})
    set_skill_context(SkillContext(registry=registry, runner=runner))
    return runner


@pytest.fixture(autouse=True)
def _reset_skill_context():
    """Reset the global SkillContext between tests."""
    set_skill_context(None)
    yield
    set_skill_context(None)


# ----------------------------------------------------------------------
# Validation gates (unchanged contract)
# ----------------------------------------------------------------------


@pytest.mark.asyncio
async def test_run_skill_unknown_skill_returns_error() -> None:
    registry = _FakeRegistry()  # empty
    runner = _FakeRunner()
    set_skill_context(SkillContext(registry=registry, runner=runner))

    tool = RunSkillTool()
    result = await tool.execute({"skill_name": "missing"}, _ctx())

    assert isinstance(result, ToolResult)
    assert result.success is False
    assert result.error is not None
    assert "missing" in result.error
    assert runner.calls == []


@pytest.mark.asyncio
async def test_run_skill_rejects_draft_state() -> None:
    skill = _make_skill("draft_skill", state=SkillLifecycleState.DRAFT)
    runner = _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute({"skill_name": "draft_skill"}, _ctx())

    assert result.success is False
    assert result.error is not None
    assert "DRAFT" in result.error
    assert runner.calls == [], "instructions must NOT render for DRAFT skills"


@pytest.mark.asyncio
async def test_run_skill_rejects_disabled_state() -> None:
    skill = _make_skill("off_skill", state=SkillLifecycleState.DISABLED)
    runner = _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute({"skill_name": "off_skill"}, _ctx())

    assert result.success is False
    assert result.error is not None
    assert "DISABLED" in result.error
    assert runner.calls == [], "instructions must NOT render for DISABLED skills"


@pytest.mark.asyncio
async def test_run_skill_rejects_block_tier() -> None:
    skill = _make_skill("blocked_skill", default_tier="block")
    runner = _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute({"skill_name": "blocked_skill"}, _ctx())

    assert result.success is False
    assert "block" in (result.error or "").lower()
    assert runner.calls == []


@pytest.mark.asyncio
async def test_run_skill_no_skill_context() -> None:
    set_skill_context(None)  # explicit
    tool = RunSkillTool()

    result = await tool.execute({"skill_name": "anything"}, _ctx())

    assert result.success is False
    assert "not initialized" in (result.error or "")


@pytest.mark.asyncio
async def test_run_skill_missing_argument() -> None:
    tool = RunSkillTool()
    result = await tool.execute({}, _ctx())
    assert result.success is False
    assert "skill_name" in (result.error or "")


# ----------------------------------------------------------------------
# Instruction-skill model (AD-S1)
# ----------------------------------------------------------------------


@pytest.mark.asyncio
async def test_returns_instructions_not_macro_result() -> None:
    skill = _make_skill("demo_skill")
    runner = _wire(skill, _FakeRunner(scripted_body="# Demo\nDo the briefing."))

    tool = RunSkillTool()
    result = await tool.execute(
        {"skill_name": "demo_skill", "args": {"foo": "bar"}}, _ctx()
    )

    assert result.success is True
    out = result.output
    assert out["skill_name"] == "demo_skill"
    assert out["execution"] == "inline"
    assert out["instructions"].startswith("# Demo")
    assert "Follow these skill instructions now" in out["directive"]
    assert runner.calls[0].args == {"foo": "bar"}


@pytest.mark.asyncio
async def test_mission_skill_returns_mission_directive() -> None:
    skill = _make_skill("heavy", execution="mission")
    _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute({"skill_name": "heavy"}, _ctx())

    assert result.success is True
    assert result.output["execution"] == "mission"
    assert "spawn_worker" in result.output["directive"]


@pytest.mark.asyncio
async def test_render_failure_is_reported() -> None:
    skill = _make_skill("flaky_skill")
    _wire(skill, _ExplodingRunner(RuntimeError("boom")))

    tool = RunSkillTool()
    result = await tool.execute({"skill_name": "flaky_skill"}, _ctx())

    assert result.success is False
    assert "boom" in (result.error or "")


@pytest.mark.asyncio
async def test_skill_invoked_event_published() -> None:
    skill = _make_skill("demo_skill")
    bus = _FakeBus()
    runner = _FakeRunner()
    registry = _FakeRegistry({"demo_skill": skill})
    set_skill_context(SkillContext(registry=registry, runner=runner))

    tool = RunSkillTool(bus=bus)
    await tool.execute({"skill_name": "demo_skill"}, _ctx())

    names = [type(e).__name__ for e in bus.published]
    assert "SkillInvoked" in names
    ev = next(e for e in bus.published if type(e).__name__ == "SkillInvoked")
    assert ev.skill_name == "demo_skill"
    assert ev.source == "model"


# ----------------------------------------------------------------------
# Progressive disclosure L3: bundled resources (AD-S2)
# ----------------------------------------------------------------------


@pytest.mark.asyncio
async def test_resource_loading(tmp_path: Path) -> None:
    root = tmp_path / "demo_skill"
    (root / "references").mkdir(parents=True)
    (root / "SKILL.md").write_text("---\nname: x\n---\nbody", encoding="utf-8")
    (root / "references" / "guide.md").write_text("guide content", encoding="utf-8")
    skill = _make_skill(
        "demo_skill",
        path=root / "SKILL.md",
        resources={
            "references": ("references/guide.md",),
            "scripts": (),
            "assets": (),
            "agents": (),
        },
    )
    _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute(
        {"skill_name": "demo_skill", "resource": "references/guide.md"}, _ctx()
    )

    assert result.success is True
    assert "guide content" in result.output["resource_content"]


@pytest.mark.asyncio
async def test_resource_path_traversal_rejected(tmp_path: Path) -> None:
    root = tmp_path / "demo_skill"
    root.mkdir(parents=True)
    (root / "SKILL.md").write_text("---\nname: x\n---\nbody", encoding="utf-8")
    secret = tmp_path / "secrets.txt"
    secret.write_text("nope", encoding="utf-8")
    skill = _make_skill("demo_skill", path=root / "SKILL.md")
    _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute(
        {"skill_name": "demo_skill", "resource": "../secrets.txt"}, _ctx()
    )

    assert result.success is False


@pytest.mark.asyncio
async def test_unregistered_resource_rejected(tmp_path: Path) -> None:
    root = tmp_path / "demo_skill"
    root.mkdir(parents=True)
    (root / "SKILL.md").write_text("---\nname: x\n---\nbody", encoding="utf-8")
    (root / "rogue.md").write_text("rogue", encoding="utf-8")
    skill = _make_skill("demo_skill", path=root / "SKILL.md")
    _wire(skill)

    tool = RunSkillTool()
    result = await tool.execute(
        {"skill_name": "demo_skill", "resource": "rogue.md"}, _ctx()
    )

    assert result.success is False
    assert "resource" in (result.error or "").lower()


# ----------------------------------------------------------------------
# Speech-tolerant name resolution (live regression 2026-08-20)
# ----------------------------------------------------------------------


class _ResolvingRegistry(_FakeRegistry):
    """A registry with the real ``resolve`` + ``list_active`` surface.

    ``_FakeRegistry`` above predates ``resolve`` on purpose — it proves the
    tool still works against a registry that has only ``get`` (mock and
    headless boots). This one carries the real thing.
    """

    def resolve(self, name: str) -> Skill:
        from dodaios.skills.relevance import skill_name_tokens

        key = str(name or "").strip()
        if key in self._skills:
            return self._skills[key]
        wanted = skill_name_tokens(key)
        if wanted:
            for candidate in sorted(self._skills.values(), key=lambda s: s.name):
                if skill_name_tokens(str(candidate.name)) == wanted:
                    return candidate
        raise KeyError(f"Skill '{name}' not in registry")

    def list_active(self) -> list[Skill]:
        return list(self._skills.values())


@pytest.mark.asyncio
async def test_run_skill_resolves_a_display_name_to_the_installed_slug() -> None:
    """The live failure, verbatim: the realtime model sent "Morning Routine"
    while the registry key was "morning-routine", and the user was told the
    skill did not exist three turns running."""
    skill = _make_skill(name="morning-routine")
    registry = _ResolvingRegistry({"morning-routine": skill})
    runner = _FakeRunner()
    set_skill_context(SkillContext(registry=registry, runner=runner))  # type: ignore[arg-type]

    result = await RunSkillTool().execute({"skill_name": "Morning Routine"}, _ctx())

    assert result.success is True
    assert result.output["skill_name"] == "morning-routine"


@pytest.mark.asyncio
async def test_run_skill_reports_the_resolved_name_not_the_guess() -> None:
    """Everything downstream — the spoken answer, the SkillInvoked event —
    must name the skill that ran, never the string that led to it."""
    skill = _make_skill(name="morning-routine")
    registry = _ResolvingRegistry({"morning-routine": skill})
    set_skill_context(SkillContext(registry=registry, runner=_FakeRunner()))  # type: ignore[arg-type]

    result = await RunSkillTool().execute({"skill_name": "MORNING ROUTINE"}, _ctx())

    assert result.output["skill_name"] == "morning-routine"


@pytest.mark.asyncio
async def test_run_skill_unknown_name_lists_what_is_installed() -> None:
    """A bare dead end ends the turn; naming the alternatives lets the model
    correct itself before the user hears anything."""
    registry = _ResolvingRegistry({"morning-routine": _make_skill(name="morning-routine")})
    set_skill_context(SkillContext(registry=registry, runner=_FakeRunner()))  # type: ignore[arg-type]

    result = await RunSkillTool().execute({"skill_name": "brew coffee"}, _ctx())

    assert result.success is False
    assert "Unknown skill: brew coffee" in result.error
    assert "morning-routine" in result.error


@pytest.mark.asyncio
async def test_run_skill_still_works_on_a_registry_without_resolve() -> None:
    """Mock and headless boots ship a reduced registry; the tool must not
    require the newer method to function."""
    skill = _make_skill(name="demo_skill")
    set_skill_context(
        SkillContext(registry=_FakeRegistry({"demo_skill": skill}), runner=_FakeRunner())  # type: ignore[arg-type]
    )

    result = await RunSkillTool().execute({"skill_name": "demo_skill"}, _ctx())

    assert result.success is True


# ----------------------------------------------------------------------
# Music: the loaded skill follows the CONNECTED service
# ----------------------------------------------------------------------


def _music_skill(plugin_id: str) -> Skill:
    fm = SkillFrontmatter(
        schema_version="1",
        name=f"plugin-{plugin_id}",
        description=f"Play music on {plugin_id}.",
        plugin_id=plugin_id,
        risk_policy=SkillRiskPolicy(default_tier="monitor"),  # type: ignore[arg-type]
        execution="inline",  # type: ignore[arg-type]
    )
    return Skill(
        path=Path("nonexistent") / f"plugin-{plugin_id}" / "SKILL.md",
        frontmatter=fm,
        body=f"Use {plugin_id}.",
        state=SkillLifecycleState.ACTIVE,
        body_hash="deadbeef",
        error=None,
    )


def _music_registry() -> _FakeRegistry:
    return _FakeRegistry(
        {
            "plugin-spotify": _music_skill("spotify"),
            "plugin-youtube_music": _music_skill("youtube_music"),
        }
    )


@pytest.mark.asyncio
async def test_a_music_skill_load_follows_the_only_connected_service(monkeypatch) -> None:
    """Live 2026-08-22 18:16: the model asked for ``plugin-spotify`` on a box
    where only YouTube Music was connected, and held Spotify's how-to."""
    from dodaios.core import music_service as ms

    monkeypatch.setattr(ms, "connected_music_services", lambda: ("youtube_music",))
    monkeypatch.setattr(ms, "preferred_music_service", lambda: "auto")
    runner = _FakeRunner()
    set_skill_context(SkillContext(registry=_music_registry(), runner=runner))  # type: ignore[arg-type]

    result = await RunSkillTool().execute({"skill_name": "plugin-spotify"}, _ctx())

    assert result.success is True
    assert result.output["skill_name"] == "plugin-youtube_music"
    assert runner.calls[0].skill.name == "plugin-youtube_music"


@pytest.mark.asyncio
async def test_a_named_music_service_is_kept(monkeypatch) -> None:
    """The user said Spotify: that wins even when YouTube Music is connected too."""
    from dodaios.core import music_service as ms

    monkeypatch.setattr(
        ms, "connected_music_services", lambda: ("spotify", "youtube_music")
    )
    monkeypatch.setattr(ms, "preferred_music_service", lambda: "youtube_music")
    set_skill_context(SkillContext(registry=_music_registry(), runner=_FakeRunner()))  # type: ignore[arg-type]
    ctx = ExecutionContext(
        trace_id=uuid4(),
        user_utterance="Spiel Jazz auf Spotify.",  # i18n-allow: spoken-input fixture
        config={},
        memory_read=None,
    )

    result = await RunSkillTool().execute({"skill_name": "plugin-spotify"}, ctx)

    assert result.success is True
    assert result.output["skill_name"] == "plugin-spotify"


@pytest.mark.asyncio
async def test_a_music_skill_stays_when_nothing_is_connected(monkeypatch) -> None:
    """Nothing to route to: the skill the model named is loaded as before, and
    its own instructions tell the user the service is not connected."""
    from dodaios.core import music_service as ms

    monkeypatch.setattr(ms, "connected_music_services", lambda: ())
    monkeypatch.setattr(ms, "preferred_music_service", lambda: "auto")
    set_skill_context(SkillContext(registry=_music_registry(), runner=_FakeRunner()))  # type: ignore[arg-type]

    result = await RunSkillTool().execute({"skill_name": "plugin-spotify"}, _ctx())

    assert result.success is True
    assert result.output["skill_name"] == "plugin-spotify"


def test_the_skill_loader_declares_that_it_only_yields_instructions() -> None:
    """The flag the realtime honesty guard reads.

    ``run-skill`` returns a skill's steps for the model to follow; it performs
    nothing itself. Callers asking "did this turn actually do anything?" must
    be able to tell that apart from a tool with an effect, which is what a
    2026-08-22 voice turn got wrong when it announced music that never played.
    """
    assert RunSkillTool.yields_instructions_only is True