Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

19.9 KB
Raw Blame
"""Pixels, identity and receipts survive George -> workspace -> coding CLI."""

from __future__ import annotations

import asyncio
import base64
import io
import re
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4

import pytest
from PIL import Image

from dodaios.agent_chat.attachments import handoff_context
from dodaios.agentic_ide import fleet_actions, prompt_history
from dodaios.agentic_ide.control import CodingSessionControl
from dodaios.agentic_ide.drop_analysis import DropAnalysis
from dodaios.brain.dispatcher import BrainDispatcher
from dodaios.brain.workspace_tool import WorkspaceOrchestrationTool
from dodaios.core import image_references
from dodaios.core.protocols import BrainDelta, ExecutionContext, ImageBlock, ToolResult
from dodaios.plugins.tool.appshot import AppshotTool
from tests.contract.test_workspace_orchestration import rig, runnable  # noqa: F401
from tests.fakes.fake_visual_sftp import FakeImagePool, FakeSftp


def png(color="red"):
    out = io.BytesIO()
    Image.new("RGB", (24, 16), color).save(out, format="PNG")
    return out.getvalue()


@pytest.fixture(autouse=True)
def image_store(monkeypatch):
    now = [0.0]
    store = image_references.ImageReferences(clock=lambda: now[0], schedule_expiry=False)
    monkeypatch.setattr(image_references, "_STORE", store)
    return store, now


@pytest.fixture
def delivery(rig, runnable, monkeypatch, tmp_path):  # noqa: F811
    orchestrator, registry, _ = rig
    registry._pty.tui_echo = True
    control = CodingSessionControl(registry)
    orchestrator.sessions = control
    monkeypatch.setattr(prompt_history, "_store_dir", lambda: tmp_path / "prompt-history")

    async def ready(owner, names, **kwargs):
        return set(names)

    async def start(owner, term):
        # No real provider or subprocess. The registry still performs the PTY
        # paste, Enter, verification and durable prompt receipt itself.
        async def output(terminal_id, data):
            term.transcript.feed(data)

        async def closed(*args):
            pass  # Fake process has no independent lifecycle.

        process = await registry._pty.spawn(
            ("codex",), "codex", owner.folder, 120, 40, output, closed
        )
        term.pty_id = process.terminal_id
        term.status = "live"

    monkeypatch.setattr(control, "_start", start)
    monkeypatch.setattr(fleet_actions, "wait_for_input_line", ready)
    monkeypatch.setattr(fleet_actions, "wait_for_prompt_ready", ready)
    ctx = ExecutionContext(uuid4(), "Fix the pictured bug", {"live_session_id": "call-A"}, None)
    return WorkspaceOrchestrationTool(orchestrator), registry, ctx


async def source_ref(source, delivery, monkeypatch, tmp_path, image_store=None):
    tool, registry, ctx = delivery
    data = png()
    if source == "upload":
        from dodaios.agent_chat import attachments, runner_brain
        from dodaios.agent_chat.service import AgentChatService
        from dodaios.agent_chat.store import AgentChatStore
        from dodaios.agentic_ide.drops import dereference

        async def describe(readable):
            # Analysis is optional; an unavailable vision model must not cost the pixels.
            return [DropAnalysis(name, reference, "image") for name, _, reference in readable]

        monkeypatch.setattr(attachments, "_analyze", describe)
        monkeypatch.setattr(runner_brain, "brain_manager", lambda: None)
        attached = await attachments.ingest(tmp_path, uploads=[("source.png", data)])
        service = AgentChatService(AgentChatStore(tmp_path / "chat.sqlite"))
        chat = service.store.create_session(
            provider="fakeprov", model="", effort="", cwd=str(tmp_path), surface="jarvis",
        )
        await service.send(chat.session_id, "Fix the pictured bug", [a.to_dict() for a in attached])
        await service.cancel(chat.session_id)
        note = next(
            event["payload"]["text"] for event in service.store.list_events(chat.session_id)
            if event["kind"] == "user_message"
        )
        ctx = ExecutionContext(
            ctx.trace_id, ctx.user_utterance,
            {"approval_ref": "agent-chat:" + chat.session_id}, None,
        )
        ref = re.search(r"img_[0-9a-f]{32}", note)[0]
        # Original upload may disappear; the selected bytes remain bound.
        (tmp_path / dereference(attached[0].reference)).unlink()
    elif source == "voice":
        # A hotkey appshot supplied into a subscription voice call: the backend
        # sees the pixels for the whole call and must be able to forward them.
        from dodaios.live.config import LiveConfig
        from dodaios.live.subscription import SubscriptionLiveVoiceSession
        from tests.fakes.fake_subscription_session import (
            SubscriptionConnection,
            subscription_provider,
        )

        async def send(_value):
            pass  # No client socket in this contract.

        session = SubscriptionLiveVoiceSession(
            session_id="call-A", providers=[subscription_provider()],
            config=SimpleNamespace(
                live=LiveConfig(configured=True, auth_mode="chatgpt_subscription"),
                brain=SimpleNamespace(reply_language="en"),
                screen_context=SimpleNamespace(ttl_s=120),
            ),
            send_binary=send, send_json=send,
        )
        session._connection = SubscriptionConnection()
        assert await session.attach_appshot(data, "image/png", "Supplied appshot")
        ref = re.search(r"img_[0-9a-f]{32}", session._pending_images[0]["text"])[0]
        image_store[1][0] += 600  # The user keeps talking before asking for the fix.
    else:
        from dodaios.appshot import service
        from dodaios.plugins.tool import appshot

        shot = SimpleNamespace(
            image=data,
            mime="image/png",
            label="active window",
            width=24,
            height=16,
            app_name="Test",
            note="Filtered capture",
        )

        async def take(**kwargs):
            assert kwargs["deliver"] is False
            return service.AppshotResult(status="captured", shot=shot)

        monkeypatch.setattr(service, "take_appshot", take)
        monkeypatch.setattr(appshot, "_app_bus", lambda: None)
        monkeypatch.setattr(
            appshot,
            "_load_config",
            lambda: SimpleNamespace(screen_context=SimpleNamespace(ttl_s=120)),
        )
        result = await AppshotTool().execute({}, ctx)
        ref = result.output["image_ref"]
        assert base64.b64decode(result.output["_image"]["data"]) == data
    return ref, data, ctx


async def assignment(delivery, ctx, action, refs, *, prompt="Fix the pictured bug"):
    tool, registry, _ = delivery
    args = {"action": action, "prompt": prompt, "image_refs": refs}
    if action == "create":
        args.update(cli="Codex", workspace="Personal Jarvis")
    else:
        resolved = await tool.execute({"action": "resolve", "workspace": "Personal Jarvis"}, ctx)
        args.update(resolved.output["target"], request_id=resolved.output["request_id"])
    return await tool.execute(args, ctx), args


@pytest.mark.parametrize("source", ["upload", "appshot", "voice"])
@pytest.mark.parametrize("action", ["create", "send"])
async def test_actual_pixels_reach_new_and_existing_cli(
    source, action, delivery, monkeypatch, tmp_path, image_store
):
    ref, data, ctx = await source_ref(source, delivery, monkeypatch, tmp_path, image_store)
    # A screenshot from a previous topic must not be included by recency.
    old = image_references.get_store().add(
        image_references.scope_for(ctx.config), png("blue"), "image/png", source="unrelated"
    )
    result, args = await assignment(delivery, ctx, action, [ref])
    assert result.success, result
    receipt = result.output["deliveries"][0] if action == "create" else result.output
    assert receipt["status"] == "accepted", receipt
    images = receipt["images"]
    assert [row["image_ref"] for row in images] == [ref]
    path = Path(images[0]["path"])
    assert path.read_bytes() == data  # noqa: ASYNC240 - tiny local test image
    with Image.open(path) as actual:
        assert actual.getpixel((0, 0)) == (255, 0, 0)
    registry = delivery[1]
    writes = "".join(text for _, text in registry._pty.writes)
    assert path.as_posix() in writes and "Open every listed file" in writes
    assert old not in writes and "\r" in writes
    found = registry.find_terminal(
        receipt["target"]["terminal_id"], receipt["target"]["workspace_id"]
    )
    assert found[1].prompt_records[-1].attachments[0]["kind"] == "image"
    if action == "send":
        before = len(registry._pty.writes)
        repeated = await delivery[0].execute(args, ctx)
        assert repeated.output == result.output
        assert len(registry._pty.writes) == before


async def test_multiple_images_and_explicit_selection_required(delivery):
    tool, registry, ctx = delivery
    store = image_references.get_store()
    refs = [
        store.add(image_references.scope_for(ctx.config), png(c), "image/png", source="upload")
        for c in ("red", "green")
    ]
    missing = await tool.execute({"action": "create", "prompt": "Fix this", "cli": "Codex"}, ctx)
    assert not missing.success and missing.output["status"] == "image_selection_required"
    assert not registry._pty.writes
    result, _ = await assignment(delivery, ctx, "send", refs)
    assert result.success, result
    assert [r["image_ref"] for r in result.output["images"]] == refs
    assert len({r["path"] for r in result.output["images"]}) == 2


@pytest.mark.parametrize("failure", ["expired", "other_conversation", "missing", "invalid_bytes"])
async def test_unavailable_image_refuses_whole_task(failure, delivery, image_store):
    store, now = image_store
    tool, registry, ctx = delivery
    scope = (
        "chat:other" if failure == "other_conversation" else image_references.scope_for(ctx.config)
    )
    ref = store.add(
        scope, b"broken" if failure == "invalid_bytes" else png(), "image/png", source="appshot"
    )
    if failure == "expired":
        now[0] = 121
    if failure == "missing":
        ref = "img_missing"
    result, _ = await assignment(delivery, ctx, "send", [ref])
    assert not result.success, result
    assert not registry._pty.writes
    assert result.output["status"] == "not_accepted"


@pytest.mark.parametrize("ending", ["call_closed", "call_bound"])
async def test_live_call_reference_ends_with_the_call(ending, delivery, image_store):
    from dodaios.live.tools import LiveTools

    store, now = image_store
    _, registry, ctx = delivery
    image_references.appshot_context(
        "call-A", png(), "image/png", SimpleNamespace(screen_context=SimpleNamespace(ttl_s=120)),
    )
    [row] = store.available("live:call-A")
    if ending == "call_closed":
        await LiveTools(None, None, "call-A", language="en", backend_model="").close()
    else:
        now[0] = image_references.LIVE_CALL_TTL_SECONDS + 1
    result, _ = await assignment(delivery, ctx, "send", [row["id"]])
    assert not result.success and result.output["status"] == "not_accepted"
    assert not registry._pty.writes


async def test_expired_create_does_not_open_empty_pane(delivery, image_store):
    store, now = image_store
    _, registry, ctx = delivery
    ref = store.add(image_references.scope_for(ctx.config), png(), "image/png", source="appshot")
    before = sum(len(owner.terminals) for owner in registry.sessions)
    now[0] = 121
    result, _ = await assignment(delivery, ctx, "create", [ref])
    assert not result.success and "expired" in result.error
    assert sum(len(owner.terminals) for owner in registry.sessions) == before


def test_upload_cannot_read_outside_chat_folder(tmp_path, image_store):
    root = tmp_path / "chat"
    root.mkdir()
    (tmp_path / "private.png").write_bytes(png())
    note = handoff_context(
        str(root), [DropAnalysis("outside", '"../private.png"', "image")], session_id="isolated"
    )
    assert "unavailable" in note
    assert not image_store[0].available("chat:isolated")


async def test_image_paths_cannot_be_truncated(delivery):
    _, registry, ctx = delivery
    ref = image_references.get_store().add(
        image_references.scope_for(ctx.config), png(), "image/png", source="appshot"
    )
    result, _ = await assignment(delivery, ctx, "send", [ref], prompt="x" * 20000)
    assert not result.success and "truncate" in result.output["reason"]
    assert not registry._pty.writes


async def test_regular_loop_keeps_appshot_pixels_out_of_text_and_into_vision():
    class Brain:
        def __init__(self):
            self.requests = []

        async def complete(self, request):
            self.requests.append(request)
            if len(self.requests) == 1:
                yield BrainDelta(tool_call={"id": "shot", "name": "take_appshot", "input": {}})
                yield BrainDelta(finish_reason="tool_use")
            else:
                yield BrainDelta(content="Captured", finish_reason="stop")

    data = base64.b64encode(png()).decode("ascii")

    class Executor:
        async def execute(self, *args, **kwargs):
            return ToolResult(
                True, {"image_ref": "img_test", "_image": {"mime": "image/png", "data": data}}
            )

    brain = Brain()
    await BrainDispatcher(
        brain, tools={"take_appshot": AppshotTool()}, executor=Executor()
    ).dispatch("Take an appshot")
    messages = brain.requests[-1].messages
    assert any(image.data_b64 == data for message in messages for image in message.images)
    assert all(data not in str(message.content) for message in messages)


async def test_user_image_input_has_scoped_ids_before_tool_routing(image_store):
    class Brain:
        async def complete(self, request):
            message = request.messages[-1]
            ref = re.search(r"img_[0-9a-f]{32}", message.content)[0]
            assert image_store[0].resolve("turn:upload", [ref])[0].data == png()
            yield BrainDelta(content="Seen", finish_reason="stop")

    await BrainDispatcher(Brain()).dispatch(
        "Fix this",
        trace_id="upload",
        images=(ImageBlock("image/png", base64.b64encode(png()).decode()),),
    )


@pytest.mark.parametrize("corrupt", [False, True])
async def test_remote_session_gets_verified_remote_bytes(delivery, monkeypatch, corrupt):
    from dodaios.computers import remote_terminal

    tool, registry, ctx = delivery
    owner = registry.sessions[0]
    term = owner.terminals[0]
    term.computer_id, term.remote_folder = "connected-host", "/work/project"
    sftp = FakeSftp()
    sftp.corrupt = corrupt
    monkeypatch.setattr(remote_terminal, "pool_for", lambda computer_id: FakeImagePool(sftp))
    # The remote PTY is the same recorded boundary; SFTP stays a separate channel.
    monkeypatch.setattr(registry, "_pool", lambda term: registry._pty)
    ref = image_references.get_store().add(
        image_references.scope_for(ctx.config), png(), "image/png", source="appshot"
    )
    result, _ = await assignment(delivery, ctx, "send", [ref])
    if corrupt:
        assert not result.success and not registry._pty.writes
        assert "verification failed" in result.output["reason"]
    else:
        assert result.success, result
        path = result.output["images"][0]["path"]
        assert path.startswith("/work/project/.jarvis/visual-references/")
        assert sftp.files["/sftp" + path] == png()
        assert sftp.files["/sftp/work/project/.jarvis/visual-references/.gitignore"] == b"*\n"
        assert path in "".join(text for _, text in registry._pty.writes)


async def test_changed_image_selection_is_a_new_assignment(delivery, monkeypatch):
    tool, registry, ctx = delivery
    store = image_references.get_store()
    refs = [
        store.add(image_references.scope_for(ctx.config), png(c), "image/png", source="upload")
        for c in ("red", "blue")
    ]
    first, args = await assignment(delivery, ctx, "send", [refs[0]])
    assert first.success
    term = registry.sessions[0].terminals[0]
    # A finished CLI is ready for a second independent work order.
    term.transcript.feed("\x1b[2J\x1b[H❯ ")
    term.last_submit_at = None
    # A still screen alone cannot prove completion to automatic selection.
    from dodaios.agentic_ide import task_state

    monkeypatch.setattr(task_state, "_read", lambda key, now: task_state.Evidence(
        "completed", now, now,
    ))
    reused = await tool.execute({**args, "image_refs": [refs[1]]}, ctx)
    assert not reused.success  # A spent request ID cannot authorize different bytes.
    second, _ = await assignment(delivery, ctx, "send", [refs[1]])
    assert second.success, second
    assert second.output["images"][0]["image_ref"] == refs[1]
    assert len(term.prompt_records) == 2


async def test_destination_move_during_transfer_refuses_delivery(delivery, monkeypatch):
    from dodaios.agentic_ide import visual_handoff

    tool, registry, ctx = delivery
    original = visual_handoff.prepare

    async def moving(owner, term, images):
        result = await original(owner, term, images)
        term.folder = owner.folder + "/moved"
        return result

    monkeypatch.setattr(visual_handoff, "prepare", moving)
    ref = image_references.get_store().add(
        image_references.scope_for(ctx.config), png(), "image/png", source="appshot"
    )
    result, _ = await assignment(delivery, ctx, "send", [ref])
    assert not result.success and "destination changed" in result.output["reason"]
    assert not registry._pty.writes


async def test_local_copy_lives_in_panes_own_worktree(delivery, tmp_path):
    _, registry, ctx = delivery
    own = tmp_path / "isolated-worktree"
    own.mkdir()
    registry.sessions[0].terminals[0].folder = str(own)
    ref = image_references.get_store().add(
        image_references.scope_for(ctx.config), png(), "image/png", source="upload"
    )
    result, _ = await assignment(delivery, ctx, "send", [ref])
    assert result.success
    assert Path(result.output["images"][0]["path"]).is_relative_to(own)


async def test_sftp_transfer_against_real_loopback_server(tmp_path, monkeypatch):
    """Exercise binary mode, exclusive creation, permissions and readback on real SFTP."""
    ssh = pytest.importorskip("asyncssh")
    from dodaios.agentic_ide.visual_handoff import prepare
    from dodaios.computers import remote_terminal

    class Server(ssh.SSHServer):
        def begin_auth(self, username):
            return False  # Test-only loopback server has no user accounts or credentials.

    project = tmp_path / "project"
    project.mkdir()
    key = ssh.generate_private_key("ssh-ed25519")
    server = await ssh.listen(
        "127.0.0.1",
        0,
        server_factory=Server,
        server_host_keys=[key],
        sftp_factory=lambda channel: ssh.SFTPServer(channel, chroot=str(tmp_path)),
    )
    try:
        async with ssh.connect("127.0.0.1", server.get_port(), known_hosts=None) as conn:

            class Pool:
                async def sftp_path(self, path):
                    return path

                async def connection(self):
                    return SimpleNamespace(conn=conn)

            monkeypatch.setattr(remote_terminal, "pool_for", lambda computer_id: Pool())
            ref = image_references.get_store().add("test", png(), "image/png", source="upload")
            images = image_references.get_store().resolve("test", [ref])
            term = SimpleNamespace(
                computer_id="test-loopback", remote_folder="/project", placing=False, agent="codex"
            )
            _, _, receipts = await prepare(None, term, images)
            local_path = tmp_path / receipts[0]["path"].lstrip("/")
            assert await asyncio.to_thread(local_path.read_bytes) == png()
    finally:
        server.close()
        await server.wait_closed()


async def test_idle_reference_bytes_expire_without_another_read():
    store = image_references.ImageReferences()
    store.add("short", png(), "image/png", source="appshot", ttl_s=0.02)
    for _ in range(100):
        if not store._entries:
            break
        await asyncio.sleep(0.01)
    assert not store._entries