Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(computers): keep remote panes apart and connections alive

rubenluetke10-beepbdc6176a05cf
diff --git a/jarvis/computers/remote_terminal.py b/jarvis/computers/remote_terminal.py
index a340a65c04..4ea3fd9c73 100644
--- a/jarvis/computers/remote_terminal.py
+++ b/jarvis/computers/remote_terminal.py
@@ -29,6 +29,7 @@ import shlex
from collections.abc import Awaitable, Callable
from dataclasses import dataclass, field
from typing import Any
+from uuid import uuid4
from jarvis.computers.service import ComputerError, get_service
from jarvis.computers.ssh import Session, close
@@ -41,6 +42,8 @@ ProbeCallback = Callable[[str], str]
#: Reconnect delays after a dropped connection; jittered, then given up.
RECONNECT_DELAYS_S = (1.0, 2.0, 4.0, 8.0, 15.0, 30.0)
+#: How long opening one pane's channel may take before it counts as failed.
+OPEN_TIMEOUT_S = 30.0
_NAME_RE = re.compile(r"[^A-Za-z0-9_-]")
@@ -90,6 +93,9 @@ class _Pane:
on_closed: ClosedCallback
on_probe: ProbeCallback | None
process: Any = None
+ #: The connection ``process`` runs on — so a pane that lost it closes THAT
+ #: one only, never a fresh connection a sibling pane already re-opened.
+ session: Session | None = None
pump: asyncio.Task[None] | None = None
closing: bool = False
exit_status: int | None = field(default=None)
@@ -162,11 +168,18 @@ class SshPtyPool:
on_probe: ProbeCallback | None = None,
meta: dict[str, Any] | None = None,
) -> RemoteSpawn:
- """Start (or re-join) the pane's tmux session and stream it."""
+ """Start (or re-join) the pane's tmux session and stream it.
+
+ Every spawn gets its own terminal id, like the local pool's. The caller's
+ ``shell_id`` is built from the pane's call-sign, and call-signs restart
+ at T1 in every workspace — keyed on it, the second workspace's T1 on the
+ same server displaced the first one's channel, and its keystrokes went
+ to the other workspace's agent.
+ """
identity = str((meta or {}).get("history_id") or shell_id)
name = tmux_session_name(identity)
pane = _Pane(
- terminal_id=shell_id,
+ terminal_id=uuid4().hex,
tmux_name=name,
command=login_shell(tmux_command(name, tuple(shell_argv), cwd, cols, rows)),
cols=cols,
@@ -183,13 +196,17 @@ class SshPtyPool:
status=409,
)
self._tmux_checked = True
- previous = self._panes.pop(shell_id, None)
- if previous is not None:
- self._drop(previous)
+ # A new viewer of the SAME tmux session replaces the old channel; the
+ # agent inside is untouched (``tmux new-session -A`` re-joins it).
+ for other_id, other in list(self._panes.items()):
+ if other.tmux_name == name:
+ self._panes.pop(other_id, None)
+ other.closing = True
+ self._drop(other)
await self._open(pane)
- self._panes[shell_id] = pane
+ self._panes[pane.terminal_id] = pane
pane.pump = asyncio.create_task(self._pump(pane), name=f"ssh-pty-{name}")
- return RemoteSpawn(terminal_id=shell_id)
+ return RemoteSpawn(terminal_id=pane.terminal_id)
def has(self, terminal_id: str) -> bool:
pane = self._panes.get(terminal_id)
@@ -260,12 +277,16 @@ class SshPtyPool:
async def _open(self, pane: _Pane) -> None:
session = await self.connection()
- pane.process = await session.conn.create_process(
- pane.command,
- term_type="xterm-256color",
- term_size=(pane.cols, pane.rows),
- encoding="utf-8",
- errors="replace",
+ pane.session = session
+ pane.process = await asyncio.wait_for(
+ session.conn.create_process(
+ pane.command,
+ term_type="xterm-256color",
+ term_size=(pane.cols, pane.rows),
+ encoding="utf-8",
+ errors="replace",
+ ),
+ timeout=OPEN_TIMEOUT_S,
)
def _drop(self, pane: _Pane) -> None:
@@ -276,8 +297,10 @@ class SshPtyPool:
pane.process.close()
async def _kill_session(self, name: str) -> None:
- with contextlib.suppress(Exception):
+ try:
await self.run(f"tmux kill-session -t {shlex.quote(name)}", timeout_s=15)
+ except Exception as exc: # noqa: BLE001 — logged: an agent may still be running there
+ log.warning("computers: could not end tmux session %s: %s", name, exc)
async def _session_alive(self, name: str) -> bool | None:
"""True/False from the server; None when the server cannot be asked."""
@@ -319,7 +342,7 @@ class SshPtyPool:
if lost:
# The connection itself is gone; asking the server anything on
# it would hang until a timeout. Start the next call afresh.
- await self._forget_connection()
+ await self._forget_connection(pane.session)
with contextlib.suppress(Exception):
pane.exit_status = pane.process.exit_status
alive = await self._session_alive(pane.tmux_name)
@@ -333,25 +356,39 @@ class SshPtyPool:
return
async def _reattach(self, pane: _Pane) -> bool:
+ """Re-open the pane's channel on whatever connection is current.
+
+ Never closes a connection up front: with several panes on one server,
+ each pane doing so cut the channels its siblings had just re-opened,
+ and the pool reconnected in circles (dozens of logins a minute for six
+ panes). ``connection()`` opens a new one only when the shared one is
+ really closed, one caller at a time.
+ """
for delay in RECONNECT_DELAYS_S:
if pane.closing:
return False
await asyncio.sleep(delay * random.uniform(0.7, 1.3)) # noqa: S311 — jitter, not crypto
- await self._forget_connection()
try:
await self._open(pane)
except Exception as exc: # noqa: BLE001 — retried with backoff
log.info("computers: re-attaching %s failed: %s", pane.tmux_name, exc)
+ # The connection it tried may be half-dead; the next try opens
+ # a new one unless a sibling already did.
+ await self._forget_connection(pane.session)
continue
log.info("computers: re-attached remote pane %s", pane.tmux_name)
return True
return False
- async def _forget_connection(self) -> None:
+ async def _forget_connection(self, expected: Session | None = None) -> None:
+ """Drop the shared connection — only while it is still ``expected``."""
async with self._lock:
- if self._session is not None:
- close(self._session)
- self._session = None
+ if self._session is None:
+ return
+ if expected is not None and self._session is not expected:
+ return
+ close(self._session)
+ self._session = None
async def _finish(self, pane: _Pane, code: int) -> None:
if self._panes.get(pane.terminal_id) is pane:
diff --git a/jarvis/computers/service.py b/jarvis/computers/service.py
index 39519f9a5a..a9e269a273 100644
--- a/jarvis/computers/service.py
+++ b/jarvis/computers/service.py
@@ -295,7 +295,12 @@ class ComputerService:
now = time.time()
try:
async with self.session(computer_id) as opened:
- result = await run_command(opened, PROBE_SCRIPT, timeout_s=_PROBE_TIMEOUT_S)
+ try:
+ result = await run_command(opened, PROBE_SCRIPT, timeout_s=_PROBE_TIMEOUT_S)
+ except SshError as exc:
+ # A probe that times out or loses the channel is a health
+ # reading like a refused login, not a crash of the check.
+ raise ComputerError(exc.message, status=502, kind=exc.kind) from exc
latency = opened.latency_ms
except ComputerError as exc:
status = _ERROR_STATUS.get(exc.kind or "", "error")
diff --git a/jarvis/computers/ssh.py b/jarvis/computers/ssh.py
index 4a8a5ed537..cd0de229da 100644
--- a/jarvis/computers/ssh.py
+++ b/jarvis/computers/ssh.py
@@ -28,6 +28,9 @@ log = logging.getLogger(__name__)
SshErrorKind = Literal["unreachable", "timeout", "auth", "host_key_changed", "protocol"]
CONNECT_TIMEOUT_S = 12.0
+#: Keepalive probes: a silent connection counts as dead after about 45 s.
+KEEPALIVE_INTERVAL_S = 15
+KEEPALIVE_COUNT_MAX = 3
#: Longest a user command may run from the UI console.
MAX_COMMAND_TIMEOUT_S = 300.0
#: Output beyond this is cut (per stream) so one ``cat`` cannot flood the UI.
@@ -133,6 +136,11 @@ async def open_session(target: SshTarget, *, timeout_s: float = CONNECT_TIMEOUT_
"password": target.password,
"connect_timeout": timeout_s,
"login_timeout": timeout_s,
+ # A connection that went quiet (laptop sleep, a NAT idle timeout) is
+ # found dead within a minute instead of freezing every pane on it:
+ # its close is what makes the terminal pool reconnect.
+ "keepalive_interval": KEEPALIVE_INTERVAL_S,
+ "keepalive_count_max": KEEPALIVE_COUNT_MAX,
}
if target.client_key is None:
options["preferred_auth"] = "password,keyboard-interactive"
@@ -196,13 +204,21 @@ async def run_command(session: Session, command: str, *, timeout_s: float) -> Co
def authorize_key_command(public_key: str) -> str:
- """The idempotent shell command that adds ``public_key`` to authorized_keys."""
+ """The idempotent shell command that adds ``public_key`` to authorized_keys.
+
+ A file whose last line has no newline (hand-edited, or written by a panel)
+ gets one first: appending straight onto it would glue the new key to the
+ end of the last existing one and break both — the server would then refuse
+ the key just planted, and the user's own key too.
+ """
quoted = shlex.quote(public_key.strip())
+ keys = "~/.ssh/authorized_keys"
return (
- "umask 077 && mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && "
- f"(grep -qxF {quoted} ~/.ssh/authorized_keys || "
- f"printf '%s\\n' {quoted} >> ~/.ssh/authorized_keys) && "
- "chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys"
+ f"umask 077 && mkdir -p ~/.ssh && touch {keys} && "
+ f"(grep -qxF {quoted} {keys} || "
+ f'{{ if [ -s {keys} ] && [ -n "$(tail -c 1 {keys})" ]; then echo >> {keys}; fi; '
+ f"printf '%s\\n' {quoted} >> {keys}; }}) && "
+ f"chmod 700 ~/.ssh && chmod 600 {keys}"
)
diff --git a/tests/unit/computers/test_authorize_key.py b/tests/unit/computers/test_authorize_key.py
new file mode 100644
index 0000000000..2b510eece5
--- /dev/null
+++ b/tests/unit/computers/test_authorize_key.py
@@ -0,0 +1,48 @@
+"""Planting the app's key must never damage the keys already on the server."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from jarvis.computers.ssh import authorize_key_command
+from jarvis.core.process_utils import NO_WINDOW_CREATIONFLAGS
+
+SH = shutil.which("sh")
+pytestmark = pytest.mark.skipif(SH is None, reason="needs a POSIX shell")
+
+NEW_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAInew personal-jarvis@test"
+OLD_KEY = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIold"
+
+
+def _run(home: Path) -> None:
+ subprocess.run( # noqa: S603 — fixed argv
+ [str(SH), "-c", authorize_key_command(NEW_KEY)],
+ env={"HOME": str(home), "PATH": str(Path(str(SH)).parent)},
+ check=True,
+ creationflags=NO_WINDOW_CREATIONFLAGS,
+ )
+
+
+def _lines(home: Path) -> list[str]:
+ return (home / ".ssh" / "authorized_keys").read_text(encoding="utf-8").splitlines()
+
+
+def test_a_last_line_without_newline_is_not_glued_to_the_new_key(tmp_path: Path) -> None:
+ ssh = tmp_path / ".ssh"
+ ssh.mkdir()
+ (ssh / "authorized_keys").write_bytes(OLD_KEY.encode()) # no trailing newline
+
+ _run(tmp_path)
+
+ assert _lines(tmp_path) == [OLD_KEY, NEW_KEY]
+
+
+def test_planting_twice_adds_the_key_once(tmp_path: Path) -> None:
+ _run(tmp_path)
+ _run(tmp_path)
+
+ assert _lines(tmp_path) == [NEW_KEY]
diff --git a/tests/unit/computers/test_remote_terminal.py b/tests/unit/computers/test_remote_terminal.py
index aad83f3680..5916e457b5 100644
--- a/tests/unit/computers/test_remote_terminal.py
+++ b/tests/unit/computers/test_remote_terminal.py
@@ -166,3 +166,49 @@ def test_tmux_command_is_create_or_attach_and_quiet() -> None:
assert "status off" in command
assert "'/srv/my app'" in command
assert remote_terminal.tmux_session_name("a/b c") == "jv-a-b-c"
+
+
+async def test_two_workspaces_same_call_sign_stay_apart(
+ pool: SshPtyPool, tmux_server: FakeTmuxServer
+) -> None:
+ """Call-signs restart at T1 per workspace; the pool must not key on them."""
+ first, second = Screen(), Screen()
+ one = await _spawn(pool, first, identity_id="ws1-t1")
+ two = await _spawn(pool, second, identity_id="ws2-t1")
+ await first.wait_for("ready")
+ await second.wait_for("ready")
+
+ assert one != two
+ assert pool.has(one) and pool.has(two)
+ assert pool.write(one, "to-first")
+ await first.wait_for("echo:to-first")
+ assert "to-first" not in second.text
+
+
+async def test_several_panes_re_attach_once_each_after_a_drop(
+ pool: SshPtyPool, tmux_server: FakeTmuxServer, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A drop must not turn into panes cutting each other's fresh channels."""
+ monkeypatch.setattr(remote_terminal, "RECONNECT_DELAYS_S", (0.05, 0.1, 0.1, 0.1))
+ screens = [Screen() for _ in range(4)]
+ terminals = [
+ await _spawn(pool, screen, identity_id=f"pane{index}")
+ for index, screen in enumerate(screens)
+ ]
+ for screen in screens:
+ await screen.wait_for("ready")
+
+ tmux_server.drop_connections()
+ for _ in range(300):
+ if len(tmux_server.tmux.attaches) >= len(screens):
+ break
+ await asyncio.sleep(0.02)
+ await asyncio.sleep(0.4)
+
+ assert sorted(tmux_server.tmux.attaches) == sorted(
+ tmux_session_name(f"pane{index}") for index in range(4)
+ ), "every pane re-joined exactly once"
+ for terminal, screen in zip(terminals, screens, strict=True):
+ assert screen.closed == []
+ assert pool.write(terminal, "still-here")
+ await screen.wait_for("echo:still-here")