Skip to content

The agent that grows with you

fix(webapp): a down SSH host no longer stalls file requests on other SSH profiles

unsupportedpastels140560affcb7
diff --git a/hermes_cli/ssh_workspace_fs.py b/hermes_cli/ssh_workspace_fs.py
index 0d757ffdb81..2e4fcfa8eb7 100644
--- a/hermes_cli/ssh_workspace_fs.py
+++ b/hermes_cli/ssh_workspace_fs.py
@@ -34,8 +34,11 @@ class _SshFsConfig:
timeout: int
+# ``_CACHE_LOCK`` guards only the dicts; a dial holds just its profile's lock, so one unreachable host
+# cannot stall file requests on every other SSH profile.
_CACHE_LOCK = threading.RLock()
_BACKENDS: dict[str, tuple[_SshFsConfig, SshWorkspaceFs]] = {}
+_DIAL_LOCKS: dict[str, threading.Lock] = {}
class SshWorkspaceFs:
@@ -281,11 +284,12 @@ def get_ssh_workspace_fs(profile_key: str, terminal_config: dict[str, Any]) -> S
raise SshWorkspaceFsError("ECONN", "SSH host and user are required")
with _CACHE_LOCK:
- cached = _BACKENDS.get(profile_key)
+ dial_lock = _DIAL_LOCKS.setdefault(profile_key, threading.Lock())
+ with dial_lock:
+ with _CACHE_LOCK:
+ cached = _BACKENDS.get(profile_key)
if cached and cached[0] == config:
return cached[1]
- if cached:
- cached[1]._env.cleanup()
try:
env = SSHEnvironment(
host=config.host,
@@ -297,7 +301,14 @@ def get_ssh_workspace_fs(profile_key: str, terminal_config: dict[str, Any]) -> S
sync_files=False,
)
except Exception as exc:
+ with _CACHE_LOCK:
+ _BACKENDS.pop(profile_key, None)
+ if cached:
+ cached[1]._env.cleanup()
raise SshWorkspaceFsError("ECONN", "Could not connect to the SSH workspace") from exc
backend = SshWorkspaceFs(env)
- _BACKENDS[profile_key] = (config, backend)
+ with _CACHE_LOCK:
+ _BACKENDS[profile_key] = (config, backend)
+ if cached:
+ cached[1]._env.cleanup()
return backend
diff --git a/tests/hermes_cli/test_ssh_workspace_fs.py b/tests/hermes_cli/test_ssh_workspace_fs.py
index ec00b86045c..8958a8fbc80 100644
--- a/tests/hermes_cli/test_ssh_workspace_fs.py
+++ b/tests/hermes_cli/test_ssh_workspace_fs.py
@@ -1,5 +1,6 @@
import base64
import io
+import threading
import pytest
@@ -150,3 +151,102 @@ def test_ssh_workspace_factory_disables_agent_file_sync(monkeypatch):
"sync_files": False,
}
]
+
+
+def _ssh_terminal(host: str) -> dict:
+ return {"backend": "ssh", "cwd": "/srv/repos", "ssh_host": host, "ssh_user": "dev"}
+
+
+def gated_ssh(monkeypatch):
+ """A fake SSHEnvironment that records each dial; a dial to a ``hanging`` host blocks until ``release``."""
+
+ class GatedSshEnvironment:
+ created: list = []
+ cleaned: list = []
+ hanging: set = set()
+ failing: set = set()
+ dialing = threading.Event()
+ release = threading.Event()
+
+ def __init__(self, **kwargs):
+ self.created.append(kwargs["host"])
+ if kwargs["host"] in self.hanging:
+ self.dialing.set()
+ self.release.wait(5)
+ if kwargs["host"] in self.failing:
+ raise OSError("connection timed out")
+ self.host = kwargs["host"]
+ self.cwd = kwargs["cwd"]
+ self.timeout = kwargs["timeout"]
+ self._remote_home = "/home/dev"
+
+ def cleanup(self):
+ self.cleaned.append(self.host)
+
+ monkeypatch.setattr(ssh_workspace_fs, "SSHEnvironment", GatedSshEnvironment)
+ monkeypatch.setattr(ssh_workspace_fs, "_BACKENDS", {})
+ yield GatedSshEnvironment
+ GatedSshEnvironment.release.set()
+
+
+def test_ssh_workspace_factory_serves_other_profiles_while_one_host_hangs(gated_ssh):
+ gated_ssh.hanging.add("down.example")
+ results: dict = {}
+
+ def open_profile(profile_key: str, host: str) -> None:
+ results[profile_key] = ssh_workspace_fs.get_ssh_workspace_fs(profile_key, _ssh_terminal(host))
+
+ down = threading.Thread(target=open_profile, args=("down", "down.example"), daemon=True)
+ up = threading.Thread(target=open_profile, args=("up", "up.example"), daemon=True)
+ down.start()
+ try:
+ assert gated_ssh.dialing.wait(5)
+ up.start()
+ up.join(2)
+ assert not up.is_alive(), "a hanging dial on one profile blocked another profile"
+ assert results["up"].cwd == "/srv/repos"
+ assert "down" not in results
+ finally:
+ gated_ssh.release.set()
+ down.join(5)
+ if up.ident is not None:
+ up.join(5)
+ assert results["down"] is not None
+
+
+def test_ssh_workspace_factory_dials_one_profile_once_for_concurrent_requests(gated_ssh):
+ gated_ssh.hanging.add("ssh.example")
+ backends: list = []
+
+ def open_profile() -> None:
+ backends.append(ssh_workspace_fs.get_ssh_workspace_fs("remote-dev", _ssh_terminal("ssh.example")))
+
+ threads = [threading.Thread(target=open_profile, daemon=True) for _ in range(2)]
+ threads[0].start()
+ assert gated_ssh.dialing.wait(5)
+ threads[1].start()
+ gated_ssh.release.set()
+ for thread in threads:
+ thread.join(5)
+ assert not thread.is_alive()
+
+ assert gated_ssh.created == ["ssh.example"]
+ assert len(backends) == 2 and backends[0] is backends[1]
+
+
+def test_ssh_workspace_factory_replaces_changed_config_and_drops_failed_redial(gated_ssh):
+ first = ssh_workspace_fs.get_ssh_workspace_fs("remote-dev", _ssh_terminal("old.example"))
+ assert ssh_workspace_fs.get_ssh_workspace_fs("remote-dev", _ssh_terminal("old.example")) is first
+
+ second = ssh_workspace_fs.get_ssh_workspace_fs("remote-dev", _ssh_terminal("new.example"))
+ assert second is not first
+ assert gated_ssh.cleaned == ["old.example"]
+
+ gated_ssh.failing.add("down.example")
+ with pytest.raises(SshWorkspaceFsError, match="Could not connect") as raised:
+ ssh_workspace_fs.get_ssh_workspace_fs("remote-dev", _ssh_terminal("down.example"))
+
+ assert raised.value.code == "ECONN"
+ assert gated_ssh.cleaned == ["old.example", "new.example"]
+ assert "remote-dev" not in ssh_workspace_fs._BACKENDS