The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
fix(groups): a standalone Desktop backend shows the gateway's Group Chats and hands its own rooms over once
Teknium29e10792b9e5
diff --git a/gateway/hosted_rooms_gateway_import.py b/gateway/hosted_rooms_gateway_import.pynew file mode 100644index 00000000000..a0a78802df7--- /dev/null+++ b/gateway/hosted_rooms_gateway_import.py@@ -0,0 +1,127 @@+"""Hand the standalone backend's Group Chats to the gateway that owns the profile (WA-19).++A native Desktop on a standalone ``hermes serve``/``hermes dashboard`` (no session authority) keeps+its rooms in the root ``shared-state.db`` and drives them in that process (``tui_gateway``+route), while the gateway keeps its own in the profile's ``state.db`` (owner claim and epoch fence+in the room insert's transaction). Once a gateway serves the profile, the standalone backend+forwards ``groups.*`` to it (``tui_gateway/methods_groups_forward.py``); this module moves the rooms+that backend already hosts into the gateway's store so the one list shows them.++Each room moves in two crash-safe steps, so it is driven by exactly one process:++1. **Fence the source.** The room's authority in ``shared-state.db`` moves to+ :func:`handoff_gateway_id` at ``epoch + 1`` (``authority.lost``, the RoomLink demotion). The+ standalone driver binds only rooms of its own authority and every room write checks it, so that+ copy is frozen from here on. The source rows are never deleted.+2. **Copy and claim.** A fenced room the gateway store does not hold is copied with its child rows,+ then claimed for this install at ``epoch + 1`` (``authority.claimed``) and owned by the host+ owner, the subject native Desktop and a signed-in Webapp tab both act as.++Re-running is a no-op: step 1 only takes rooms still on the local authority, step 2 only copies+fenced rooms the target lacks. A crash between them is finished by the next start.++``shared-state.db`` is one root file with no profile column, so only the root home's gateway (the+default profile, whose ``state.db`` sits beside it) imports; a named profile's standalone rooms stay+where they are for gateway-less use.+"""++from __future__ import annotations++import logging+from pathlib import Path++logger = logging.getLogger(__name__)++SOURCE_NAME = "shared-state.db"+_OWNER = "gateway.hosted.owner.v1:"+++def handoff_gateway_id() -> str:+ """The authority a standalone room is fenced to once the gateway store holds it."""+ from gateway.hosted_rooms import local_authority_gateway_id+ return f"{local_authority_gateway_id()}:gateway-store"+++def source_for(target_db: Path) -> Path | None:+ """The standalone store this gateway store imports from, or ``None`` (named profile, tests)."""+ target_db = Path(target_db)+ source = target_db.with_name(SOURCE_NAME)+ return source if target_db.name == "state.db" and source.is_file() else None+++def _fence_source(source: Path, local_id: str, handoff_id: str) -> int:+ from gateway.hosted_room_replicas import demote_room+ from gateway.hosted_rooms import list_rooms, MAX_ROOM_LIST_LIMIT+ fenced, offset = 0, 0+ while True:+ page = list_rooms(source, limit=MAX_ROOM_LIST_LIMIT, offset=offset)+ for room in page:+ if room["authority_gateway_id"] == local_id:+ demote_room(source, room_id=room["room_id"], observed_gateway_id=handoff_id,+ observed_epoch=int(room["authority_epoch"]) + 1)+ fenced += 1+ if len(page) < MAX_ROOM_LIST_LIMIT:+ return fenced+ offset += len(page)+++def _fenced_ids(source: Path, handoff_id: str) -> set[str]:+ from contextlib import closing+ from gateway.hosted_rooms import _read_connection+ with closing(_read_connection(source)) as conn:+ return {str(row[0]) for row in conn.execute(+ "SELECT room_id FROM hosted_rooms WHERE authority_gateway_id=? AND disbanded_at IS NULL",+ (handoff_id,))}+++def _claim(conn, room_id: str, local_id: str, handoff_id: str, owner: str, now: float) -> None:+ from gateway.hosted_room_replicas import _append_control_event, _control_event+ row = conn.execute("SELECT authority_epoch, next_seq FROM hosted_rooms WHERE room_id=?", (room_id,)).fetchone()+ epoch = int(row["authority_epoch"]) + 1+ _append_control_event(conn, room_id, int(row["next_seq"]), epoch, _control_event("claimed", epoch, {+ "previous_gateway_id": handoff_id, "authority_gateway_id": local_id, "authority_epoch": epoch}), now)+ conn.execute("""UPDATE hosted_rooms SET authority_gateway_id=?, authority_epoch=?, next_seq=next_seq+1,+ revision=revision+1, updated_at=? WHERE room_id=?""", (local_id, epoch, now, room_id))+ conn.execute("INSERT OR IGNORE INTO state_meta(key, value) VALUES (?, ?)", (_OWNER + room_id, owner))+++def import_standalone_rooms(target_db: Path, *, runtime_epoch: int | None = None) -> int:+ """Move the standalone backend's live rooms into ``target_db`` once; returns rooms copied.++ ``runtime_epoch``: the authority's writer epoch, checked inside the copy's transaction so a+ superseded gateway process never writes the store. Never raises: a locked or unreadable source+ leaves both stores as they were and is retried on the next start.+ """+ import sqlite3+ from gateway.hosted_rooms import HostedRoomError, _transaction, local_authority_gateway_id+ from gateway.hosted_rooms_common import clock+ from gateway.hosted_rooms_legacy_import import _copy_rows+ from gateway.owner_identity import host_owner_subject+ from hermes_state_runtime import RuntimeStoreError, _epoch++ source = source_for(target_db)+ if source is None:+ return 0+ try:+ local_id, handoff_id = local_authority_gateway_id(), handoff_gateway_id()+ _fence_source(source, local_id, handoff_id)+ fenced = _fenced_ids(source, handoff_id)+ if not fenced:+ return 0+ owner, now = host_owner_subject(), clock(None)+ with _transaction(target_db, immediate=True) as conn:+ if runtime_epoch is not None:+ _epoch(conn, runtime_epoch)+ held = {str(row[0]) for row in conn.execute("SELECT room_id FROM hosted_rooms")}+ missing = frozenset(fenced - held)+ if not missing:+ return 0+ copied = _copy_rows(conn, source, only=missing)+ for room_id in sorted(missing):+ _claim(conn, room_id, local_id, handoff_id, owner, now)+ except (OSError, sqlite3.Error, HostedRoomError, RuntimeStoreError) as exc:+ logger.warning("hosted rooms: could not import the standalone Group Chats from %s (%s); "+ "will retry on the next start", source, exc)+ return 0+ logger.info("hosted rooms: imported %d standalone Group Chat(s) from %s", copied, source)+ return copieddiff --git a/gateway/hosted_rooms_legacy_import.py b/gateway/hosted_rooms_legacy_import.pyindex 3aae609e073..b2f3d113537 100644--- a/gateway/hosted_rooms_legacy_import.py+++ b/gateway/hosted_rooms_legacy_import.py@@ -66,8 +66,9 @@ def _select_expressions(legacy: sqlite3.Connection, target: sqlite3.Connection,return pairs-def _copy_rows(target: sqlite3.Connection, source: Path) -> int:+def _copy_rows(target: sqlite3.Connection, source: Path, *, only: frozenset[str] | None = None) -> int:"""Copy every room ``target`` does not have yet, with all of its child rows; returns rooms copied.+ ``only``: copy just these room ids and no install-wide (room-less) table rows.A room id present in both stores is skipped as a unit: grafting the legacy events under thisstore's room would leave ``next_seq`` behind ``MAX(seq)`` and every later append colliding.@@ -97,8 +98,11 @@ def _copy_rows(target: sqlite3.Connection, source: Path) -> int:rows = legacy.execute(f"SELECT {', '.join(expr for _, expr in pairs)} FROM {name}")if "room_id" in columns:room_index = columns.index("room_id")- rows = (row for row in rows if str(row[room_index]) not in existing)+ rows = (row for row in rows if str(row[room_index]) not in existing+ and (only is None or str(row[room_index]) in only))verb = "INSERT"+ elif only is not None:+ continueelse:verb = "INSERT OR IGNORE"if name == "hosted_rooms":diff --git a/gateway/session_hosted_service.py b/gateway/session_hosted_service.pyindex 6c503edf05f..15e131ee9a4 100644--- a/gateway/session_hosted_service.py+++ b/gateway/session_hosted_service.py@@ -252,6 +252,8 @@ async def _ensure_hosted_service(runner, authority):service = getattr(authority, 'hosted_room_service', None)if service is None:loop = asyncio.get_running_loop()+ from gateway.hosted_rooms_gateway_import import import_standalone_rooms+ await asyncio.to_thread(import_standalone_rooms, Path(authority.db.db_path), runtime_epoch=authority.epoch)service = await asyncio.to_thread(CanonicalHostedRoomService, authority, loop)authority.hosted_room_service = serviceif not getattr(service, '_transport_installed', False):diff --git a/tests/gateway/test_hosted_rooms_gateway_import.py b/tests/gateway/test_hosted_rooms_gateway_import.pynew file mode 100644index 00000000000..1b5662f7ae7--- /dev/null+++ b/tests/gateway/test_hosted_rooms_gateway_import.py@@ -0,0 +1,67 @@+"""The gateway takes over the standalone backend's Group Chats once, and each room keeps one driver (WA-19)."""++import sqlite3+from pathlib import Path++from gateway import hosted_rooms+from gateway.hosted_rooms_gateway_import import handoff_gateway_id, import_standalone_rooms+from gateway.owner_identity import host_owner_subject++MEMBERS = [{"member_id": "a", "profile": "default", "handle": "a"},+ {"member_id": "b", "profile": "default", "handle": "b"}]+++def _stores(tmp_path: Path) -> tuple[Path, Path]:+ source, target = tmp_path / "shared-state.db", tmp_path / "state.db"+ with sqlite3.connect(target) as conn: # the gateway's SessionDB table the owner claim lives in+ conn.execute("CREATE TABLE state_meta (key TEXT PRIMARY KEY, value TEXT)")+ return source, target+++def _standalone_room(source: Path, room_id: str) -> None:+ hosted_rooms.create_room(source, room_id=room_id, name=room_id, members=MEMBERS,+ authority_gateway_id=hosted_rooms.local_authority_gateway_id())+ hosted_rooms.append_event(+ source, room_id=room_id, event_id=f"user:{room_id}", kind="message.user",+ actor={"kind": "user", "id": "desktop"}, payload={"text": "hello"},+ authority_gateway_id=hosted_rooms.local_authority_gateway_id(), authority_epoch=1)+++def test_standalone_rooms_move_once_with_history_owner_and_one_driver(tmp_path):+ source, target = _stores(tmp_path)+ _standalone_room(source, "room-desk")++ assert import_standalone_rooms(target) == 1+ assert import_standalone_rooms(target) == 0 # restart: no duplicate, no second claim++ local = hosted_rooms.local_authority_gateway_id()+ [room] = hosted_rooms.list_rooms(target)+ assert (room["room_id"], room["authority_gateway_id"]) == ("room-desk", local)+ kinds = [e["kind"] for e in hosted_rooms.read_events(target, room_id="room-desk")["events"]]+ assert kinds.count("message.user") == 1 and kinds[-2:] == ["authority.lost", "authority.claimed"]+ with sqlite3.connect(target) as conn:+ assert conn.execute("SELECT value FROM state_meta WHERE key=?",+ ("gateway.hosted.owner.v1:room-desk",)).fetchone() == (host_owner_subject(),)+ # The source is kept, fenced away from the standalone driver (it binds only its own authority).+ [kept] = hosted_rooms.list_rooms(source)+ assert kept["authority_gateway_id"] == handoff_gateway_id()+++def test_a_room_made_standalone_after_the_import_moves_on_the_next_start(tmp_path):+ source, target = _stores(tmp_path)+ _standalone_room(source, "room-one")+ import_standalone_rooms(target)+ _standalone_room(source, "room-two") # gateway was down: Desktop kept working standalone++ assert import_standalone_rooms(target) == 1+ assert sorted(r["room_id"] for r in hosted_rooms.list_rooms(target)) == ["room-one", "room-two"]+++def test_a_named_profile_store_imports_nothing(tmp_path):+ source, _ = _stores(tmp_path)+ _standalone_room(source, "room-desk")+ named = tmp_path / "profiles" / "work" / "state.db"+ named.parent.mkdir(parents=True)++ assert import_standalone_rooms(named) == 0+ assert hosted_rooms.list_rooms(source)[0]["authority_gateway_id"] == hosted_rooms.local_authority_gateway_id()diff --git a/tests/tui_gateway/test_groups_owner_forward.py b/tests/tui_gateway/test_groups_owner_forward.pynew file mode 100644index 00000000000..f6aa6790e7e--- /dev/null+++ b/tests/tui_gateway/test_groups_owner_forward.py@@ -0,0 +1,93 @@+"""A standalone backend sends Group Chat calls to the gateway that owns the profile (WA-19)."""++from __future__ import annotations++import pytest++import tui_gateway.server as srv+from tui_gateway import methods_groups, owner_forward++MEMBERS = [{"member_id": "default", "profile": "default", "handle": "hermes"},+ {"member_id": "ops", "profile": "ops", "handle": "ops"}]+++def home(tmp_path, monkeypatch):+ path = tmp_path / ".hermes"+ (path / "profiles" / "ops").mkdir(parents=True)+ (path / "profiles" / "ops" / "config.yaml").write_text("{}\n")+ monkeypatch.setenv("HERMES_HOME", str(path))+ methods_groups.stop_hosted_room_service(timeout=1.0)+ methods_groups.start_hosted_room_service()+ yield path+ methods_groups.stop_hosted_room_service(timeout=1.0)+++def owner(monkeypatch):+ """A ready gateway serving the home; records what reached it."""+ calls = []+ endpoint = object()+ monkeypatch.setattr(owner_forward, "discover_owner", lambda home: endpoint)++ def forward_rpc(home, seen_endpoint, method, params):+ assert seen_endpoint is endpoint+ calls.append((method, dict(params)))+ if method == "groups.capabilities":+ return {"driver": True, "methods": ["groups.list", "groups.create", "groups.attachment.upload"]}+ if method == "groups.list":+ return {"rooms": [{"room_id": "room-webapp"}], "next_offset": None}+ return {"room": {"room_id": params.get("room_id")}}+ monkeypatch.setattr(owner_forward, "forward_rpc", forward_rpc)+ return calls+++def test_room_calls_go_to_the_owning_gateway_and_nothing_lands_locally(home, owner):+ created = srv._methods["groups.create"](1, {"room_id": "room-desk", "name": "Desk", "members": MEMBERS})+ listed = srv._methods["groups.list"](2, {})++ assert created["result"]["room"]["room_id"] == "room-desk"+ assert listed["result"]["rooms"] == [{"room_id": "room-webapp"}]+ assert [method for method, _ in owner] == ["groups.create", "groups.list"]+ from gateway.hosted_rooms import default_db_path, list_rooms+ assert list_rooms(default_db_path()) == [] # no second store, no local driver for it+++def test_capabilities_advertise_only_what_this_wire_carries(home, owner):+ result = srv._methods["groups.capabilities"](1, {})["result"]+ assert result["methods"] == ["groups.list", "groups.create"]+++def test_owner_refusal_is_returned_not_run_locally(home, monkeypatch):+ from hermes_cli.gateway_client import GatewayRPCError+ monkeypatch.setattr(owner_forward, "discover_owner", lambda home: object())++ def refuse(*_args):+ raise GatewayRPCError("permission_denied", code=4030, data={"reason": "permission_denied"})+ monkeypatch.setattr(owner_forward, "forward_rpc", refuse)++ reply = srv._methods["groups.create"](1, {"room_id": "room-desk", "name": "Desk", "members": MEMBERS})+ assert reply["error"]["code"] == 4030 and reply["error"]["data"] == {"reason": "permission_denied"}+ from gateway.hosted_rooms import default_db_path, list_rooms+ assert list_rooms(default_db_path()) == []+++def test_without_a_gateway_rooms_stay_standalone(home):+ created = srv._methods["groups.create"](1, {"room_id": "room-desk", "name": "Desk", "members": MEMBERS})+ assert "error" not in created, created+ assert [r["room_id"] for r in srv._methods["groups.list"](2, {})["result"]["rooms"]] == ["room-desk"]+++def test_gateway_less_list_hides_rooms_handed_to_the_gateway(home):+ from gateway.hosted_rooms import default_db_path+ from gateway.hosted_rooms_gateway_import import import_standalone_rooms+ import sqlite3+ srv._methods["groups.create"](1, {"room_id": "room-desk", "name": "Desk", "members": MEMBERS})+ target = default_db_path().with_name("state.db")+ with sqlite3.connect(target) as conn:+ conn.execute("CREATE TABLE IF NOT EXISTS state_meta (key TEXT PRIMARY KEY, value TEXT)")+ assert import_standalone_rooms(target) == 1++ assert srv._methods["groups.list"](2, {})["result"]["rooms"] == []+ refused = srv._methods["groups.send"](3, {"room_id": "room-desk", "event_id": "e1", "payload": {"text": "hi"}})+ assert "error" in refused # the standalone copy is fenced: one driver per roomdiff --git a/tui_gateway/methods_groups.py b/tui_gateway/methods_groups.pyindex 2f94d7d17fb..e5360817370 100644--- a/tui_gateway/methods_groups.py+++ b/tui_gateway/methods_groups.py@@ -11,6 +11,9 @@ import importlibimport osimport threading+# Imported, so never rebound: it reads the bound server from this module, not server.py's globals.+from .methods_groups_forward import forward_bound as _forward_to_owner+_registry = HandlerRegistry()method = _registry.method@@ -189,9 +192,12 @@ def _room_method((only ``ReplicaError`` when ``replica_only``) to a client error with ``{"reason"}`` datawhen ``with_reason``; anything else maps to ``code``."""error_class = _room_error_class # closure cell: handlers run under server.py globals+ owner_call = _forward_to_ownerdef dec(fn):def handler(rid, params: dict) -> dict:+ if (forwarded := owner_call(rid, name, params)) is not None:+ return forwardedargs = (rid, params)if service_code is not None:service = get_hosted_room_service()@@ -214,8 +220,10 @@ def _room_method(@method("groups.capabilities")-def _(rid, params: dict, _catalog=_local_catalog, _methods=_METHODS) -> dict:+def _(rid, params: dict, _catalog=_local_catalog, _methods=_METHODS, _owner=_forward_to_owner) -> dict:"""Describe the hosted-room protocol implemented by this gateway."""+ if (forwarded := _owner(rid, "groups.capabilities", params)) is not None:+ return forwardedfrom gateway.hosted_rooms import MAX_LOG_LIMIT, PROTOCOL_VERSION, local_authority_gateway_idservice = get_hosted_room_service()driver_ready = bool(service and service.runtime.status()["running"])@@ -351,7 +359,8 @@ def _(rid, params: dict, db_path) -> dict:db_path, include_disbanded=params.get("include_disbanded") is True, limit=limit,offset=offset)next_offset = offset + limit if len(rooms) == limit else None- return _ok(rid, {"rooms": rooms, "next_offset": next_offset})+ from tui_gateway.methods_groups_forward import local_rooms+ return _ok(rid, {"rooms": local_rooms(rooms), "next_offset": next_offset})@_room_method(diff --git a/tui_gateway/methods_groups_forward.py b/tui_gateway/methods_groups_forward.pynew file mode 100644index 00000000000..b4b91187022--- /dev/null+++ b/tui_gateway/methods_groups_forward.py@@ -0,0 +1,80 @@+"""Standalone backend -> owning gateway for Group Chat calls (WA-19).++A standalone ``hermes serve``/``hermes dashboard`` has no session authority: on its own it keeps+rooms in the root ``shared-state.db`` and drives them here. Once a ready gateway serves the+profile, that gateway's store is the one Group Chat list native Desktop and the Webapp share, so+the room calls a Desktop makes go to it instead (the owner-forward path prompts already use:+``tui_gateway/owner_forward.py``). The gateway imports the rooms this backend already hosts once+(``gateway/hosted_rooms_gateway_import.py``) and fences them here, so each room has one driver.++With no gateway running, the calls run locally exactly as before. RoomLink's gateway-to-gateway+methods (``groups.peer.*``, replication, promote/demote) are not Desktop calls and stay local.+"""++from __future__ import annotations++import logging++logger = logging.getLogger(__name__)++#: The Desktop's room calls this wire serves; the gateway route serves each (GROUP_METHODS).+FORWARDED = frozenset({+ "groups.capabilities", "groups.list", "groups.create", "groups.state", "groups.log",+ "groups.send", "groups.rename", "groups.disband", "groups.stop", "groups.retry",+ "groups.approve"})+#: The owner was found but did not answer: a verdict for this call, never a local write instead.+OWNER_LOST_CODE = 5124+++def _home(server, params: dict):+ from pathlib import Path+ profile = str(params.get("profile") or "").strip()+ if profile and profile != str(server._current_profile_name() or "").strip():+ home = server._profile_home(profile)+ if home is None:+ raise ValueError(f"profile '{profile}' is unavailable")+ return Path(home)+ from hermes_constants import get_hermes_home+ return Path(get_hermes_home())+++def forward(server, rid, method: str, params: dict) -> dict | None:+ """The owning gateway's answer to ``method`` as this wire's envelope, or ``None`` when no ready+ gateway serves the profile (the caller runs it locally)."""+ from tui_gateway import owner_forward+ if method not in FORWARDED:+ return None+ try:+ home = _home(server, params)+ endpoint = owner_forward.discover_owner(home)+ except owner_forward.OwnerNotRunning:+ return None+ except ValueError as exc:+ return server._err(rid, 4000, str(exc))+ from hermes_cli.gateway_client import GatewayRPCError+ try:+ result = owner_forward.forward_rpc(home, endpoint, method, params)+ except GatewayRPCError as exc:+ return server._err(rid, exc.code if isinstance(exc.code, int) else 5000, str(exc), exc.data or None)+ except Exception as exc: # health: allow BLE001 -- transport boundary: any dial/IO failure is one refusal+ logger.warning("Group Chat %s: the gateway that owns it did not answer: %s", method, exc)+ return server._err(rid, OWNER_LOST_CODE, "The Hermes gateway did not answer. Try again.")+ result = result if isinstance(result, dict) else {}+ if method == "groups.capabilities" and isinstance(result.get("methods"), list):+ # Advertise only what this wire can carry (attachments and Discard have no contract here).+ result = {**result, "methods": [name for name in result["methods"] if name in FORWARDED]}+ return server._ok(rid, result)+++def forward_bound(rid, method: str, params: dict) -> dict | None:+ """:func:`forward` on the server ``methods_groups`` is bound to (``None``: none bound yet)."""+ from tui_gateway import methods_groups+ server = methods_groups._bound_server+ return None if server is None else forward(server, rid, method, params)+++def local_rooms(rooms: list[dict]) -> list[dict]:+ """A gateway-less listing without the rooms already handed to the gateway's store."""+ from gateway.hosted_rooms_gateway_import import handoff_gateway_id+ handoff = handoff_gateway_id()+ return [room for room in rooms if room.get("authority_gateway_id") != handoff]diff --git a/tui_gateway/owner_forward.py b/tui_gateway/owner_forward.pyindex 77d9d4347cb..db5e206b88b 100644--- a/tui_gateway/owner_forward.py+++ b/tui_gateway/owner_forward.py@@ -216,6 +216,14 @@ class _Relay:self.emit("message.complete", self.sid, {"text": ""})+def forward_rpc(home: Path, endpoint, method: str, params: dict) -> Any:+ """One request/answer RPC on the owner serving ``home`` (Group Chat calls, WA-19)."""+ async def call():+ async with connect_owner(home, endpoint) as client:+ return await client.rpc(method, **params)+ return asyncio.run(call())++def forward_prompt(sid: str, session: dict, text: str, *, emit=None, home: Path | None = None) -> dict:"""Admit ``text`` into the owner's FIFO for this session; the owner's receipt.