Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat(chat): relay a standalone worker's chat events to the backend
trohitg77be4d4b9dc1
diff --git a/server/config/chat_defaults.json b/server/config/chat_defaults.jsonindex fd5f69f2..c4a6e11a 100644--- a/server/config/chat_defaults.json+++ b/server/config/chat_defaults.json@@ -7,7 +7,10 @@"stop_grace_s": 45},"hub": {- "subscriber_queue_size": 512+ "subscriber_queue_size": 512,+ "_relay_comment": "A standalone worker sends its chat events to the backend's hub (services/chat/relay.py): at most relay_queue_size wait to go, relay_batch_size to a frame.",+ "relay_queue_size": 4096,+ "relay_batch_size": 200},"stream": {"flush_ms": 50,diff --git a/server/services/approvals/events.py b/server/services/approvals/events.pyindex 8439c576..4923f9a3 100644--- a/server/services/approvals/events.py+++ b/server/services/approvals/events.py@@ -6,7 +6,8 @@ or discarded), ``undone``, ``restored``, ``sending``, ``sent``, ``failed``,the message, never who it goes to. The client refetches the drafts it shows(``list_approvals`` / ``get_approvals``) through the authorized handlers.-Broadcast directly: no Temporal consumer listens for these.+Broadcast directly: no Temporal consumer listens for these. On a standalone+worker the backend sends them (``services/chat/relay.py``)."""from __future__ import annotations@@ -36,10 +37,16 @@ def approval_lifecycle_event(change: ApprovalChange) -> WorkflowEvent:async def broadcast_approval_change(change: ApprovalChange) -> None:+ from services.chat.relay import active_relayfrom services.status_broadcaster import get_status_broadcasterevent = approval_lifecycle_event(change)- await get_status_broadcaster().broadcast({"type": WIRE_KEY, "data": event.model_dump(mode="json", exclude_none=True)})+ data = event.model_dump(mode="json", exclude_none=True)+ relay = active_relay()+ if relay is not None:+ relay.offer_broadcast(WIRE_KEY, data, session_id=change.workflow_id)+ return+ await get_status_broadcaster().broadcast({"type": WIRE_KEY, "data": data})__all__ = ["SOURCE", "WIRE_KEY", "approval_lifecycle_event", "broadcast_approval_change"]diff --git a/server/services/authz/ws_surface.py b/server/services/authz/ws_surface.pyindex 4f49dcc2..737e3c67 100644--- a/server/services/authz/ws_surface.py+++ b/server/services/authz/ws_surface.py@@ -62,10 +62,15 @@ def is_internal_caller(headers: Mapping[str, str], secret_key: str) -> bool:#: Everything the activity worker legitimately needs. Both execute#: handlers already carry their own ``/ws/internal`` identity branch.+#: ``chat_run_publish`` is how a standalone worker's chat events reach the+#: backend's sockets (``services/chat/relay.py``): it publishes run events+#: and two identity-only broadcasts, reads nothing and writes no row, and+#: refuses any socket but this one.INTERNAL_SOCKET_HANDLERS: frozenset[str] = frozenset({"execute_node","execute_ai_node",+ "chat_run_publish","ping",})diff --git a/server/services/chat/__init__.py b/server/services/chat/__init__.pyindex 4480e551..5e2d2abc 100644--- a/server/services/chat/__init__.py+++ b/server/services/chat/__init__.py@@ -14,6 +14,7 @@ imports ``nodes/``.- ``activities``: the Temporal activities MachinaWorkflow starts andfinishes runs through.- ``watchdog``: the periodic sweep main.py runs.+- ``relay``: a standalone worker's events, sent to the backend's hub.Thread rows (and ``chat.updated``) stay with ``services/chat_thread.py``.@@ -28,6 +29,7 @@ from services.ws_handler_registry import register_ws_handlers as _register_ws_hafrom .handlers import WS_HANDLERS as _CHAT_WS_HANDLERSfrom .hub import get_chat_hub+from .relay import WS_HANDLERS as _RELAY_WS_HANDLERSdef _drop_socket(websocket) -> None:@@ -35,6 +37,7 @@ def _drop_socket(websocket) -> None:_register_ws_handlers(_CHAT_WS_HANDLERS)+_register_ws_handlers(_RELAY_WS_HANDLERS)_register_disconnect_listener(_drop_socket)__all__ = ["get_chat_hub"]diff --git a/server/services/chat/hub.py b/server/services/chat/hub.pyindex 9ba8aa8f..38da790d 100644--- a/server/services/chat/hub.py+++ b/server/services/chat/hub.py@@ -20,8 +20,9 @@ go through ``StatusBroadcaster.broadcast``, which reaches every socket.follows: the client takes a fresh snapshot instead of reading a streamwith a hole in it.-Delivery is in-process. An activity on a worker in another process-publishes into that process's hub, which no socket here reads.+Publish through :func:`publish_run_event`. It delivers in-process, except on+a standalone worker, where no socket is: there it hands the event to the relay+(``services/chat/relay.py``), which sends it to the backend's hub."""from __future__ import annotations@@ -229,6 +230,11 @@ class ChatRunHub:if subscriber is not None and session_id in subscriber.sessions:subscriber.offer(frame)+ def resync(self, session_id: str) -> None:+ """Tell the session's subscribers to take a fresh snapshot: events+ for it were lost before they reached this hub (the relay)."""+ self._fan_out(session_id, self.resync_frame(session_id))+def resync_frame(self, session_id: str) -> str:event = WorkflowEvent(id=f"resync:{uuid4().hex}",@@ -256,6 +262,40 @@ def get_chat_hub() -> ChatRunHub:return _hub+def publish_run_event(+ *,+ run_id: str,+ session_id: str,+ workflow_id: Optional[str],+ suffix: str,+ fields: Optional[Mapping[str, Any]] = None,+ event_key: Optional[str] = None,+) -> Optional[WorkflowEvent]:+ """Publish a run event where its subscribers are: this process's hub, or,+ on a standalone worker, the backend's through the relay (None then)."""+ from services.chat.relay import active_relay++ relay = active_relay()+ if relay is not None:+ relay.offer_run_event(+ run_id=run_id,+ session_id=session_id,+ workflow_id=workflow_id,+ suffix=suffix,+ fields=fields,+ event_key=event_key,+ )+ return None+ return get_chat_hub().publish(+ run_id=run_id,+ session_id=session_id,+ workflow_id=workflow_id,+ suffix=suffix,+ fields=fields,+ event_key=event_key,+ )++def reset_chat_hub_for_tests(hub: Optional[ChatRunHub] = None) -> Optional[ChatRunHub]:"""Swap the process hub (tests). Returns the previous one."""global _hubdiff --git a/server/services/chat/ledger.py b/server/services/chat/ledger.pyindex 630270b7..cdc65428 100644--- a/server/services/chat/ledger.py+++ b/server/services/chat/ledger.py@@ -62,7 +62,7 @@ from core.logging import get_loggerfrom models.chat import LIVE_STATES, TERMINAL_STATES, ChatRunfrom models.database import ChatMessagefrom services.chat.config import runs_setting, steps_setting-from services.chat.hub import get_chat_hub+from services.chat.hub import publish_run_eventfrom services.chat_thread import delivery_forlogger = get_logger(__name__)@@ -370,7 +370,7 @@ async def post_reply(def _publish(run: ChatRun, suffix: str, fields: Dict[str, Any], event_key: str) -> None:try:- get_chat_hub().publish(+ publish_run_event(run_id=run.run_id,session_id=run.session_id,workflow_id=run.workflow_id,diff --git a/server/services/chat/parts.py b/server/services/chat/parts.pyindex 04e06bde..318bd14b 100644--- a/server/services/chat/parts.py+++ b/server/services/chat/parts.py@@ -78,10 +78,10 @@ async def run_parts(database: Any, run_id: str) -> List[ChatRunPart]:def _publish(stream: Mapping[str, Any], suffix: str, fields: Dict[str, Any], event_key: str) -> None:try:- from services.chat.hub import get_chat_hub+ from services.chat.hub import publish_run_eventworkflow_id = stream.get("workflow_id")- get_chat_hub().publish(+ publish_run_event(run_id=stream["run_id"],session_id=stream["session_id"],workflow_id=workflow_id if isinstance(workflow_id, str) else None,diff --git a/server/services/chat/relay.py b/server/services/chat/relay.pynew file mode 100644index 00000000..5707d8a4--- /dev/null+++ b/server/services/chat/relay.py@@ -0,0 +1,443 @@+"""Chat events from a worker in another process.++The sockets that follow a chat session are in the backend, and so is the hub+that writes to them (``services/chat/hub.py``). An activity on a standalone+worker (``python -m services.temporal.worker``) runs where no socket is, so+there the run events it publishes, and the identity-only ``chat.updated`` and+``approval_lifecycle`` broadcasts it makes, travel to the backend over the+worker's own ``/ws/internal`` connection as ``chat_run_publish`` frames, in+the order they were made. The backend publishes each run event into its hub,+which numbers and de-duplicates it as it would its own, and sends each+broadcast to its sockets.++- **Only on a standalone worker.** ``run_standalone_worker`` starts the+ relay; the backend and its embedded workers publish straight into the hub.+- **In order.** One queue and one writer per process.+- **Bounded, never blocking.** Past ``hub.relay_queue_size``+ (``config/chat_defaults.json``) an event is dropped, and so is a frame the+ connection lost. The next frame that gets through ends with a resync for+ the sessions those belonged to, so their subscribers take a fresh snapshot+ instead of reading a stream with a hole in it. An event is a courtesy: the+ rows are the record.+- **Internal only.** The backend takes ``chat_run_publish`` from+ ``/ws/internal`` alone (the worker token, ``services/authz``) and refuses+ it on a client's socket.+"""++from __future__ import annotations++import asyncio+from collections import deque+from contextlib import suppress+from typing import Any, AsyncIterator, Awaitable, Callable, Deque, Dict, List, Mapping, Optional, Protocol, Set++import orjson+from pydantic_core import to_jsonable_python++from core.logging import get_logger+from services.chat.config import hub_setting+from services.chat.events import RUN_SUFFIXES+from services.ws_handler_registry import ws_handler++logger = get_logger(__name__)++#: WebSocket ``type`` of a relay frame.+WIRE_TYPE = "chat_run_publish"+#: Broadcasts a worker may hand the backend to send, and the CloudEvent type+#: (prefix) each must carry. Both are identity only: clients refetch through+#: authorized handlers.+RELAYED_BROADCASTS: Mapping[str, str] = {+ "chat.updated": "com.opencompany.chat.updated",+ "approval_lifecycle": "com.opencompany.approval.",+}+#: Most items one frame may carry.+MAX_ITEMS = 500+_MAX_ID = 256+_RETRY_DELAYS_S = (0.5, 1.0, 2.0, 5.0, 10.0)+++class RelayConnection(Protocol):+ """What the relay needs of a connection to the backend."""++ async def send_str(self, data: str) -> None: ...++ def __aiter__(self) -> AsyncIterator[Any]: ...++ async def close(self) -> None: ...+++Connect = Callable[[], Awaitable[RelayConnection]]+++def _dumps(value: Any) -> str:+ return orjson.dumps(value, default=to_jsonable_python).decode()+++class ChatRelay:+ """Sends this process's chat events to the backend, in order."""++ def __init__(+ self,+ connect: Connect,+ *,+ queue_size: int,+ batch_size: int,+ retry_delays: tuple = _RETRY_DELAYS_S,+ ) -> None:+ self._connect = connect+ self._queue_size = max(1, int(queue_size))+ self._batch_size = max(1, min(int(batch_size), MAX_ITEMS - 1))+ self._retry_delays = retry_delays+ self._queue: Deque[Dict[str, Any]] = deque()+ #: Sessions that lost an event since their last resync went out.+ self._lost: Set[str] = set()+ self._loop: Optional[asyncio.AbstractEventLoop] = None+ self._wake: Optional[asyncio.Event] = None+ self._task: Optional[asyncio.Task] = None+ self._closing = False+ self._overflowing = False+ #: Events dropped so far (a full queue, or a frame the connection lost).+ self.dropped = 0++ # ---- producing ---------------------------------------------------++ def start(self) -> None:+ self._loop = asyncio.get_running_loop()+ self._wake = asyncio.Event()+ self._task = self._loop.create_task(self._run(), name="chat-relay")++ def offer_run_event(+ self,+ *,+ run_id: str,+ session_id: str,+ workflow_id: Optional[str],+ suffix: str,+ fields: Optional[Mapping[str, Any]] = None,+ event_key: Optional[str] = None,+ ) -> None:+ self._offer(+ {+ "kind": "run_event",+ "run_id": run_id,+ "session_id": session_id,+ "workflow_id": workflow_id,+ "suffix": suffix,+ "fields": dict(fields or {}),+ "event_key": event_key,+ }+ )++ def offer_broadcast(self, wire_key: str, data: Mapping[str, Any], *, session_id: Optional[str] = None) -> None:+ self._offer({"kind": "broadcast", "type": wire_key, "data": dict(data), "session_id": session_id})++ def _offer(self, item: Dict[str, Any]) -> None:+ if self._loop is None or self._closing:+ self._drop([item])+ return+ if len(self._queue) >= self._queue_size:+ if not self._overflowing:+ logger.warning("Chat relay queue is full; dropping events until it drains", queue_size=self._queue_size)+ self._overflowing = True+ self._drop([item])+ return+ self._overflowing = False+ self._queue.append(item)+ self._notify()++ def _notify(self) -> None:+ if self._loop is None or self._wake is None:+ return+ try:+ running = asyncio.get_running_loop()+ except RuntimeError:+ running = None+ if running is self._loop:+ self._wake.set()+ else:+ self._loop.call_soon_threadsafe(self._wake.set)++ def _drop(self, items: List[Dict[str, Any]]) -> None:+ for item in items:+ if item.get("kind") == "resync":+ self._lost.update(item.get("session_ids") or ())+ continue+ self.dropped += 1+ session_id = item.get("session_id")+ if isinstance(session_id, str) and session_id:+ self._lost.add(session_id)++ # ---- sending -----------------------------------------------------++ async def _run(self) -> None:+ attempt = 0+ while True:+ if self._closing and not self._queue:+ return+ try:+ connection = await self._connect()+ except asyncio.CancelledError:+ raise+ except Exception as error: # noqa: BLE001 - the backend may be restarting+ if self._closing:+ self._drop(list(self._queue))+ self._queue.clear()+ return+ if attempt == 0 or attempt % 10 == 9:+ logger.warning("Chat relay cannot reach the backend", error=str(error), attempt=attempt + 1)+ await asyncio.sleep(self._retry_delays[min(attempt, len(self._retry_delays) - 1)])+ attempt += 1+ continue+ if attempt:+ logger.info("Chat relay reached the backend", attempts=attempt + 1)+ attempt = 0+ reader = asyncio.create_task(self._drain(connection))+ try:+ await self._pump(connection, reader)+ return+ except asyncio.CancelledError:+ raise+ except Exception as error: # noqa: BLE001 - reconnect and carry on+ logger.warning("Chat relay connection lost", error=str(error))+ finally:+ reader.cancel()+ with suppress(BaseException):+ await reader+ with suppress(Exception):+ await connection.close()++ async def _pump(self, connection: RelayConnection, reader: asyncio.Task) -> None:+ assert self._wake is not None+ while True:+ if not self._queue and not self._lost:+ if self._closing:+ return+ self._wake.clear()+ if self._queue or self._closing:+ continue+ waiter = asyncio.ensure_future(self._wake.wait())+ done, _ = await asyncio.wait({waiter, reader}, return_when=asyncio.FIRST_COMPLETED)+ if reader in done:+ waiter.cancel()+ raise ConnectionError("the backend closed the relay connection")+ continue+ batch = [self._queue.popleft() for _ in range(min(self._batch_size, len(self._queue)))]+ resynced = sorted(self._lost)+ if resynced:+ batch.append({"kind": "resync", "session_ids": resynced})+ self._lost.difference_update(resynced)+ try:+ await connection.send_str(_dumps({"type": WIRE_TYPE, "items": batch}))+ except Exception:+ self._drop(batch)+ raise++ @staticmethod+ async def _drain(connection: RelayConnection) -> None:+ """Read what the backend sends back (its answers, pings) so the+ connection stays healthy; returns when it closes."""+ async for _message in connection:+ pass++ async def close(self, timeout: float = 3.0) -> None:+ """Send what is queued (for at most ``timeout`` seconds), then stop."""+ self._closing = True+ self._notify()+ task, self._task = self._task, None+ if task is None:+ return+ try:+ await asyncio.wait_for(asyncio.shield(task), timeout)+ except (asyncio.TimeoutError, Exception): # noqa: BLE001 - stopping regardless+ task.cancel()+ with suppress(BaseException):+ await task+ if self._queue:+ self._drop(list(self._queue))+ self._queue.clear()+++# ---- the standalone worker's relay ----------------------------------------+++class _AiohttpConnection:+ def __init__(self, session: Any, websocket: Any) -> None:+ self._session = session+ self._websocket = websocket++ async def send_str(self, data: str) -> None:+ await self._websocket.send_str(data)++ def __aiter__(self) -> AsyncIterator[Any]:+ return self._websocket.__aiter__()++ async def close(self) -> None:+ try:+ await self._websocket.close()+ finally:+ await self._session.close()+++def backend_connector(url: Optional[str] = None) -> Connect:+ """Opens the worker's own ``/ws/internal`` connection, with the worker+ token (``services/authz``)."""++ async def connect() -> RelayConnection:+ import aiohttp++ from core.config import Settings+ from services.authz import internal_socket_headers+ from services.temporal.ws_client import internal_ws_url++ session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=None, connect=10))+ try:+ websocket = await session.ws_connect(+ url or internal_ws_url(),+ headers=internal_socket_headers(Settings().secret_key),+ heartbeat=20,+ )+ except BaseException:+ await session.close()+ raise+ return _AiohttpConnection(session, websocket)++ return connect+++_active: Optional[ChatRelay] = None+++def active_relay() -> Optional[ChatRelay]:+ """The relay this process sends its chat events through, or None when+ they go straight into this process's hub."""+ return _active+++def start_relay(connect: Optional[Connect] = None) -> ChatRelay:+ """Send this process's chat events to the backend from now on. For a+ standalone worker; call from inside its event loop."""+ global _active+ if _active is not None:+ return _active+ relay = ChatRelay(+ connect or backend_connector(),+ queue_size=hub_setting("relay_queue_size"),+ batch_size=hub_setting("relay_batch_size"),+ )+ relay.start()+ _active = relay+ logger.info("Chat events go to the backend through the relay")+ return relay+++async def stop_relay() -> None:+ """Send what is queued, then publish locally again."""+ global _active+ relay, _active = _active, None+ if relay is not None:+ await relay.close()+ if relay.dropped:+ logger.warning("Chat relay dropped events", dropped=relay.dropped)+++# ---- the backend's side -----------------------------------------------------+++def _from_worker(websocket: Any) -> bool:+ scope = getattr(websocket, "scope", None)+ return isinstance(scope, dict) and scope.get("path") == "/ws/internal"+++def _identifier(value: Any) -> bool:+ return isinstance(value, str) and 0 < len(value) <= _MAX_ID+++def _run_event(item: Mapping[str, Any]) -> Optional[Dict[str, Any]]:+ workflow_id = item.get("workflow_id")+ fields = item.get("fields")+ event_key = item.get("event_key")+ if not (_identifier(item.get("run_id")) and _identifier(item.get("session_id"))):+ return None+ if item.get("suffix") not in RUN_SUFFIXES:+ return None+ if workflow_id is not None and not _identifier(workflow_id):+ return None+ if fields is not None and not isinstance(fields, dict):+ return None+ if event_key is not None and not isinstance(event_key, str):+ return None+ return {+ "run_id": item["run_id"],+ "session_id": item["session_id"],+ "workflow_id": workflow_id,+ "suffix": item["suffix"],+ "fields": fields or {},+ "event_key": event_key,+ }+++def _broadcast(item: Mapping[str, Any]) -> Optional[Dict[str, Any]]:+ wire_key = item.get("type")+ data = item.get("data")+ expected = RELAYED_BROADCASTS.get(wire_key) if isinstance(wire_key, str) else None+ if expected is None or not isinstance(data, dict):+ return None+ event_type = data.get("type")+ if not isinstance(event_type, str) or not event_type.startswith(expected):+ return None+ return {"type": wire_key, "data": data}+++@ws_handler()+async def handle_chat_run_publish(data: Dict[str, Any], websocket: Any) -> Dict[str, Any]:+ """Publish a standalone worker's chat events here. ``/ws/internal`` only."""+ if not _from_worker(websocket):+ logger.warning("chat_run_publish refused on a client socket")+ return {"success": False, "error": "access_denied"}+ items = data.get("items")+ if not isinstance(items, list) or len(items) > MAX_ITEMS:+ return {"success": False, "error": "invalid_items"}++ from services.chat.hub import get_chat_hub+ from services.status_broadcaster import get_status_broadcaster++ hub = get_chat_hub()+ published = refused = 0+ for item in items:+ kind = item.get("kind") if isinstance(item, dict) else None+ try:+ if kind == "run_event" and (event := _run_event(item)) is not None:+ hub.publish(**event)+ elif kind == "broadcast" and (message := _broadcast(item)) is not None:+ await get_status_broadcaster().broadcast(message)+ elif kind == "resync" and isinstance(item.get("session_ids"), list):+ for session_id in item["session_ids"]:+ if _identifier(session_id):+ hub.resync(session_id)+ else:+ refused += 1+ continue+ except Exception: # noqa: BLE001 - one bad item never loses the rest+ logger.warning("A relayed chat event could not be published", kind=kind, exc_info=True)+ refused += 1+ continue+ published += 1+ if refused:+ logger.warning("chat_run_publish refused items", refused=refused, published=published)+ return {"success": True, "published": published, "refused": refused}+++WS_HANDLERS = {WIRE_TYPE: handle_chat_run_publish}++__all__ = [+ "MAX_ITEMS",+ "RELAYED_BROADCASTS",+ "WIRE_TYPE",+ "WS_HANDLERS",+ "ChatRelay",+ "active_relay",+ "backend_connector",+ "handle_chat_run_publish",+ "start_relay",+ "stop_relay",+]diff --git a/server/services/chat/steps.py b/server/services/chat/steps.pyindex 6933cc73..ed22516d 100644--- a/server/services/chat/steps.py+++ b/server/services/chat/steps.py@@ -72,9 +72,9 @@ def _stream_of(context: Mapping[str, Any]) -> Optional[Mapping[str, Any]]:def _publish(step: Step, suffix: str, fields: Dict[str, Any], event_key: str) -> None:try:- from services.chat.hub import get_chat_hub+ from services.chat.hub import publish_run_event- get_chat_hub().publish(+ publish_run_event(run_id=step.run_id,session_id=step.session_id,workflow_id=step.workflow_id,diff --git a/server/services/chat/stream.py b/server/services/chat/stream.pyindex 652ca543..b8f9f4ee 100644--- a/server/services/chat/stream.py+++ b/server/services/chat/stream.py@@ -42,9 +42,9 @@ Publish = Callable[..., Any]def _default_publish(**kwargs: Any) -> Any:- from services.chat.hub import get_chat_hub+ from services.chat.hub import publish_run_event- return get_chat_hub().publish(**kwargs)+ return publish_run_event(**kwargs)def segment_id(run_id: str, iteration: int, attempt: int) -> str:diff --git a/server/services/chat_thread.py b/server/services/chat_thread.pyindex 3253a9ad..5e735325 100644--- a/server/services/chat_thread.py+++ b/server/services/chat_thread.py@@ -109,11 +109,18 @@ def chat_updated(*, session_id: str, role: Optional[str]) -> WorkflowEvent:async def announce_chat_updated(session_id: str, role: Optional[str]) -> None:"""Send ``chat.updated`` for a session: a message was added (``role``)or the thread was cleared (``role`` None). Never raises."""+ from services.chat.relay import active_relayfrom services.status_broadcaster import get_status_broadcasterevent = chat_updated(session_id=session_id, role=role)+ data = event.model_dump(mode="json", exclude_none=True)+ relay = active_relay()+ if relay is not None:+ # A standalone worker: the backend sends it (services/chat/relay.py).+ relay.offer_broadcast(WIRE_KEY, data, session_id=session_id)+ returntry:- await get_status_broadcaster().broadcast({"type": WIRE_KEY, "data": event.model_dump(mode="json", exclude_none=True)})+ await get_status_broadcaster().broadcast({"type": WIRE_KEY, "data": data})except Exception:logger.warning("chat.updated broadcast failed", session_id=session_id, exc_info=True)diff --git a/server/services/temporal/worker.py b/server/services/temporal/worker.pyindex 151bb2b0..a6b31fe8 100644--- a/server/services/temporal/worker.py+++ b/server/services/temporal/worker.py@@ -762,6 +762,11 @@ async def run_standalone_worker(]registered_plugin_activities = collect_plugin_activities()+ # No socket follows a chat here: its events go to the backend's hub.+ from services.chat.relay import start_relay, stop_relay++ start_relay()+try:worker = Worker(client,@@ -795,6 +800,7 @@ async def run_standalone_worker(await worker.run()finally:+ await stop_relay()# Cleanup session on shutdownif not session.closed:await session.close()diff --git a/server/services/temporal/ws_client.py b/server/services/temporal/ws_client.pyindex d49f4b5e..69c0218d 100644--- a/server/services/temporal/ws_client.py+++ b/server/services/temporal/ws_client.py@@ -23,7 +23,7 @@ from core.config import Settingslogger = get_logger(__name__)-def _default_ws_url() -> str:+def internal_ws_url() -> str:"""Resolve the activity-side WS URL from current ``Settings``.Deferred to first call so module import doesn't require the full env@@ -46,7 +46,7 @@ class WSConnectionPool:"""def __init__(self, url: Optional[str] = None, pool_size: int = 100):- self.url = url if url is not None else _default_ws_url()+ self.url = url if url is not None else internal_ws_url()self.pool_size = pool_sizeself._session: Optional[aiohttp.ClientSession] = Noneself._lock = asyncio.Lock()diff --git a/server/tests/services/chat/test_relay.py b/server/tests/services/chat/test_relay.pynew file mode 100644index 00000000..8a1211cb--- /dev/null+++ b/server/tests/services/chat/test_relay.py@@ -0,0 +1,377 @@+"""A standalone worker's chat events reach the backend (services/chat/relay.py).++On the worker: events leave in the order they were made, a full queue or a+lost frame costs only those events and ends in a resync for their sessions,+and close() sends what is queued. On the backend: ``chat_run_publish`` takes+the worker socket alone, publishes run events into the hub (which numbers and+de-duplicates them), sends only the two allowed broadcasts, and refuses+anything malformed without losing the rest. Publishers route through the+relay only while one is active.+"""++from __future__ import annotations++import asyncio+import inspect+import json+from types import SimpleNamespace+from typing import Any, Dict, List, Optional++import pytest++import services.chat.relay as relay_module+from services.chat.hub import publish_run_event+from services.chat.relay import ChatRelay, handle_chat_run_publish+from tests.services.chat._helpers import FakeSocket+++async def settle(times: int = 10) -> None:+ for _ in range(times):+ await asyncio.sleep(0)+++async def until(condition, tries: int = 200) -> None:+ for _ in range(tries):+ if condition():+ return+ await asyncio.sleep(0)+ raise AssertionError("the relay never got there")+++class FakeConnection:+ """Records the frames the relay sends; ``fail`` makes sends raise."""++ def __init__(self, *, fail: bool = False) -> None:+ self.frames: List[Dict[str, Any]] = []+ self.fail = fail+ self._closed = asyncio.Event()++ async def send_str(self, data: str) -> None:+ if self.fail:+ raise ConnectionError("the connection dropped")+ self.frames.append(json.loads(data))++ def __aiter__(self):+ async def messages():+ await self._closed.wait()+ if False: # pragma: no cover - makes this an async generator+ yield None++ return messages()++ async def close(self) -> None:+ self._closed.set()++ def items(self) -> List[Dict[str, Any]]:+ return [item for frame in self.frames for item in frame["items"]]+++def connector(*connections: FakeConnection, gate: Optional[asyncio.Event] = None):+ pending = list(connections)++ async def connect():+ if gate is not None:+ await gate.wait()+ return pending.pop(0)++ return connect+++def offer(relay: ChatRelay, suffix: str, *, session_id: str = "wf", run_id: str = "r_1", **fields: Any) -> None:+ event_key = fields.pop("event_key", None)+ relay.offer_run_event(+ run_id=run_id,+ session_id=session_id,+ workflow_id=session_id,+ suffix=suffix,+ fields=fields,+ event_key=event_key,+ )+++# ---- the worker's side ------------------------------------------------------+++async def test_events_leave_in_order():+ connection = FakeConnection()+ relay = ChatRelay(connector(connection), queue_size=16, batch_size=2)+ relay.start()+ offer(relay, "started", kind="message")+ offer(relay, "text.content", message_id="m", delta="Hel")+ offer(relay, "text.content", message_id="m", delta="lo")+ relay.offer_broadcast("chat.updated", {"type": "com.opencompany.chat.updated"}, session_id="wf")+ await settle()+ await relay.close()+ items = connection.items()+ assert [item.get("suffix") or item["kind"] for item in items] == ["started", "text.content", "text.content", "broadcast"]+ assert [item["fields"].get("delta") for item in items[1:3]] == ["Hel", "lo"]+ assert all(len(frame["items"]) <= 2 and frame["type"] == "chat_run_publish" for frame in connection.frames)+ assert relay.dropped == 0+++async def test_a_full_queue_drops_and_then_resyncs_the_session():+ connection = FakeConnection()+ gate = asyncio.Event()+ relay = ChatRelay(connector(connection, gate=gate), queue_size=2, batch_size=10)+ relay.start()+ offer(relay, "started")+ offer(relay, "step.started", step_id="s1", step_name="Checking")+ offer(relay, "step.finished", step_id="s1", state="done")+ assert relay.dropped == 1+ gate.set()+ await settle()+ await relay.close()+ items = connection.items()+ assert [item.get("suffix") for item in items[:2]] == ["started", "step.started"]+ # The hole comes last: the subscribers re-snapshot after what did arrive.+ assert items[-1] == {"kind": "resync", "session_ids": ["wf"]}+++async def test_a_lost_frame_is_not_resent_but_resyncs_after_reconnecting():+ broken, healthy = FakeConnection(fail=True), FakeConnection()+ relay = ChatRelay(connector(broken, healthy), queue_size=16, batch_size=10, retry_delays=(0,))+ relay.start()+ offer(relay, "started", session_id="wf")+ # The resync goes out as soon as a connection is back.+ await until(lambda: healthy.frames)+ assert relay.dropped == 1+ offer(relay, "step.started", session_id="wf", step_id="s1", step_name="Checking")+ await relay.close()+ assert healthy.items() == [+ {"kind": "resync", "session_ids": ["wf"]},+ {+ "kind": "run_event",+ "run_id": "r_1",+ "session_id": "wf",+ "workflow_id": "wf",+ "suffix": "step.started",+ "fields": {"step_id": "s1", "step_name": "Checking"},+ "event_key": None,+ },+ ]+++async def test_close_sends_what_is_queued():+ connection = FakeConnection()+ relay = ChatRelay(connector(connection), queue_size=16, batch_size=10)+ relay.start()+ offer(relay, "started")+ offer(relay, "finished", outcome={"type": "success"})+ await relay.close()+ assert [item["suffix"] for item in connection.items()] == ["started", "finished"]+++async def test_a_backend_that_never_answers_costs_only_the_queued_events():+ async def unreachable():+ raise OSError("connection refused")++ relay = ChatRelay(unreachable, queue_size=16, batch_size=10, retry_delays=(0,))+ relay.start()+ offer(relay, "started")+ await settle()+ await relay.close(timeout=0.5)+ assert relay.dropped == 1+++# ---- routing ------------------------------------------------------------------+++class RecordingRelay:+ def __init__(self) -> None:+ self.run_events: List[Dict[str, Any]] = []+ self.broadcasts: List[tuple] = []++ def offer_run_event(self, **event: Any) -> None:+ self.run_events.append(event)++ def offer_broadcast(self, wire_key: str, data: Dict[str, Any], *, session_id: Optional[str] = None) -> None:+ self.broadcasts.append((wire_key, data, session_id))+++async def test_run_events_go_to_the_local_hub_unless_a_relay_is_active(monkeypatch, hub):+ socket = FakeSocket()+ hub.subscribe(socket, "wf")+ assert publish_run_event(run_id="r_1", session_id="wf", workflow_id="wf", suffix="started") is not None+ recording = RecordingRelay()+ monkeypatch.setattr(relay_module, "_active", recording)+ assert publish_run_event(run_id="r_1", session_id="wf", workflow_id="wf", suffix="step.started", fields={"step_id": "s"}) is None+ await settle()+ assert [event["data"]["seq"] for event in socket.events()] == [1]+ assert recording.run_events == [+ {"run_id": "r_1", "session_id": "wf", "workflow_id": "wf", "suffix": "step.started", "fields": {"step_id": "s"}, "event_key": None}+ ]+++async def test_chat_and_approval_broadcasts_go_through_an_active_relay(monkeypatch, frames):+ from services.approvals.events import broadcast_approval_change+ from services.approvals.listeners import ApprovalChange+ from services.chat_thread import announce_chat_updated++ recording = RecordingRelay()+ monkeypatch.setattr(relay_module, "_active", recording)+ await announce_chat_updated("wf", "assistant")+ await broadcast_approval_change(ApprovalChange(stage="sent", approval_id="a_1", workflow_id="wf", status="sent", revision=3))+ assert frames == []+ (chat_key, chat_data, chat_session), (approval_key, approval_data, approval_session) = recording.broadcasts+ assert (chat_key, chat_data["type"], chat_data["data"]["session_id"], chat_session) == (+ "chat.updated",+ "com.opencompany.chat.updated",+ "wf",+ "wf",+ )+ assert (approval_key, approval_data["type"], approval_session) == ("approval_lifecycle", "com.opencompany.approval.sent", "wf")+++def test_only_the_standalone_worker_starts_the_relay():+ from pathlib import Path++ from services.temporal.worker import run_standalone_worker++ source = inspect.getsource(run_standalone_worker)+ assert "start_relay()" in source and "await stop_relay()" in source+ main = (Path(__file__).resolve().parents[3] / "main.py").read_text(encoding="utf-8")+ assert "start_relay" not in main+++# ---- the backend's side -------------------------------------------------------+++def worker_socket() -> FakeSocket:+ return FakeSocket(path="/ws/internal")+++def run_event(suffix: str, **extra: Any) -> Dict[str, Any]:+ item = {"kind": "run_event", "run_id": "r_1", "session_id": "wf", "workflow_id": "wf", "suffix": suffix, "fields": {}, "event_key": None}+ item.update(extra)+ return item+++async def test_a_client_socket_is_refused(hub):+ subscriber = FakeSocket()+ hub.subscribe(subscriber, "wf")+ reply = await handle_chat_run_publish({"type": "chat_run_publish", "items": [run_event("started")]}, FakeSocket())+ await settle()+ assert reply == {"success": False, "error": "access_denied"}+ assert subscriber.sent == []+++async def test_the_worker_socket_publishes_into_the_hub(hub):+ subscriber = FakeSocket()+ hub.subscribe(subscriber, "wf")+ reply = await handle_chat_run_publish(+ {+ "items": [+ run_event("started", fields={"kind": "message"}, event_key="started"),+ run_event("started", fields={"kind": "message"}, event_key="started"),+ run_event("step.started", fields={"step_id": "s1", "step_name": "Checking"}),+ ]+ },+ worker_socket(),+ )+ await settle()+ assert reply == {"success": True, "published": 3, "refused": 0}+ events = subscriber.events()+ # The hub numbers them and drops the repeated key, as for its own.+ assert [(event["type"], event["data"]["seq"]) for event in events] == [+ ("com.opencompany.chat.run.started", 1),+ ("com.opencompany.chat.run.step.started", 2),+ ]+ assert events[0]["data"]["hub_epoch"] == hub.epoch+++async def test_malformed_items_are_refused_without_losing_the_rest(hub, frames):+ subscriber = FakeSocket()+ hub.subscribe(subscriber, "wf")+ reply = await handle_chat_run_publish(+ {+ "items": [+ run_event("not.a.suffix"),+ run_event("started", run_id=""),+ run_event("started", fields=["not", "a", "dict"]),+ {"kind": "broadcast", "type": "node_status", "data": {"type": "com.opencompany.chat.updated"}},+ {"kind": "broadcast", "type": "chat.updated", "data": {"type": "com.opencompany.approval.sent"}},+ {"kind": "mystery"},+ "not an item",+ run_event("started"),+ ]+ },+ worker_socket(),+ )+ await settle()+ assert reply == {"success": True, "published": 1, "refused": 7}+ assert [event["data"]["seq"] for event in subscriber.events()] == [1]+ assert frames == []+++async def test_too_many_items_are_refused(hub):+ reply = await handle_chat_run_publish({"items": [run_event("started")] * (relay_module.MAX_ITEMS + 1)}, worker_socket())+ assert reply == {"success": False, "error": "invalid_items"}+++async def test_the_two_relayed_broadcasts_reach_every_socket(hub, frames):+ chat = {"specversion": "1.0", "type": "com.opencompany.chat.updated", "data": {"session_id": "wf"}}+ approval = {"specversion": "1.0", "type": "com.opencompany.approval.sent", "data": {"approval_id": "a_1"}}+ reply = await handle_chat_run_publish(+ {+ "items": [+ {"kind": "broadcast", "type": "chat.updated", "data": chat},+ {"kind": "broadcast", "type": "approval_lifecycle", "data": approval},+ ]+ },+ worker_socket(),+ )+ assert reply["published"] == 2+ assert frames == [{"type": "chat.updated", "data": chat}, {"type": "approval_lifecycle", "data": approval}]+++async def test_a_resync_tells_the_sessions_subscribers(hub):+ subscriber, other = FakeSocket(), FakeSocket()+ hub.subscribe(subscriber, "wf")+ hub.subscribe(other, "elsewhere")+ await handle_chat_run_publish({"items": [{"kind": "resync", "session_ids": ["wf"]}]}, worker_socket())+ await settle()+ (event,) = subscriber.events()+ assert event["data"]["name"] == "opencompany.resync"+ assert other.sent == []+++# ---- end to end ---------------------------------------------------------------+++class Loopback(FakeConnection):+ """Hands each frame to the backend's handler, as /ws/internal would."""++ def __init__(self) -> None:+ super().__init__()+ self.replies: List[Dict[str, Any]] = []+ self._socket = worker_socket()++ async def send_str(self, data: str) -> None:+ self.replies.append(await handle_chat_run_publish(json.loads(data), self._socket))+++async def test_a_worker_run_reaches_a_backend_subscriber_in_order(hub):+ subscriber = FakeSocket()+ hub.subscribe(subscriber, "wf")+ loopback = Loopback()+ relay = ChatRelay(connector(loopback), queue_size=64, batch_size=3)+ relay.start()+ offer(relay, "started", kind="message")+ offer(relay, "text.started", message_id="r_1.0.0")+ for index, delta in enumerate(["Saturday ", "works", "."]):+ offer(relay, "text.content", message_id="r_1.0.0", delta=delta, event_key=f"text:{index}")+ offer(relay, "text.ended", message_id="r_1.0.0", final=True)+ offer(relay, "finished", outcome={"type": "success"}, result={"no_reply": False})+ await relay.close()+ await settle()+ events = subscriber.events()+ assert [event["data"]["seq"] for event in events] == [1, 2, 3, 4, 5, 6, 7]+ assert "".join(event["data"].get("delta", "") for event in events) == "Saturday works."+ assert events[-1]["type"] == "com.opencompany.chat.run.finished"+ assert all(reply["success"] and reply["refused"] == 0 for reply in loopback.replies)++[email protected](autouse=True)+def _no_relay_left_behind(monkeypatch):+ monkeypatch.setattr(relay_module, "_active", None)+ yielddiff --git a/server/tests/test_internal_socket_surface.py b/server/tests/test_internal_socket_surface.pyindex eb687eb0..b6f8553f 100644--- a/server/tests/test_internal_socket_surface.py+++ b/server/tests/test_internal_socket_surface.py@@ -235,6 +235,10 @@ def test_allowlist_is_exactly_what_the_worker_needs():assert INTERNAL_SOCKET_HANDLERS == {"execute_node","execute_ai_node",+ # A standalone worker's chat events (services/chat/relay.py): it+ # publishes into the hub, reads nothing, writes no row, and refuses+ # every other socket (tests/services/chat/test_relay.py).+ "chat_run_publish","ping",}, "widening this set grants unauthenticated access — justify it in review"