Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
refactor(plugin-extraction): J -- 4 hardcoded refreshers move into plugin folders
trohitgde32b5a269bc
diff --git a/server/nodes/android/__init__.py b/server/nodes/android/__init__.pyindex 5b6793ae..a2067e4e 100644--- a/server/nodes/android/__init__.py+++ b/server/nodes/android/__init__.py@@ -9,10 +9,13 @@ wifi_automation, ...) live alongside as siblings.Wiring is body-free; both registrations are idempotent."""+from services.status_broadcaster import register_service_refreshfrom services.ws_handler_registry import register_router, register_ws_handlersfrom . import _routerfrom ._handlers import WS_HANDLERS+from ._refresh import refresh_android_statusregister_ws_handlers(WS_HANDLERS)register_router(_router.router, name="android")+register_service_refresh(refresh_android_status)diff --git a/server/nodes/android/_refresh.py b/server/nodes/android/_refresh.pynew file mode 100644index 00000000..b406cfae--- /dev/null+++ b/server/nodes/android/_refresh.py@@ -0,0 +1,127 @@+"""Android service-status refresh callback (Wave 11.I, milestone J).++Moved from ``services/status_broadcaster._auto_reconnect_android_relay``+(plus the ``_auto_reconnect_android_relay_body`` helper). Plugin+packages register their own callback via+``status_broadcaster.register_service_refresh``; the broadcaster no+longer hardcodes a per-service refresh.++Load-bearing: this is the path that auto-reconnects the Android relay+from a stored pairing session after server restart. Without it the+relay sits idle until the user manually clicks Connect. Runs once on+lifespan startup (post-V).+"""++from __future__ import annotations++import logging+from typing import TYPE_CHECKING++from opentelemetry import trace++if TYPE_CHECKING:+ from services.status_broadcaster import StatusBroadcaster++logger = logging.getLogger(__name__)+tracer = trace.get_tracer(__name__)+++async def refresh_android_status(broadcaster: "StatusBroadcaster") -> None:+ """Auto-reconnect to the Android relay if a stored pairing exists.++ Called once per ``_refresh_all_services`` cycle (post-V: at+ lifespan startup only).+ """+ with tracer.start_as_current_span("broadcaster.refresh_android") as span:+ await _auto_reconnect_body(broadcaster, span)+++async def _auto_reconnect_body(broadcaster: "StatusBroadcaster", span) -> None:+ try:+ # Already connected? Refresh the cached snapshot and stop.+ from ._relay.manager import get_current_relay_client++ existing = get_current_relay_client()+ if existing and existing.is_connected():+ broadcaster._status["android"] = {+ "connected": True,+ "paired": existing.is_paired(),+ "device_id": existing.paired_device_id,+ "device_name": existing.paired_device_name,+ "connected_devices": list(existing.get_connected_devices()),+ "connection_type": "relay",+ "qr_data": existing.qr_data,+ "session_token": existing.session_token,+ }+ logger.debug("[StatusBroadcaster] Android relay already connected")+ await broadcaster.broadcast({+ "type": "android_status",+ "data": broadcaster._status["android"],+ })+ span.set_attribute("path", "already_connected")+ return++ # Look for a stored pairing session.+ from core.container import container++ database = container.database()+ session = await database.get_android_relay_session()+ if not session:+ span.set_attribute("path", "no_session")+ logger.debug("[StatusBroadcaster] No stored Android relay session")+ return++ relay_url = session.get("relay_url")+ api_key = session.get("api_key")+ device_id = session.get("device_id")++ if not relay_url or not api_key:+ span.set_attribute("path", "session_missing_creds")+ logger.debug(+ "[StatusBroadcaster] Stored session missing relay URL or API key"+ )+ return++ span.set_attribute("path", "auto_reconnect")+ logger.info(+ "[StatusBroadcaster] Auto-reconnecting to Android relay...",+ relay_url=relay_url,+ device_id=device_id,+ )++ from ._relay.manager import get_relay_client++ client, error = await get_relay_client(relay_url, api_key)++ if client and client.is_connected():+ logger.info("[StatusBroadcaster] Android relay reconnected successfully")+ # The relay server creates a new session on each connect, so+ # pairing may be lost -- mirror whatever the new client+ # reports.+ broadcaster._status["android"] = {+ "connected": True,+ "paired": client.is_paired(),+ "device_id": client.paired_device_id,+ "device_name": client.paired_device_name,+ "connected_devices": list(client.get_connected_devices()),+ "connection_type": "relay",+ "qr_data": client.qr_data,+ "session_token": client.session_token,+ }+ await broadcaster.broadcast({+ "type": "android_status",+ "data": broadcaster._status["android"],+ })+ span.set_attribute("reconnect_ok", True)+ else:+ span.set_attribute("reconnect_ok", False)+ logger.warning(+ "[StatusBroadcaster] Failed to reconnect Android relay: %s", error+ )+ # Stored session is stale; drop it.+ await database.clear_android_relay_session()+ except Exception as exc: # noqa: BLE001 -- mirror pre-migration behaviour+ span.record_exception(exc)+ logger.debug(+ "[StatusBroadcaster] Could not auto-reconnect Android relay: %s", exc+ )diff --git a/server/nodes/google/__init__.py b/server/nodes/google/__init__.pyindex b7fd63f9..7a67bcf8 100644--- a/server/nodes/google/__init__.py+++ b/server/nodes/google/__init__.py@@ -24,10 +24,13 @@ the FastAPI app pick up the plugin's surface without ever importingthis module by name."""+from services.status_broadcaster import register_service_refreshfrom services.ws_handler_registry import register_router, register_ws_handlersfrom . import _routerfrom ._handlers import WS_HANDLERS+from ._refresh import refresh_google_statusregister_ws_handlers(WS_HANDLERS)register_router(_router.router, name="google")+register_service_refresh(refresh_google_status)diff --git a/server/nodes/google/_refresh.py b/server/nodes/google/_refresh.pynew file mode 100644index 00000000..132f0d34--- /dev/null+++ b/server/nodes/google/_refresh.py@@ -0,0 +1,66 @@+"""Google Workspace service-status refresh callback (Wave 11.I, milestone J).++Moved from ``services/status_broadcaster._refresh_google_status``.+Plugin packages register their own callback via+``status_broadcaster.register_service_refresh``; the broadcaster no+longer hardcodes a per-service refresh.++Reads OAuth tokens via ``auth_service.get_oauth_tokens("google")`` and+mirrors the result into the broadcaster cache.+"""++from __future__ import annotations++import logging+from typing import TYPE_CHECKING++from opentelemetry import trace++if TYPE_CHECKING:+ from services.status_broadcaster import StatusBroadcaster++logger = logging.getLogger(__name__)+tracer = trace.get_tracer(__name__)+++async def refresh_google_status(broadcaster: "StatusBroadcaster") -> None:+ """Refresh Google cache + broadcast. One pass per+ ``_refresh_all_services`` cycle.+ """+ with tracer.start_as_current_span("broadcaster.refresh_google") as span:+ try:+ from core.container import container++ auth_service = container.auth_service()+ tokens = await auth_service.get_oauth_tokens(+ "google", customer_id="owner"+ )+ if not tokens or not tokens.get("access_token"):+ broadcaster._status["google"] = {+ "connected": False,+ "email": None,+ "name": None,+ }+ else:+ broadcaster._status["google"] = {+ "connected": True,+ "email": tokens.get("email"),+ "name": tokens.get("name"),+ }+ logger.debug(+ "[StatusBroadcaster] Google status: connected as %s",+ tokens.get("email"),+ )++ await broadcaster.broadcast({+ "type": "google_status",+ "data": broadcaster._status["google"],+ })+ span.set_attribute(+ "connected", bool(broadcaster._status["google"]["connected"])+ )+ except Exception as exc: # noqa: BLE001+ span.record_exception(exc)+ logger.debug(+ "[StatusBroadcaster] Could not refresh Google status: %s", exc+ )diff --git a/server/nodes/twitter/__init__.py b/server/nodes/twitter/__init__.pyindex 5e4a0f2d..42ab274d 100644--- a/server/nodes/twitter/__init__.py+++ b/server/nodes/twitter/__init__.py@@ -18,13 +18,18 @@ FastAPI app pick up the plugin's surface without ever importing thismodule by name."""+from services.status_broadcaster import register_service_refreshfrom services.ws_handler_registry import register_router, register_ws_handlersfrom . import _routerfrom ._handlers import WS_HANDLERS+from ._refresh import refresh_twitter_status# WebSocket handlers (3 message types) and the OAuth-callback HTTP# router self-register so ``routers/websocket.py`` and ``main.py``-# stay free of per-plugin imports.+# stay free of per-plugin imports. Plus the service-status refresh+# callback (Wave 11.I, milestone J) -- the broadcaster fans out to+# this on lifespan startup instead of hardcoding the call.register_ws_handlers(WS_HANDLERS)register_router(_router.router, name="twitter")+register_service_refresh(refresh_twitter_status)diff --git a/server/nodes/twitter/_refresh.py b/server/nodes/twitter/_refresh.pynew file mode 100644index 00000000..5aa04ee0--- /dev/null+++ b/server/nodes/twitter/_refresh.py@@ -0,0 +1,78 @@+"""Twitter service-status refresh callback (Wave 11.I, milestone J).++Moved from ``services/status_broadcaster._refresh_twitter_status``.+Plugin packages register their own callback via+``status_broadcaster.register_service_refresh``; the broadcaster no+longer hardcodes a per-service refresh.++Reads OAuth tokens via ``auth_service.get_oauth_tokens("twitter")`` and+mirrors the result into the broadcaster cache. No plugin internals+are reached -- pure auth-service read + broadcast. The function still+runs in the broadcaster's TaskGroup so OTel spans aggregate the same+way they did pre-migration.+"""++from __future__ import annotations++import logging+from typing import TYPE_CHECKING++from opentelemetry import trace++if TYPE_CHECKING:+ from services.status_broadcaster import StatusBroadcaster++logger = logging.getLogger(__name__)+tracer = trace.get_tracer(__name__)+++async def refresh_twitter_status(broadcaster: "StatusBroadcaster") -> None:+ """Refresh Twitter cache + broadcast. One pass per+ ``_refresh_all_services`` cycle.+ """+ with tracer.start_as_current_span("broadcaster.refresh_twitter") as span:+ try:+ from core.container import container++ auth_service = container.auth_service()+ tokens = await auth_service.get_oauth_tokens(+ "twitter", customer_id="owner"+ )+ if not tokens or not tokens.get("access_token"):+ broadcaster._status["twitter"] = {+ "connected": False,+ "username": None,+ "user_id": None,+ "name": None,+ "profile_image_url": None,+ }+ else:+ # User info is stored in the OAuth token record.+ # email field carries the ``@username`` form.+ email = tokens.get("email", "")+ name = tokens.get("name", "")+ username = email.lstrip("@") if email.startswith("@") else email+ broadcaster._status["twitter"] = {+ "connected": True,+ "username": username or None,+ "user_id": None,+ "name": name or None,+ "profile_image_url": None,+ }+ logger.debug(+ "[StatusBroadcaster] Twitter status: connected as @%s",+ username,+ )++ await broadcaster.broadcast({+ "type": "twitter_status",+ "data": broadcaster._status["twitter"],+ })+ span.set_attribute(+ "connected", bool(broadcaster._status["twitter"]["connected"])+ )+ except Exception as exc: # noqa: BLE001 -- mirror pre-migration behaviour+ span.record_exception(exc)+ logger.debug(+ "[StatusBroadcaster] Could not refresh Twitter status: %s", exc+ )diff --git a/server/nodes/whatsapp/__init__.py b/server/nodes/whatsapp/__init__.pyindex ca602d52..5e2c1b6e 100644--- a/server/nodes/whatsapp/__init__.py+++ b/server/nodes/whatsapp/__init__.py@@ -19,9 +19,11 @@ Two self-registrations happen on package import:"""from services._supervisor import register_supervisor+from services.status_broadcaster import register_service_refreshfrom services.ws_handler_registry import register_ws_handlersfrom ._handlers import WS_HANDLERS+from ._refresh import refresh_whatsapp_statusfrom ._runtime import WhatsAppRuntime, get_whatsapp_runtime# Supervisor: ensures shutdown_all_supervisors() reaches us.@@ -33,6 +35,11 @@ register_supervisor(WhatsAppRuntime.get_instance())# typing / presence / diagnostics / stop). Idempotent on re-import.register_ws_handlers(WS_HANDLERS)+# Service-status refresh callback (Wave 11.I, milestone J) -- the+# broadcaster fans out to this on lifespan startup instead of+# hardcoding the call.+register_service_refresh(refresh_whatsapp_status)+__all__ = ["WhatsAppRuntime","get_whatsapp_runtime",diff --git a/server/nodes/whatsapp/_refresh.py b/server/nodes/whatsapp/_refresh.pynew file mode 100644index 00000000..e8cd4cd4--- /dev/null+++ b/server/nodes/whatsapp/_refresh.py@@ -0,0 +1,62 @@+"""WhatsApp service-status refresh callback (Wave 11.I, milestone J).++Moved from ``services/status_broadcaster._refresh_whatsapp_status``.+Plugin packages register their own callback via+``status_broadcaster.register_service_refresh``; the broadcaster no+longer hardcodes a per-service refresh.++Calls the Go-service ``status`` RPC via the plugin's own client and+mirrors the result into the broadcaster cache. Silently swallows+exceptions if the WhatsApp Go service is unavailable -- mirrors+pre-migration behaviour.+"""++from __future__ import annotations++import logging+import time+from typing import TYPE_CHECKING++from opentelemetry import trace++if TYPE_CHECKING:+ from services.status_broadcaster import StatusBroadcaster++logger = logging.getLogger(__name__)+tracer = trace.get_tracer(__name__)+++async def refresh_whatsapp_status(broadcaster: "StatusBroadcaster") -> None:+ """Refresh WhatsApp cache + broadcast. One pass per+ ``_refresh_all_services`` cycle.+ """+ with tracer.start_as_current_span("broadcaster.refresh_whatsapp") as span:+ try:+ from ._service import get_client++ client = await get_client()+ status_data = await client.call("status")++ broadcaster._status["whatsapp"] = {+ "connected": status_data.get("connected", False),+ "has_session": status_data.get("has_session", False),+ "running": status_data.get("running", False),+ "pairing": status_data.get("pairing", False),+ "device_id": status_data.get("device_id"),+ "qr": None,+ "timestamp": time.time(),+ }+ logger.debug(+ "[StatusBroadcaster] Refreshed WhatsApp status: connected=%s",+ status_data.get("connected"),+ )+ await broadcaster.broadcast({+ "type": "whatsapp_status",+ "data": broadcaster._status["whatsapp"],+ })+ span.set_attribute("connected", bool(status_data.get("connected", False)))+ except Exception as exc: # noqa: BLE001+ span.record_exception(exc)+ logger.debug(+ "[StatusBroadcaster] Could not refresh WhatsApp status: %s", exc+ )diff --git a/server/services/status_broadcaster.py b/server/services/status_broadcaster.pyindex 7db5ceed..c1ad7e56 100644--- a/server/services/status_broadcaster.py+++ b/server/services/status_broadcaster.py@@ -127,20 +127,21 @@ class StatusBroadcaster:logger.info(f"[StatusBroadcaster] Client disconnected. Total: {len(self._connections)}")async def _refresh_all_services(self):- """Spawn per-service refresh tasks; each broadcasts as soon as- its own status is ready.+ """Fan out plugin-registered refresh callbacks.Uses ``asyncio.TaskGroup`` (Python 3.11+) for structuredconcurrency: every refresh runs as an independent task andpublishes its own ``<service>_status`` message the moment itscache slot is populated. The slowest service no longer gates- the others — the credentials/status UI hydrates incrementally.+ the others -- the credentials/status UI hydrates incrementally.- Plugin packages register their own refresh callbacks via- :func:`register_service_refresh` (see- ``nodes/<group>/__init__.py``). The hardcoded core refreshes- below predate the registry; they'll move into their own plugin- folders in subsequent waves.+ Wave 11.I, milestone J: every refresher now lives in its+ plugin folder (``nodes/<plugin>/_refresh.py``) and registers+ via :func:`register_service_refresh`. The broadcaster has zero+ per-plugin knowledge in this dispatch loop. Wave 11.I, V: this+ method runs ONCE at FastAPI lifespan startup (no longer per+ WS-client connect). State changes after that flow through the+ originating code path's event-driven broadcasts.Each callback swallows its own exceptions, so TaskGroup neversees one. Kept inside try/except* defensively in case a future@@ -149,11 +150,6 @@ class StatusBroadcaster:with tracer.start_as_current_span("broadcaster.refresh_all_services"):try:async with asyncio.TaskGroup() as tg:- tg.create_task(self._refresh_whatsapp_status())- tg.create_task(self._refresh_twitter_status())- tg.create_task(self._refresh_google_status())- tg.create_task(self._auto_reconnect_android_relay())- # Plugin-registered refreshes (telegram, future ones)for callback in list(_SERVICE_REFRESH_CALLBACKS):tg.create_task(callback(self))except* Exception as eg:@@ -315,218 +311,14 @@ class StatusBroadcaster:})# =========================================================================- # WhatsApp Status Updates+ # Status updates -- per-service refresh bodies live in their plugin+ # folders (Wave 11.I, milestone J): nodes/<plugin>/_refresh.py.+ # The plugin's __init__.py self-registers via+ # services.status_broadcaster.register_service_refresh; the central+ # _refresh_all_services fans out to every registered callback with+ # zero per-plugin knowledge.# =========================================================================- async def _refresh_whatsapp_status(self):- """Fetch fresh WhatsApp status from Go service and update cache.-- Called on client connect to ensure initial_status has accurate data.- Silently fails if WhatsApp service is unavailable. Span emitted- via OTel for cold-start benchmarking.- """- with tracer.start_as_current_span("broadcaster.refresh_whatsapp") as span:- try:- from nodes.whatsapp._service import get_client- import time-- client = await get_client()- status_data = await client.call("status")-- self._status["whatsapp"] = {- "connected": status_data.get("connected", False),- "has_session": status_data.get("has_session", False),- "running": status_data.get("running", False),- "pairing": status_data.get("pairing", False),- "device_id": status_data.get("device_id"),- "qr": None,- "timestamp": time.time()- }- logger.debug(f"[StatusBroadcaster] Refreshed WhatsApp status: connected={status_data.get('connected')}")- await self.broadcast({- "type": "whatsapp_status",- "data": self._status["whatsapp"],- })- span.set_attribute("connected", bool(status_data.get("connected", False)))- except Exception as e:- span.record_exception(e)- # Don't fail client connection if WhatsApp service is down- logger.debug(f"[StatusBroadcaster] Could not refresh WhatsApp status: {e}")-- async def _refresh_twitter_status(self):- """Fetch Twitter status from stored OAuth tokens in database.-- Called on client connect to check if user is authenticated with Twitter.- Uses OAuth token system (matches REST endpoint and node handlers).- Span emitted via OTel for cold-start benchmarking.- """- with tracer.start_as_current_span("broadcaster.refresh_twitter") as span:- try:- from core.container import container- auth_service = container.auth_service()-- # Get tokens from OAuth system (NOT api_key system)- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")- if not tokens or not tokens.get("access_token"):- self._status["twitter"] = {- "connected": False,- "username": None,- "user_id": None,- "name": None,- "profile_image_url": None- }- else:- # User info is stored in the OAuth token record- email = tokens.get("email", "") # Stored as "@username"- name = tokens.get("name", "")- username = email.lstrip("@") if email.startswith("@") else email-- self._status["twitter"] = {- "connected": True,- "username": username or None,- "user_id": None,- "name": name or None,- "profile_image_url": None- }- logger.debug(f"[StatusBroadcaster] Twitter status: connected as @{username}")-- await self.broadcast({- "type": "twitter_status",- "data": self._status["twitter"],- })- span.set_attribute("connected", bool(self._status["twitter"]["connected"]))- except Exception as e:- span.record_exception(e)- logger.debug(f"[StatusBroadcaster] Could not refresh Twitter status: {e}")-- async def _refresh_google_status(self):- """Fetch Google Workspace status from stored OAuth tokens.-- Called on client connect to check if user is authenticated with Google.- Span emitted via OTel for cold-start benchmarking.- """- with tracer.start_as_current_span("broadcaster.refresh_google") as span:- try:- from core.container import container- auth_service = container.auth_service()-- tokens = await auth_service.get_oauth_tokens("google", customer_id="owner")- if not tokens or not tokens.get("access_token"):- self._status["google"] = {- "connected": False,- "email": None,- "name": None,- }- else:- self._status["google"] = {- "connected": True,- "email": tokens.get("email"),- "name": tokens.get("name"),- }- logger.debug(f"[StatusBroadcaster] Google status: connected as {tokens.get('email')}")-- await self.broadcast({- "type": "google_status",- "data": self._status["google"],- })- span.set_attribute("connected", bool(self._status["google"]["connected"]))- except Exception as e:- span.record_exception(e)- logger.debug(f"[StatusBroadcaster] Could not refresh Google status: {e}")-- async def _auto_reconnect_android_relay(self):- """Auto-reconnect to Android relay if there's a stored pairing session.-- Called on client connect to re-establish relay connection after server restart.- The stored session contains relay URL, API key, and paired device info.- Span emitted via OTel for cold-start benchmarking.- """- with tracer.start_as_current_span("broadcaster.refresh_android") as span:- await self._auto_reconnect_android_relay_body(span)-- async def _auto_reconnect_android_relay_body(self, span):- try:- # Check if already connected- from nodes.android._relay.manager import get_current_relay_client- existing = get_current_relay_client()- if existing and existing.is_connected():- # Already connected, just refresh status- self._status["android"] = {- "connected": True,- "paired": existing.is_paired(),- "device_id": existing.paired_device_id,- "device_name": existing.paired_device_name,- "connected_devices": list(existing.get_connected_devices()),- "connection_type": "relay",- "qr_data": existing.qr_data,- "session_token": existing.session_token- }- logger.debug("[StatusBroadcaster] Android relay already connected")- await self.broadcast({- "type": "android_status",- "data": self._status["android"],- })- span.set_attribute("path", "already_connected")- return-- # Check for stored session- from core.container import container- database = container.database()-- session = await database.get_android_relay_session()- if not session:- span.set_attribute("path", "no_session")- logger.debug("[StatusBroadcaster] No stored Android relay session")- return-- relay_url = session.get("relay_url")- api_key = session.get("api_key")- device_id = session.get("device_id")- session.get("device_name")-- if not relay_url or not api_key:- span.set_attribute("path", "session_missing_creds")- logger.debug("[StatusBroadcaster] Stored session missing relay URL or API key")- return-- span.set_attribute("path", "auto_reconnect")- logger.info("[StatusBroadcaster] Auto-reconnecting to Android relay...",- relay_url=relay_url, device_id=device_id)-- # Attempt to reconnect- from nodes.android._relay.manager import get_relay_client- client, error = await get_relay_client(relay_url, api_key)-- if client and client.is_connected():- logger.info("[StatusBroadcaster] Android relay reconnected successfully")- # Update status - connected to relay but need to check if still paired- # The relay server creates a new session on each connect, so pairing is lost- # Update the cached status to reflect the current state- self._status["android"] = {- "connected": True,- "paired": client.is_paired(),- "device_id": client.paired_device_id,- "device_name": client.paired_device_name,- "connected_devices": list(client.get_connected_devices()),- "connection_type": "relay",- "qr_data": client.qr_data,- "session_token": client.session_token- }- await self.broadcast({- "type": "android_status",- "data": self._status["android"],- })- span.set_attribute("reconnect_ok", True)- else:- span.set_attribute("reconnect_ok", False)- logger.warning(f"[StatusBroadcaster] Failed to reconnect Android relay: {error}")- # Clear the stored session since reconnect failed- await database.clear_android_relay_session()-- except Exception as e:- span.record_exception(e)- logger.debug(f"[StatusBroadcaster] Could not auto-reconnect Android relay: {e}")-async def update_whatsapp_status(self,connected: bool,