Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
"""Core controller that coordinates between modules and handlers"""
from __future__ import annotations
import asyncio
import concurrent.futures
import json
import logging
import threading
import uuid
from typing import TYPE_CHECKING, Optional, Dict, Any
from config import paths
from config.platform_registry import get_platform_descriptor
from config.v2_config import (
DEFAULT_AGENT_BACKEND,
DEFAULT_AGENT_IDLE_TIMEOUT_SECONDS,
DEFAULT_AGENT_PROGRESS_STYLE,
)
from modules.im import BaseIMClient, MessageContext, IMFactory
from modules.im.multi import MultiIMClient
from modules.agent_router import AgentRouter
from modules.agents.service import AgentService
from modules.agents.catalog import AGENT_BACKENDS, NATIVE_CLI_BACKENDS
from modules.claude_client import ClaudeClient
from modules.session_manager import SessionManager
from modules.settings_manager import SettingsManager, MultiSettingsManager
from core.handlers import (
CommandHandlers,
SessionHandler,
SettingsHandler,
MessageHandler,
)
from core.agent_auth_service import AgentAuthService
from core.audio_asr import AudioAsrService
from core.citations import CitationBundle
from core.message_context import build_context_session_key
from core.message_dispatcher import ConsolidatedMessageDispatcher
from core.message_output import MessageOutput
from core.processing_indicator import ProcessingIndicatorService
from core.run_settlement import SETTLED_BY_NO_TERMINAL_RESULT
from core.runtime_commands import RuntimeCommandWatcher
from core.runtime_activation import RuntimeActivationRegistry
from core.runtime_ownership import RuntimeOwnershipProvider
from core.runtime_recovery import SessionDeliveryRecoveryHandler
from core.runtime_work import RuntimeWorkLane, RuntimeWorkSupervisor
from core.scheduled_tasks import ScheduledTaskService
from core.show_git import ShowGitCheckpointService
from core.update_checker import UpdateChecker
from core.watches import ManagedWatchService
from core.web_ui_watchdog import watch_web_ui
from core.vibe_agents import VibeAgent, VibeAgentStore
from core.blocking import run_blocking
from vibe.i18n import get_supported_languages, t as i18n_t
from core.loop_watchdog import EventLoopWatchdog
from storage import loop_writers
from vibe.runtime import current_process_owns_service_instance, mark_service_instance_started
if TYPE_CHECKING:
pass
logger = logging.getLogger(__name__)
_RUNTIME_WORK_SHUTDOWN_GRACE_SECONDS = 10.0
_MODEL_HUB_SNAPSHOT_RECONCILE_INTERVAL_SECONDS = 5 * 60
class RemovedPlatformIMClient(BaseIMClient):
"""No-op sink for stale replies after an IM platform is hot-disabled."""
def __init__(self, platform: str):
from config.v2_config import AvibeConfig
from modules.im.formatters.avibe_formatter import AvibeFormatter
super().__init__(AvibeConfig())
self.platform = platform
self.formatter = AvibeFormatter()
def get_default_parse_mode(self) -> Optional[str]:
return None
def should_use_thread_for_reply(self) -> bool:
return False
def supports_message_editing(self, context: Optional[MessageContext] = None) -> bool:
return False
async def send_message(
self,
context: MessageContext,
text: str,
parse_mode: Optional[str] = None,
reply_to: Optional[str] = None,
) -> Optional[str]:
logger.info("Dropping stale outbound message for removed IM platform %s", self.platform)
return None
async def send_message_with_buttons(
self,
context: MessageContext,
text: str,
keyboard,
parse_mode: Optional[str] = None,
) -> Optional[str]:
logger.info("Dropping stale outbound button message for removed IM platform %s", self.platform)
return None
async def edit_message(
self,
context: MessageContext,
message_id: str,
text: Optional[str] = None,
keyboard: Optional[Any] = None,
parse_mode: Optional[str] = None,
) -> bool:
return False
async def remove_inline_keyboard(
self,
context: MessageContext,
message_id: str,
text: Optional[str] = None,
parse_mode: Optional[str] = None,
) -> bool:
return False
async def answer_callback(self, callback_id: str, text: Optional[str] = None, show_alert: bool = False) -> bool:
return False
def register_handlers(self):
return None
def run(self):
return None
def stop(self):
return None
async def get_user_info(self, user_id: str) -> Dict[str, Any]:
return {"id": user_id, "platform": self.platform, "removed": True}
async def get_channel_info(self, channel_id: str) -> Dict[str, Any]:
return {"id": channel_id, "platform": self.platform, "removed": True}
async def add_reaction(self, context: MessageContext, message_id: str, emoji: str) -> bool:
return False
async def remove_reaction(self, context: MessageContext, message_id: str, emoji: str) -> bool:
return False
async def send_typing_indicator(self, context: MessageContext) -> bool:
return False
async def clear_typing_indicator(self, context: MessageContext) -> bool:
return False
async def delete_message(self, context: MessageContext, message_id: str) -> bool:
return False
async def send_dm(self, user_id: str, text: str, **kwargs):
return None
def format_markdown(self, text: str) -> str:
return text
def _optional_target_str(value: Any) -> Optional[str]:
if value is None:
return None
text = str(value).strip()
return text or None
def _target_agent_variant(value: Any, backend: Any = None, agent_name: Any = None) -> Optional[str]:
variant = _optional_target_str(value)
if variant is None:
return None
sentinel_values = {"default", *AGENT_BACKENDS}
backend_text = _optional_target_str(backend)
if backend_text:
sentinel_values.add(backend_text)
agent_name_text = _optional_target_str(agent_name)
if agent_name_text:
sentinel_values.add(agent_name_text)
return None if variant in sentinel_values else variant
def _refresh_status_bubble_config(controller: Any) -> None:
"""Best-effort, mtime-guarded reload so Web UI changes to the status-bubble
settings (progress style + heartbeat/no-output thresholds) take effect for
turns that never pass through an IM inbound handler first (e.g. scheduled /
background agent runs), where nothing else calls ``_refresh_config_from_disk``
before the getters read ``controller.config``.
Implemented as a module-level helper taking the controller explicitly and
resolving the reload via ``getattr`` so lightweight test stubs that invoke the
getters unbound (a bare ``SimpleNamespace`` self) simply skip the refresh
instead of raising ``AttributeError``.
"""
refresh = getattr(controller, "_refresh_config_from_disk", None)
if callable(refresh):
refresh()
# Idle eviction and the runtime generation sweep run on this cadence.
_IDLE_SWEEP_INTERVAL_SECONDS = 60
class Controller:
"""Main controller that coordinates all bot operations"""
def __init__(self, config):
"""Initialize controller with configuration"""
self.config = config
self._config_mtime: Optional[float] = None
self._loop: Optional[asyncio.AbstractEventLoop] = None
self._im_thread: Optional[threading.Thread] = None
self._im_run_exception: Optional[BaseException] = None
self._shutdown_requested = False
self._shutdown_task: asyncio.Task[None] | None = None
self._runtime_work_shutdown_task: asyncio.Task[None] | None = None
self._shutdown_tainted = False
self._service_lock_safe_to_release = False
self.controller_id = uuid.uuid4().hex
self._runtime_work_shutdown_grace_seconds = (
_RUNTIME_WORK_SHUTDOWN_GRACE_SECONDS
)
self.enabled_platforms = list(getattr(config, "enabled_platforms", lambda: [config.platform])())
self.primary_platform = getattr(getattr(config, "platforms", None), "primary", config.platform)
self._reconcile_lock: Optional[asyncio.Lock] = None
self._removed_im_clients: Dict[str, BaseIMClient] = {}
# Session tracking (must be initialized before handlers)
self.claude_sessions: Dict[str, Any] = {}
self.receiver_tasks: Dict[str, asyncio.Task] = {}
self.stored_session_mappings: Dict[str, str] = {}
self.session_last_activity: Dict[str, float] = {}
# Monotonic baseline of when each session's CURRENT turn went active
# (idle→active transition). Unlike ``session_last_activity`` — which is
# bumped on every streamed event — this is NOT touched mid-turn, so the
# Running tab can report an accurate "busy for" duration instead of
# seconds-since-last-chunk.
self.session_turn_started: Dict[str, float] = {}
self.claude_active_sessions: set[str] = set()
# The live streaming turn-sink registry now lives on the turn owner
# (``self.session_turns.active_turn_sinks``); the register/pop/get methods +
# the ``active_turn_sinks`` property below delegate to it.
# Per-session turn gate, published by ``core.internal_server.create_app``
# once the internal server is built on the loop. Persisted Session inputs
# route through it so their source policy can queue, steer, or replace via
# the same durable lifecycle (in_flight + turn.start / turn.end + Stop).
# ``None`` until the server is up; callers then fall back to the direct path.
self.session_turn_gate: Optional[Any] = None
# Per-session turn owner (FSM). Created here so the controller owns it from
# birth — boot stale-reset (below) and the OpenCode poll restore both run
# before the internal server binds. ``core.internal_server.create_app`` later
# binds the routing-context builder + exposes the gate endpoints; the gate,
# dispatcher, and scheduler all share this one owner's in_flight + flush state.
from core.session_turns import SessionTurnManager
self.runtime_activation = RuntimeActivationRegistry()
self.session_turns = SessionTurnManager(self)
# The CLI reaches Computer Use through the controller's internal IPC.
# Keep the driver proxy lazy so an enabled desktop toggle does not
# create a driver process until an active Turn requests it.
self.computer_use_server = None
self.computer_script_sessions = None
self.computer_script_interactions = None
self.computer_js_kernels = None
self.runtime_ownership = RuntimeOwnershipProvider(
self.session_turns._sqlite_engine()
)
self.runtime_work_supervisor = RuntimeWorkSupervisor(
on_lease_lost=lambda: self.request_shutdown("service lease lost")
)
self._runtime_work_tokens = [
self.runtime_work_supervisor.register(
RuntimeWorkLane.SESSION_DELIVERIES,
SessionDeliveryRecoveryHandler(self.session_turns),
)
]
# The internal server publishes the Session gate before waiting on this
# event. Controller startup owns backend restoration, durable owner
# recovery, and supervisor activation, then releases HTTP serving and
# the scheduler/watch services together.
self._delivery_recovery_complete = asyncio.Event()
self._init_model_hub()
from core.mcp_hub.runtime import McpRuntime
# Started lazily on the first MCP request; owns its own thread and loop.
self.mcp_runtime = McpRuntime()
# Initialize core modules
self._init_modules()
# Initialize handlers
self._init_handlers()
# Initialize agents (depends on handlers/session handler)
self._init_agents()
self.agent_auth_service = AgentAuthService(self)
from core.backend_restart import BackendRestartCoordinator
self.backend_restart_coordinator = BackendRestartCoordinator(
self,
self.agent_auth_service._apply_backend_runtime_refresh,
renew=self.agent_auth_service.renew_backend_runtime,
)
if self.model_hub_service is not None:
self.model_hub_service.migration_guard = self.backend_restart_coordinator.migration_guard
self.model_hub_service.migration_reconcile_auth = self.backend_restart_coordinator.reconcile_migration_auth
self.backend_restart_coordinator.restore_migration_blocks()
self.vibe_agent_store = VibeAgentStore()
self.vibe_agent_store.ensure_builtin_default_agents(
self._enabled_agent_backends(),
)
# Setup callbacks
self._setup_callbacks()
# Consolidated message dispatcher
self.message_dispatcher = ConsolidatedMessageDispatcher(self)
self.scheduled_task_service = ScheduledTaskService(self)
self._runtime_work_tokens.extend(
self.scheduled_task_service.register_controller_runtime_work_lanes()
)
self.watch_service = ManagedWatchService(self)
self.runtime_command_watcher = RuntimeCommandWatcher(self)
self.show_git_checkpoint_service = ShowGitCheckpointService()
# Background task for cleanup
self.cleanup_task: Optional[asyncio.Task] = None
self.vibey_model_supply_task: Optional[asyncio.Task] = None
from modules.agents.vibey.recovery import VibeyRecovery
# The single owner of Vibey's settlement outside a Turn.
self.vibey_recovery = VibeyRecovery(self)
self.trace_retention_task: Optional[asyncio.Task] = None
self.web_ui_watchdog_task: Optional[asyncio.Task] = None
self.mcp_discovery_task: Optional[asyncio.Task] = None
self._trace_retention_executor: Optional[Any] = None
self._trace_retention_cancel_event: Optional[threading.Event] = None
self._trace_retention_future: Optional[Any] = None
# Initialize update checker (use default config if not present)
from config.v2_config import UpdateConfig
update_config = getattr(config, "update", None) or UpdateConfig()
self.update_checker = UpdateChecker(self, update_config)
# Restore session mappings on startup (after handlers are initialized)
self.session_handler.restore_session_mappings()
# Clean only pre-durable status projections. Durable Turn owners remain
# running until backend restoration and exact reconciliation complete.
self.session_turns.reset_legacy_ownerless_status()
@staticmethod
def _derive_primary_platform(config) -> str:
enabled = list(getattr(config, "enabled_platforms", lambda: [getattr(config, "platform", "slack")])())
configured_primary = getattr(getattr(config, "platforms", None), "primary", getattr(config, "platform", "slack"))
if enabled:
return configured_primary if configured_primary in enabled else enabled[0]
return "avibe"
@staticmethod
def _settings_platforms_for(enabled_platforms: list[str], primary_platform: str) -> list[str]:
platforms = list(enabled_platforms)
if primary_platform not in platforms:
platforms.append(primary_platform)
if "avibe" not in platforms:
platforms.append("avibe")
return platforms
def _enabled_agent_backends(self) -> list[str]:
result: list[str] = []
agent_config = getattr(self.config, "agents", None)
if agent_config is None:
return list(getattr(self.agent_service, "agents", {}).keys()) or [DEFAULT_AGENT_BACKEND]
for backend in AGENT_BACKENDS:
cfg = getattr(agent_config, backend, None)
if bool(getattr(cfg, "enabled", False)):
result.append(backend)
return result
def _builtin_agent_models(self) -> list[tuple[str, str]]:
"""The models the built-in native Agents run, by backend name: claude, codex, opencode."""
store = self.vibe_agent_store
selections = []
for backend in sorted(NATIVE_CLI_BACKENDS):
agent = store.get_builtin_default_agent_for_backend(backend)
model = str(getattr(agent, "model", None) or "").strip()
if model:
selections.append((backend, model))
return selections
def _reconcile_vibey_agent_model(self) -> None:
"""Give the built-in Vibey Agent its catalog's first model while it has none.
A Vibey Agent without a model cannot run a turn, so this only makes an
unrunnable Agent runnable. Decided under the store's write lock, it never
replaces a model the user chose. It runs whenever the Vibey catalog
changes (``backend_catalog_changed``) and at every start, so a hand-off a
failed write or an exit lost heals on the next one. No built-in exists
when a user Agent already holds the name.
"""
from vibe.authorization import instance_owner_context
models = self.model_hub_service.store.load().agents["vibey"].models
if not models:
return
store = self.vibe_agent_store
vibey = store.get_builtin_default_agent_for_backend("vibey", enabled_only=False)
if vibey is not None and not str(vibey.model or "").strip():
vibey = store.update(
vibey.name,
model=models[0].id,
only_if_model_unset=True,
user_context=instance_owner_context(),
)
# The main Bot's Agent follows Vibey's model (docs/plans/bots.md D9). On a new install it is created
# before any model exists, so it takes Vibey's here, under the same never-replace-a-choice rule.
main = store.get_main_bot_agent()
if main is None or str(main.model or "").strip():
return
follows = vibey is not None and str(vibey.model or "").strip()
store.update(
main.name,
model=vibey.model if follows else models[0].id,
reasoning_effort=vibey.reasoning_effort if follows else None,
only_if_model_unset=True,
user_context=instance_owner_context(),
)
async def _start_vibey_without_a_model(self) -> None:
"""Start a Vibey with nothing to run on, once a models.dev copy can describe a starting model.
Only while the built-in Agent has no model: a person who emptied Vibey's list keeps
the Agent's model, so this never replaces a choice; it only makes an unrunnable
Agent runnable, as ``_reconcile_vibey_agent_model`` does.
"""
vibey = self.vibe_agent_store.get_builtin_default_agent_for_backend("vibey", enabled_only=False)
if vibey is None or str(vibey.model or "").strip():
return
await self.model_hub_service.start_vibey_without_models()
async def _model_hub_catalog_changed(self, backend: str) -> None:
"""Follow a committed Model Hub catalog change for one backend.
Model Hub's current config is read through ``model_hub_runtime``; the app
config (``self.config``) carries none of it, so nothing here copies it there.
"""
if backend == "vibey":
try:
self._reconcile_vibey_agent_model()
except Exception:
# The catalog change stands; the next start reconciles again.
logger.warning("Vibey Agent model reconciliation failed", exc_info=True)
# Each runtime moves to the committed catalog at its next turn;
# nothing restarts or drains. A disabled backend has nothing to adopt.
agent = getattr(getattr(self, "agent_service", None), "agents", {}).get(backend)
adopt_catalog = getattr(agent, "adopt_model_hub_catalog", None)
if callable(adopt_catalog):
await adopt_catalog()
async def _seed_vibey_model_supply(self) -> None:
"""Seed Vibey's Model Hub entry from existing Sources, then reconcile its built-in Agent's model.
A Vibey an earlier seed left with nothing to run on (an earlier build's empty seed,
or one that ran without a models.dev copy) starts here too, not only on the next
snapshot reconcile.
"""
model_hub_service = getattr(self, "model_hub_service", None)
if model_hub_service is None:
return
try:
await model_hub_service.seed_vibey_supply()
except Exception:
logger.warning("Vibey starting model supply failed", exc_info=True)
try:
await self._start_vibey_without_a_model()
except Exception:
logger.warning("Vibey's starting model failed at startup", exc_info=True)
try:
self._reconcile_vibey_agent_model()
except Exception:
logger.warning("Vibey Agent model reconciliation failed", exc_info=True)
def get_native_session_service(self):
if self.native_session_service is None:
from modules.agents.native_sessions.service import AgentNativeSessionService
self.native_session_service = AgentNativeSessionService()
return self.native_session_service
def _create_formatter(self, platform: str):
return get_platform_descriptor(platform).create_formatter()
@staticmethod
def _runtime_reconcile_signature(config, platform: str) -> tuple[Any, ...]:
descriptor = get_platform_descriptor(platform)
platform_config = descriptor.get_config(config)
if platform_config is None:
return ()
return tuple(getattr(platform_config, field, None) for field in descriptor.runtime_reconcile_field_names())
def _ensure_agent_route_for_platform(self, platform: str) -> None:
if platform in self.agent_router.platform_routes:
return
fallback = self.agent_router.platform_routes.get(self.primary_platform)
if fallback is None:
from modules.agent_router import PlatformRoute
fallback = PlatformRoute(default=self.agent_router.global_default)
self.agent_router.platform_routes[platform] = fallback
def _register_client_runtime(self, platform: str, client: BaseIMClient) -> None:
client.formatter = self._create_formatter(platform)
if platform not in self.platform_settings_managers:
self.settings_manager.add_platform(platform)
self.platform_settings_managers = self.settings_manager.managers
self._ensure_agent_route_for_platform(platform)
self._inject_runtime_dependencies(platform, client)
def _build_platform_client(self, platform: str, config) -> BaseIMClient:
descriptor = get_platform_descriptor(platform)
client = descriptor.create_client(config)
self._register_client_runtime(platform, client)
return client
def _sync_config_references(self, new_config) -> None:
self.config = new_config
governor = getattr(self, "_agent_resource_governor", None)
if governor is not None:
governor.update_config(getattr(new_config, "resource_governance", {"mode": "auto"}))
self.processing_indicator.config = new_config
self.audio_asr_service.config = new_config
for handler_name in ("command_handler", "settings_handler", "message_handler", "session_handler"):
handler = getattr(self, handler_name, None)
if handler is not None:
handler.config = new_config
handler.im_client = self.im_client
handler.settings_manager = self.settings_manager
handler.sessions = self.sessions
for agent in getattr(getattr(self, "agent_service", None), "agents", {}).values():
agent.config = new_config
agent.im_client = self.im_client
agent.settings_manager = self.settings_manager
agent.sessions = self.sessions
self.claude_client.config = new_config.claude
primary_formatter = self.im_clients.get(self.primary_platform, self.im_clients["avibe"]).formatter
if primary_formatter is not None:
self.claude_client.formatter = primary_formatter
async def reconcile_platforms(self, new_config) -> dict[str, Any]:
"""Hot-apply IM platform enablement and runtime credential changes."""
if self._reconcile_lock is None:
self._reconcile_lock = asyncio.Lock()
async with self._reconcile_lock:
current_enabled = list(self.enabled_platforms)
next_enabled = list(getattr(new_config, "enabled_platforms", lambda: [])())
current_set = set(current_enabled)
next_set = set(next_enabled)
removed = [platform for platform in current_enabled if platform not in next_set]
added = [platform for platform in next_enabled if platform not in current_set]
rebuilt = [
platform
for platform in next_enabled
if platform in current_set
and self._runtime_reconcile_signature(self.config, platform)
!= self._runtime_reconcile_signature(new_config, platform)
]
next_primary = self._derive_primary_platform(new_config)
for platform in removed + rebuilt:
self.im_clients.pop(platform, None)
self._removed_im_clients[platform] = RemovedPlatformIMClient(platform)
await asyncio.to_thread(self.im_client.remove_client, platform)
self.enabled_platforms = next_enabled
self.primary_platform = next_primary
self.settings_manager.set_primary_platform(next_primary)
self.platform_settings_managers = self.settings_manager.managers
for platform in removed:
self.settings_manager.remove_platform(platform)
self.agent_router.platform_routes.pop(platform, None)
for platform in next_enabled:
self._ensure_agent_route_for_platform(platform)
self._ensure_agent_route_for_platform("avibe")
for platform in rebuilt + added:
client = self._build_platform_client(platform, new_config)
self.im_clients[platform] = client
self.im_client.add_client(platform, client)
self._removed_im_clients.pop(platform, None)
self.im_client.set_primary_platform(next_primary)
self._sync_config_references(new_config)
logger.info(
"Hot-reconciled IM platforms: added=%s removed=%s rebuilt=%s primary=%s",
added,
removed,
rebuilt,
next_primary,
)
return {
"ok": True,
"added": added,
"removed": removed,
"rebuilt": rebuilt,
"enabled": next_enabled,
"primary": next_primary,
}
async def reconcile_agent_backends(self, backends: list[str]) -> dict[str, Any]:
"""Hot-apply persisted config for the requested Agent backends."""
from modules.agents.catalog import AGENT_BACKENDS
requested: list[str] = []
for backend in backends:
if backend not in AGENT_BACKENDS:
raise ValueError(f"Unsupported agent backend: {backend}")
if backend not in requested:
requested.append(backend)
states: dict[str, str] = {}
for backend in requested:
states[backend] = await self.backend_restart_coordinator.request_restart(backend, config_save=True)
logger.info("Hot-reconciled Agent backends: %s", states)
return {
"ok": True,
"backends": requested,
"states": states,
}
def _init_modules(self):
"""Initialize core modules"""
runtime_clients: Dict[str, BaseIMClient] = IMFactory.create_clients(self.config)
for platform, client in runtime_clients.items():
client.formatter = self._create_formatter(platform)
self.primary_platform = self._derive_primary_platform(self.config)
self.im_clients = dict(runtime_clients)
from modules.im.avibe import AvibeBot, AvibeConfig
self.im_clients["avibe"] = AvibeBot(AvibeConfig())
self.im_client = MultiIMClient(
dict(runtime_clients),
primary_platform=self.primary_platform,
auxiliary_clients={"avibe": self.im_clients["avibe"]},
)
self._removed_im_clients = {}
formatter = self.im_clients.get(self.primary_platform, self.im_clients["avibe"]).formatter
self.claude_client = ClaudeClient(self.config.claude, formatter)
# Initialize managers
self.session_manager = SessionManager()
self.settings_manager = MultiSettingsManager(
self._settings_platforms_for(self.enabled_platforms, self.primary_platform),
primary_platform=self.primary_platform,
)
self.platform_settings_managers = self.settings_manager.managers
self.sessions = self.settings_manager.sessions
self.native_session_service = None
self.processing_indicator = ProcessingIndicatorService(self)
self.audio_asr_service = AudioAsrService(self.config)
self._migrate_discord_guild_scope_from_config()
# Migrate legacy per-channel language into global config
self._migrate_language_from_settings()
# Legacy backend router. It is kept for platform runtime compatibility;
# product routing is resolved through VibeAgentStore.
self.agent_router = AgentRouter.from_file(None, platform=self.primary_platform)
for platform in self.enabled_platforms:
if platform not in self.agent_router.platform_routes:
self.agent_router.platform_routes[platform] = self.agent_router.platform_routes[self.primary_platform]
if "avibe" not in self.agent_router.platform_routes:
self.agent_router.platform_routes["avibe"] = self.agent_router.platform_routes[self.primary_platform]
for platform, client in runtime_clients.items():
self._inject_runtime_dependencies(platform, client)
def _migrate_discord_guild_scope_from_config(self) -> None:
if "discord" not in self.platform_settings_managers:
return
discord_config = getattr(self.config, "discord", None)
if not discord_config:
return
allowlist = getattr(discord_config, "guild_allowlist", None) or []
denylist = getattr(discord_config, "guild_denylist", None) or []
if not allowlist and not denylist:
return
manager = self.platform_settings_managers["discord"]
if manager.has_guild_scope():
return
from config.v2_settings import GuildSettings
store = manager.get_store()
default_enabled = not bool(allowlist)
guilds = {str(guild_id): GuildSettings(enabled=True) for guild_id in allowlist if str(guild_id)}
for guild_id in denylist:
guilds[str(guild_id)] = GuildSettings(enabled=False)
store.set_guilds_for_platform("discord", guilds, default_enabled=default_enabled)
store.save()
logger.info("Migrated Discord guild access from config to settings")
def _inject_runtime_dependencies(self, platform: str, client: BaseIMClient) -> None:
settings_manager = self.platform_settings_managers[platform]
settings_manager.require_mention_default = lambda: bool(
getattr(getattr(client, "config", None), "require_mention", False)
)
setter = getattr(client, "set_settings_manager", None)
if callable(setter):
setter(settings_manager)
controller_setter = getattr(client, "set_controller", None)
if callable(controller_setter):
controller_setter(self)
logger.info("Injected settings_manager and controller into %s client", platform)
def _get_lang(self) -> str:
self._refresh_config_from_disk()
return getattr(self.config, "language", "en")
def _t(self, key: str, **kwargs) -> str:
return i18n_t(key, self._get_lang(), **kwargs)
def _refresh_config_from_disk(self) -> None:
"""Hot-reload mutable message-processing settings from config.json.
Called on every ``_t()`` invocation (guarded by mtime check).
Refreshes: language, show_duration, ack_mode, include_time_info, include_user_info,
reply_enhancements, agent_progress_style, agent_status_heartbeat_ms,
agent_status_no_output_ms, and mutable platform message filters.
"""
try:
config_path = paths.get_config_path()
if not config_path.exists():
return
mtime = config_path.stat().st_mtime
if self._config_mtime != mtime:
from config.v2_config import V2Config
v2_config = V2Config.load()
self.config.language = v2_config.language
self.config.show_duration = v2_config.show_duration
self.config.ack_mode = v2_config.ack_mode
self.config.include_time_info = v2_config.include_time_info
self.config.include_user_info = v2_config.include_user_info
self.config.reply_enhancements = v2_config.reply_enhancements
self.config.agent_progress_style = v2_config.agent_progress_style
self.config.agent_status_heartbeat_ms = v2_config.agent_status_heartbeat_ms
self.config.agent_status_no_output_ms = v2_config.agent_status_no_output_ms
self.config.resource_governance = v2_config.runtime.resource_governance
self.config.harness_prompt_echo = v2_config.runtime.harness_prompt_echo
governor = getattr(self, "_agent_resource_governor", None)
if governor is not None:
governor.update_config(self.config.resource_governance)
self.config.audio_asr = v2_config.audio_asr
self.config.remote_access = v2_config.remote_access
audio_asr_service = getattr(self, "audio_asr_service", None)
if audio_asr_service is not None:
audio_asr_service.config = self.config
mutable_platform_attrs = (
"require_mention",
"guild_allowlist",
"guild_denylist",
"thread_auto_archive_minutes",
"allowed_chat_ids",
"allowed_user_ids",
"disable_link_unfurl",
"forum_auto_topic",
)
for platform, client in self.im_clients.items():
im_cfg = getattr(client, "config", None)
if im_cfg is None:
continue
latest_platform_config = get_platform_descriptor(platform).get_config(v2_config)
if latest_platform_config is None:
continue
for attr in mutable_platform_attrs:
if hasattr(im_cfg, attr) and hasattr(latest_platform_config, attr):
setattr(im_cfg, attr, getattr(latest_platform_config, attr))
self._config_mtime = mtime
except Exception as err:
logger.debug("Failed to reload config from disk: %s", err)
def _migrate_language_from_settings(self) -> None:
"""Persist legacy per-channel language into global config if missing."""
try:
config_path = paths.get_config_path()
if not config_path.exists():
return
config_payload = json.loads(config_path.read_text(encoding="utf-8"))
if isinstance(config_payload, dict) and "language" in config_payload:
return
settings_path = paths.get_settings_path()
if not settings_path.exists():
return
settings_payload = json.loads(settings_path.read_text(encoding="utf-8"))
channels = settings_payload.get("channels") if isinstance(settings_payload, dict) else None
if not isinstance(channels, dict):
return
counts: dict[str, int] = {}
supported_languages = set(get_supported_languages())
for payload in channels.values():
if not isinstance(payload, dict):
continue
value = payload.get("language")
if value in supported_languages:
counts[value] = counts.get(value, 0) + 1
if not counts:
return
chosen = sorted(counts.items(), key=lambda item: (-item[1], item[0]))[0][0]
if len(counts) > 1:
logger.warning(
"Multiple per-channel languages found; using '%s' for global config (%s)",
chosen,
counts,
)
from config.v2_config import update_config_fields
update_config_fields(lambda cfg: setattr(cfg, "language", chosen))
self.config.language = chosen
logger.info("Migrated legacy per-channel language to global config: %s", chosen)
except Exception as err:
logger.warning("Failed to migrate legacy language setting: %s", err)
def _init_handlers(self):
"""Initialize all handlers with controller reference"""
# Initialize session_handler first as other handlers depend on it
self.session_handler = SessionHandler(self)
self.command_handler = CommandHandlers(self)
self.settings_handler = SettingsHandler(self)
self.message_handler = MessageHandler(self)
# Set cross-references between handlers
self.message_handler.set_session_handler(self.session_handler)
def _init_agents(self):
from core.session_activities import SessionActivityRegistry
from modules.agents.vibey import VibeyAgent
from modules.agents.claude_agent import ClaudeAgent
from modules.agents.codex import CodexAgent
from modules.agents.opencode import OpenCodeAgent
from storage.db import get_cached_sqlite_engine
from storage.session_activities import SQLiteSessionActivityStore
activity_store = SQLiteSessionActivityStore(get_cached_sqlite_engine())
self.agent_service = AgentService(
self,
activities=SessionActivityRegistry(
activity_store,
activation_registry=self.runtime_activation,
),
activation_registry=self.runtime_activation,
)
self.agent_service.register(ClaudeAgent(self))
if self.config.codex:
try:
self.agent_service.register(CodexAgent(self, self.config.codex))
except Exception as e:
logger.error(f"Failed to initialize Codex agent: {e}")
if self.config.opencode:
try:
self.agent_service.register(OpenCodeAgent(self, self.config.opencode))
except Exception as e:
logger.error(f"Failed to initialize OpenCode agent: {e}")
# Built in: always registered, never unregistered.
self.agent_service.register(VibeyAgent(self))
def _setup_callbacks(self):
"""Setup callback connections between modules"""
def inbound(callback):
return self._dispatch_to_controller_loop(
callback,
wait_for_owner_recovery=True,
)
# Command handlers dict
# Admin protection for "set_cwd" and "settings" is now handled by
# the centralized auth pipeline (core.auth.check_auth) in IM entry points.
command_handlers = {
"start": inbound(self.command_handler.handle_start),
"new": inbound(self.command_handler.handle_new),
"cwd": inbound(self.command_handler.handle_cwd),
"set_cwd": inbound(self.command_handler.handle_set_cwd),
"resume": inbound(self.command_handler.handle_resume),
"setup": inbound(self.command_handler.handle_setup),
"settings": inbound(self.settings_handler.handle_settings),
"agent": inbound(self.settings_handler.handle_routing),
"stop": inbound(self.command_handler.handle_stop),
"bind": inbound(self.command_handler.handle_bind),
}
# IM inbound messages funnel through ``core.services.dispatch``
# alongside the CLI and the upcoming Web UI / N3 socket path so all
# three callers exercise the same business API. The lambda preserves
# the existing ``(context, text)`` callback shape that the IM clients
# know how to invoke.
from core.services.dispatch import dispatch_turn
async def _on_im_message(context, text):
await dispatch_turn(self, context, text)
# Register callbacks with the IM client
self.im_client.register_callbacks(
on_message=self._dispatch_im_message_to_controller_loop(
_on_im_message,
wait_for_owner_recovery=True,
),
on_command=command_handlers,
on_callback_query=inbound(self.message_handler.handle_callback_query),
on_settings_update=inbound(self.settings_handler.handle_settings_update),
on_change_cwd=inbound(self.command_handler.handle_change_cwd_submission),
on_routing_update=inbound(self.settings_handler.handle_routing_update),
on_routing_modal_update=inbound(
self.settings_handler.handle_routing_modal_update
),
on_resume_session=inbound(
self.session_handler.handle_resume_session_submission
),
on_ready=self._dispatch_to_controller_loop(self._on_runtime_ready),
on_transport_ready=self._dispatch_to_controller_loop(self._on_im_ready),
)
async def _await_runtime_owner_recovery(self) -> None:
recovery_complete = getattr(self, "_delivery_recovery_complete", None)
if recovery_complete is not None:
await recovery_complete.wait()
def _dispatch_to_controller_loop(
self,
callback,
*,
wait_for_owner_recovery: bool = False,
):
async def _wrapped(*args, **kwargs):
async def _invoke():
if wait_for_owner_recovery:
await self._await_runtime_owner_recovery()
return await callback(*args, **kwargs)
loop = self._loop
if loop is None:
return await _invoke()
try:
current_loop = asyncio.get_running_loop()
except RuntimeError:
current_loop = None
if current_loop is loop:
return await _invoke()
future = asyncio.run_coroutine_threadsafe(_invoke(), loop)
return await asyncio.wrap_future(future)
return _wrapped
def _dispatch_im_message_to_controller_loop(
self,
callback,
*,
wait_for_owner_recovery: bool = False,
):
tracked_platforms = {"telegram", "wechat"}
async def _wrapped(context, *args, **kwargs):
async def _invoke():
if wait_for_owner_recovery:
await self._await_runtime_owner_recovery()
return await callback(context, *args, **kwargs)
platform = self._platform_for_im_callback_context(context)
if platform in tracked_platforms:
return await self._run_on_controller_loop(_invoke)
self._schedule_controller_callback(_invoke)
return None
return _wrapped
def _platform_for_im_callback_context(self, context) -> str:
platform = str(
getattr(context, "platform", None)
or (getattr(context, "platform_specific", None) or {}).get("platform")
or ""
).strip()
if platform:
return platform
im_client = getattr(self, "im_client", None)
primary_platform = str(getattr(im_client, "primary_platform", "") or "").strip()
if primary_platform:
return primary_platform
module = str(getattr(type(im_client), "__module__", "") or "")
if module.startswith("modules.im.wechat"):
return "wechat"
if module.startswith("modules.im.telegram"):
return "telegram"
return ""
def _dispatch_to_controller_loop_background(self, callback):
async def _wrapped(*args, **kwargs):
self._schedule_controller_callback(callback, *args, **kwargs)
return _wrapped
async def _run_on_controller_loop(self, callback, *args, **kwargs):
loop = self._loop
if loop is None:
return await callback(*args, **kwargs)
try:
current_loop = asyncio.get_running_loop()
except RuntimeError:
current_loop = None
if current_loop is loop:
return await callback(*args, **kwargs)
future = asyncio.run_coroutine_threadsafe(callback(*args, **kwargs), loop)
return await asyncio.wrap_future(future)
def _schedule_controller_callback(self, callback, *args, **kwargs) -> None:
async def _runner():
await callback(*args, **kwargs)
loop = self._loop
if loop is None:
task = asyncio.create_task(_runner())
task.add_done_callback(self._log_background_callback_result)
return
try:
current_loop = asyncio.get_running_loop()
except RuntimeError:
current_loop = None
if current_loop is loop:
task = loop.create_task(_runner())
task.add_done_callback(self._log_background_callback_result)
return
future = asyncio.run_coroutine_threadsafe(_runner(), loop)
future.add_done_callback(self._log_background_callback_result)
@staticmethod
def _log_background_callback_result(future) -> None:
try:
future.result()
except (asyncio.CancelledError, concurrent.futures.CancelledError):
return
except Exception:
logger.error("Background IM message callback failed", exc_info=True)
def _publish_readiness_unless_im_runtime_failed(self) -> None:
"""Announce a started service instance, unless the IM runtime already died.
Readiness is what the upgrade supervisor waits for before it decides an
upgrade worked, so publishing it is the one irreversible statement this
startup makes: after it, nothing rolls back. The IM runtime is the piece
most likely to fail on a bad release and the only piece that fails on
another thread, which is why this asks it rather than assuming.
It asks for the recorded exception and not `Thread.is_alive()`. A Web-only
install with no IM platform configured has `im_client.run()` return
immediately and legitimately, so a liveness check would refuse readiness
forever on a service that is working perfectly.
Reaching this at all is now most of the answer. `MultiIMClient.run()`
emits the ready callback unconditionally once the platform threads are
started -- with zero platforms configured too, which is why a Web-only
install still gets here -- so an aggregate runtime that dies before that
point never calls this at all. Readiness is withheld by absence, rather
than by winning a race with the event loop's first pass the way it had
to when `run()` announced.
Which leaves the recorded exception as the weaker of the two checks
rather than the load-bearing one: the emitting thread blocks on this
callback, so it cannot record a failure while the callback runs. It is
kept because it belongs to the primitive and not to one caller -- move
the emission point and it is the check that survives the move.
"""
if self._im_run_exception is not None:
logger.error("Not publishing service readiness: the IM runtime failed during startup")
return
mark_service_instance_started()
def _run_im_runtime(self) -> None:
try:
self.im_client.run()
except BaseException as exc: # noqa: BLE001
self._im_run_exception = exc
logger.error("IM runtime thread exited with error: %s", exc, exc_info=True)
finally:
loop = self._loop
if loop is not None:
try:
# Scheduled whether or not the loop has started yet. The
# `is_running()` guard that used to stand here dropped the
# stop in exactly the case that matters: this thread is
# started just before `run_forever()`, so an adapter that
# fails immediately -- the shape a bad release takes -- lands
# here while the loop is merely created, and the stop was
# discarded. `run_forever()` then ran forever with no IM
# runtime and nothing left to ask it to stop. A callback
# queued before the loop starts runs as soon as it does.
loop.call_soon_threadsafe(loop.stop)
except RuntimeError:
# A closed loop has already stopped; nothing left to ask.
logger.debug("IM runtime could not signal the event loop to stop", exc_info=True)
async def _restore_active_polls(self, platforms: set[str]) -> None:
try:
await self._restore_active_polls_or_raise(platforms)
except Exception as e:
logger.error(f"Failed to restore active polls: {e}", exc_info=True)
async def _restore_active_polls_or_raise(self, platforms: set[str]) -> None:
coordinator = getattr(self, "backend_restart_coordinator", None)
if coordinator is not None and "opencode" in coordinator._blocked_backends():
coordinator.restore_migration_blocks()
return
opencode_agent = self.agent_service.agents.get("opencode")
if opencode_agent and hasattr(opencode_agent, "restore_active_polls"):
restored = await opencode_agent.restore_active_polls(platforms) # type: ignore[attr-defined]
if restored > 0:
logger.info(f"Restored {restored} active OpenCode poll(s)")
async def restore_polls_on_ready_transports(self) -> None:
"""Restore durable OpenCode polls on every transport that can deliver now.
IM-ready events restore polls at startup. OpenCode enabled in a running
controller missed them, so this runs once it registers. Raises when the
restore fails, so its caller can retry it; a restore skips every poll
already running, so a retry is safe.
"""
platforms = {"avibe"}
for platform in list(getattr(self, "im_clients", {}) or {}):
if self.is_im_transport_ready(platform):
platforms.add(platform)
if self.primary_platform in platforms:
platforms.add("")
await self._restore_active_polls_or_raise(platforms)
async def _recover_vibey_agent_runtime_state(self) -> None:
"""Settle what the previous process left of Vibey's work (T2, T3 admission, J5).
``vibey_recovery`` is the single owner, retrying until every Session has settled.
"""
await self.vibey_recovery.start()
async def _on_im_ready(self, *, platform: str) -> None:
"""Restore transport-owned state only after that transport can deliver."""
logger.info("IM transport ready, restoring state for %s", platform)
platforms = {platform}
if platform == self.primary_platform:
platforms.add("")
await self._restore_active_polls(platforms)
# Poll registration is durable-owner evidence needed by startup recovery.
# All work admission and user-visible delivery remain behind the barrier.
await self._await_runtime_owner_recovery()
# Interruption reports for turns on this platform were held back during
# recovery precisely because it could not deliver them yet.
notify_turns = getattr(self.session_turns, "notify_transport_ready", None)
if callable(notify_turns):
try:
reported = await notify_turns(platform)
if reported:
logger.info(
"Reported %d interrupted turn(s) on %s", reported, platform
)
except Exception:
logger.exception("Failed to report interrupted turns for %s", platform)
self.scheduled_task_service.notify_transport_ready(platform)
notify_update_checker = getattr(self.update_checker, "notify_transport_ready", None)
if callable(notify_update_checker):
notify_update_checker(platform)
try:
await self.update_checker.check_and_send_post_update_notification(ready_platform=platform)
except Exception as e:
logger.error(f"Failed to send post-update notification: {e}", exc_info=True)
def is_im_transport_ready(self, platform: str) -> bool:
return self.im_client.is_transport_ready(platform)
async def _on_runtime_ready(self) -> None:
"""Start aggregate services, and announce readiness once they are started.
The steps below the readiness line are best-effort by construction: each
catches its own failure and continues, so none of them can mean the
service is not up. The steps above it are the opposite -- a failure
there aborts this function, so the steps below never run, and a service
with no durable delivery owners, no scheduled tasks and no watch service
is not a started service however alive its process looks.
Readiness is what the upgrade supervisor waits for before it decides an
upgrade worked, so which side of that line a step falls on decides
whether a release dying in its own startup gets rolled back or gets
recorded as a success. It used to be announced from `run()`, before this
callback had run at all: recovery could fail, request shutdown, and
re-raise, and the supervisor had already been told the service was up.
Put the line here and a startup step added later is inside the claim or
outside it because someone chose, not because of where it was written.
"""
logger.info("IM runtime ready, starting core services")
workbench_platforms = {"avibe"}
if self.primary_platform == "avibe":
workbench_platforms.add("")
await self._restore_active_polls(workbench_platforms)
try:
await self._recover_runtime_owners()
except Exception:
self.request_shutdown("runtime owner recovery failed")
raise
agent_service = getattr(self, "agent_service", None)
codex_agent = getattr(agent_service, "agents", {}).get("codex")
prepare_codex_hub = getattr(codex_agent, "prepare_model_hub_runtime", None)
if callable(prepare_codex_hub):
try:
await prepare_codex_hub()
except Exception as exc: # noqa: BLE001 - direct Codex remains usable
logger.warning("Codex Hub model catalog preparation failed: %s", exc)
# --- everything above must have succeeded for the service to be up ---
# A no-op in any process that does not hold the service lock, so the
# embedded and test paths that run a controller are unaffected.
self._publish_readiness_unless_im_runtime_failed()
try:
await self.update_checker.check_and_send_post_update_notification(ready_platform="avibe")
except Exception as e:
logger.error(f"Failed to send post-update notification: {e}", exc_info=True)
try:
self.update_checker.start()
except Exception as e:
logger.error(f"Failed to start update checker: {e}", exc_info=True)
try:
self.scheduled_task_service.start()
except Exception as e:
logger.error("Failed to start scheduled task service: %s", e, exc_info=True)
try:
self.watch_service.start()
except Exception as e:
logger.error("Failed to start watch service: %s", e, exc_info=True)
try:
await self.runtime_command_watcher.start()
except Exception as e:
logger.error("Failed to start runtime command watcher: %s", e, exc_info=True)
try:
if "opencode" not in getattr(agent_service, "agents", {}):
# OpenCode is disabled, so no agent will adopt the servers a
# crashed controller recorded; disabling stops its work. The
# idle sweep retries a server that survived its stop.
from modules.agents.opencode.server import StopOutcome, stop_recorded_servers_sync
from vibe.desktop_runtime import desktop_caller_provenance
async def stop_leftover_servers() -> None:
outcomes = await asyncio.to_thread(stop_recorded_servers_sync, desktop_caller_provenance())
if any(outcome is StopOutcome.FAILED for outcome in outcomes):
raise RuntimeError("an OpenCode server a previous controller left did not stop")
await agent_service.run_until_done("disabled-startup:opencode", stop_leftover_servers)
except Exception as e:
logger.error("Failed to stop OpenCode servers a previous controller left: %s", e, exc_info=True)
try:
self._start_model_hub_snapshot_reconcile_loop()
except Exception as e:
logger.error(
"Failed to start Model Hub snapshot reconciliation: %s",
e,
exc_info=True,
)
try:
# The service's own `stop()` retires the schedule at shutdown.
start_discovery = getattr(
getattr(self, "model_hub_service", None),
"start_background_discovery",
None,
)
if callable(start_discovery):
start_discovery()
except Exception as e:
logger.error(
"Failed to start Model Hub background discovery: %s",
e,
exc_info=True,
)
try:
# Off the readiness path: on a first start the seed may wait for a
# models.dev copy, and nothing below needs it.
if getattr(self, "vibey_model_supply_task", None) is None:
self.vibey_model_supply_task = asyncio.create_task(self._seed_vibey_model_supply())
except Exception as e:
logger.error("Failed to start Vibey's model supply seed: %s", e, exc_info=True)
try:
if self.cleanup_task is None or self.cleanup_task.done():
self.cleanup_task = asyncio.create_task(self.periodic_cleanup())
except Exception as e:
logger.error("Failed to start idle session cleanup: %s", e, exc_info=True)
try:
if self.trace_retention_task is None or self.trace_retention_task.done():
self.trace_retention_task = asyncio.create_task(self._agent_events_retention_loop())
except Exception as e:
logger.error("Failed to start agent trace-event retention: %s", e, exc_info=True)
try:
# Only the service process owns the runtime's Web UI; a controller
# embedded in a test or tool must never start one.
if current_process_owns_service_instance() and (
self.web_ui_watchdog_task is None or self.web_ui_watchdog_task.done()
):
self.web_ui_watchdog_task = asyncio.create_task(watch_web_ui(lambda: self.shutdown_requested))
except Exception as e:
logger.error("Failed to start the Web UI watchdog: %s", e, exc_info=True)
try:
# Off the readiness path: native MCP handshake must not delay startup.
task = asyncio.create_task(self.mcp_runtime.submit("discover", {}))
def _log_mcp_discovery(done: asyncio.Task) -> None:
if done.cancelled():
return
exc = done.exception()
if exc is not None:
logger.error("MCP Hub discovery failed", exc_info=exc)
task.add_done_callback(_log_mcp_discovery)
self.mcp_discovery_task = task
except Exception as e:
logger.error("Failed to start MCP Hub discovery: %s", e, exc_info=True)
async def _recover_runtime_owners(self) -> None:
"""Restore durable execution owners before any producer can admit work."""
model_hub_service = getattr(self, "model_hub_service", None)
recover_model_hub = getattr(
model_hub_service,
"recover_runtime_intent",
None,
)
if callable(recover_model_hub):
try:
await recover_model_hub()
except Exception:
logger.exception(
"Model Hub runtime recovery failed; retaining blocked backend admission"
)
finally:
coordinator = getattr(self, "backend_restart_coordinator", None)
if coordinator is not None:
coordinator.restore_migration_blocks()
# recovery.md: T2 before T4. Vibey settles what a restart left open first; a
# command still running for a Turn not yet settled is left to the Turn owner
# below, which continues the Turn (its own T2 waits for the command) or has the
# command handed to a Watch before its notice names it. Best effort: its single
# owner retries what it could not settle.
try:
await self._recover_vibey_agent_runtime_state()
except Exception:
logger.exception("Failed to recover Vibey's runtime state")
recover_deliveries = getattr(
self.session_turns,
"recover_durable_delivery_state",
None,
)
if callable(recover_deliveries):
try:
recovered = await recover_deliveries(service_restart=True)
if recovered:
logger.info(
"Recovered durable Session delivery owners for %s",
",".join(recovered),
)
except Exception:
logger.exception("Failed to recover durable Session delivery owners")
raise
recover_queue = getattr(
self.session_turns,
"recover_persisted_agent_run_queue",
None,
)
if callable(recover_queue):
try:
recovered = await recover_queue()
if recovered:
logger.info(
"Recovered persisted Workbench Agent Run queues for %s",
",".join(recovered),
)
except Exception:
logger.exception("Failed to recover persisted Workbench Agent Run queues")
raise
try:
self.scheduled_task_service.recover_processing_requests()
except Exception:
logger.exception("Failed to recover fallback request owners")
raise
await self.runtime_work_supervisor.activate()
self._delivery_recovery_complete.set()
# Utility methods used by handlers
def get_cwd(self, context: MessageContext) -> str:
"""Get the current cwd without creating an Agent Session row."""
payload = context.platform_specific or {}
source = str(payload.get("turn_source") or "human")
return self.resolve_agent_run_target(context, source=source, create_session=False).workdir
def resolve_agent_run_target(
self,
context: MessageContext,
*,
base_session_id: Optional[str] = None,
source: str = "human",
create_session: bool = True,
):
"""Resolve the shared execution target for one agent turn."""
from core.services.agent_run_target import resolve_agent_run_target
return resolve_agent_run_target(
context,
controller=self,
base_session_id=base_session_id,
source=source,
create_session=create_session,
)
def _get_settings_key(self, context: MessageContext) -> str:
"""Get settings key based on context.
For DM contexts, returns user_id so per-user settings apply.
For channel contexts, returns channel_id for per-channel settings. A
Telegram forum topic returns a runtime thread key so its explicit
override can win before the parent channel fallback.
Relies on the ``is_dm`` flag set by the IM layer in
``context.platform_specific`` (see Phase 2 of the refactoring).
"""
from core.message_context import resolve_context_scope_settings_key
return resolve_context_scope_settings_key(context)
def _get_session_key(self, context: MessageContext) -> str:
"""Get a globally unique session-scope key.
Unlike ``_get_settings_key`` (which returns a raw ID for settings
lookup routed by platform), this key must be unique across all
platforms so that sessions, polls, and message-consolidation
tracking never collide.
"""
platform = context.platform or (context.platform_specific or {}).get("platform") or self.primary_platform
from core.message_context import resolve_context_settings_key
settings_key = resolve_context_settings_key(context)
return build_context_session_key(context, platform=platform, settings_key=settings_key)
def _get_turn_sink_key(self, context: MessageContext) -> str:
"""Get the live turn sink's key for ``context``.
Thread-scoped, unlike ``_get_session_key``: the sink is one agent
session's turn-concurrency slot, so sharing it across a channel's
threads made ``dispatch_turn`` refuse unrelated sessions' turns. See
``core.message_context.build_context_turn_sink_key``.
"""
from core.message_context import build_context_turn_sink_key
return build_context_turn_sink_key(context, session_key=self._get_session_key(context))
def backend_alive(self, context: MessageContext) -> Optional[bool]:
"""Best-effort backend liveness for the concise status bubble's footer.
Delegates to ``AgentService.backend_alive`` (which dispatches to the
per-backend probe). Returns ``None`` when unknown — the dispatcher
treats ``None``/missing as alive so it never false-alarms ⚠️.
"""
service = getattr(self, "agent_service", None)
probe = getattr(service, "backend_alive", None)
if not callable(probe):
return None
try:
return probe(context)
except Exception:
logger.debug("backend_alive delegation failed", exc_info=True)
return None
# ---- concise status-bubble settings (read by the message dispatcher) ----
def get_progress_style_for_context(self, context: MessageContext) -> str:
"""Resolve the process-message UX style for this context: concise|verbose|off.
Currently a global config setting; per-channel overrides can layer on top
here later without touching the dispatcher.
"""
_refresh_status_bubble_config(self)
value = getattr(self.config, "agent_progress_style", DEFAULT_AGENT_PROGRESS_STYLE)
return value if value in ("concise", "verbose", "off") else DEFAULT_AGENT_PROGRESS_STYLE
def uses_concise_status_bubble(self, context: MessageContext) -> bool:
"""True when this turn renders a concise status bubble (Slack/Discord +
progress_style=concise). The single source of truth shared by the message
dispatcher (which creates the bubble) and the processing indicator (which
suppresses its ack-message/reaction so there is no duplicate signal)."""
# Resolve platform with the SAME fallback the dispatcher's _get_platform
# uses (config.platform) so the bubble-creation gate and this suppression
# gate never disagree on an edge config; both then read the SAME
# ``supports_status_bubble`` capability rather than a hardcoded platform set.
platform = (
context.platform
or (context.platform_specific or {}).get("platform")
or getattr(self.config, "platform", None)
or self.primary_platform
)
if not get_platform_descriptor(platform).capabilities.supports_status_bubble:
return False
return self.get_progress_style_for_context(context) == "concise"
def get_heartbeat_interval_ms_for_context(self, context: MessageContext) -> int:
_refresh_status_bubble_config(self)
value = getattr(self.config, "agent_status_heartbeat_ms", 8000)
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 8000
def get_no_output_hint_after_ms_for_context(self, context: MessageContext) -> int:
_refresh_status_bubble_config(self)
value = getattr(self.config, "agent_status_no_output_ms", 180000)
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else 180000
def get_im_client_for_context(self, context: Optional[MessageContext] = None) -> BaseIMClient:
if context is None:
return self.im_clients[self.primary_platform]
platform = context.platform or (context.platform_specific or {}).get("platform") or self.primary_platform
client = self.im_clients.get(platform)
if client is not None:
return client
removed_client = self._removed_im_clients.get(platform)
if removed_client is not None:
return removed_client
return self.im_clients[self.primary_platform]
def _get_im_client_for_platform(self, platform: str) -> BaseIMClient:
client = self.im_clients.get(platform)
if client is not None:
return client
removed_client = self._removed_im_clients.get(platform)
if removed_client is not None:
return removed_client
return self.im_clients[self.primary_platform]
# --- Streaming turn sinks -------------------------------------------
# A live SSE caller registers a sink before dispatching a turn so the
# async agent receiver can forward chunks to the open stream and mark the
# turn complete. See ``core/services/dispatch.py`` and the
# ``ConsolidatedMessageDispatcher._stream_chunk`` consumer.
@property
def active_turn_sinks(self) -> Dict[str, Dict[str, Any]]:
# Owned by the turn owner (FSM); exposed here for back-compat readers.
return self.session_turns.active_turn_sinks
def register_turn_sink(self, session_key: str, *, on_chunk, done_event, turn_token=None, context=None) -> None:
self.session_turns.register_turn_sink(
session_key,
on_chunk=on_chunk,
done_event=done_event,
turn_token=turn_token,
context=context,
)
def pop_turn_sink(self, session_key: str, done_event=None) -> None:
self.session_turns.pop_turn_sink(session_key, done_event)
def get_turn_sink(self, session_key: str) -> Optional[Dict[str, Any]]:
return self.session_turns.get_turn_sink(session_key)
def bind_context_to_turn_sink(
self,
context: MessageContext,
*,
agent_session_id: Optional[str] = None,
backend_base_session_id: Optional[str] = None,
) -> Optional[Dict[str, Any]]:
return self.session_turns.bind_context_to_turn_sink(
context,
agent_session_id=agent_session_id,
backend_base_session_id=backend_base_session_id,
)
def settle_bound_turn_sink(self, binding: Optional[Dict[str, Any]]) -> bool:
return self.session_turns.settle_bound_turn_sink(binding)
def mark_turn_complete(
self,
context: Optional[MessageContext] = None,
*,
settled_by: str = SETTLED_BY_NO_TERMINAL_RESULT,
) -> None:
"""Release a streaming turn sink whose turn finished WITHOUT emitting a
result (missing/disabled backend, dedup, inline-stop, error, or any
synchronous no-agent path) so the SSE dispatch closes promptly instead
of waiting out the safety timeout. No-op for non-streaming turns or
when an agent turn is genuinely in flight (the result emit releases it)."""
if context is None:
return
from core.message_context import resolve_turn_sink_key
sink = self.get_turn_sink(resolve_turn_sink_key(self, context))
if sink is None:
return
# Turn-token guard (mirrors ``_stream_chunk`` / ``_is_active_turn``): a
# SUPERSEDED or OLDER turn ending (a stopped turn whose backend later fires
# turn/completed, or a scheduled/watch run that carries no token) must not
# close the CURRENT turn's stream — the ONE active-turn token rule (shared
# with _stream_chunk + _is_active_turn) decides if this emit is the live
# turn's; a different OR absent token is stale, fail-open when tokenless.
from core.session_turns import emit_matches_active_turn
if not emit_matches_active_turn(sink, context):
return
# Record WHY the waiter is being released, so ``dispatch_turn`` can tell its
# caller. An ``agent_run`` released with a no-result settlement must be
# terminalized by that caller: nothing else will ever do it (see
# docs/plans/agent-run-zombie-settlement.md). ``setdefault`` keeps a real
# terminal result — which always runs before this ``finally`` — as the
# winning settlement.
#
# The default is the no-dispatch case this method was written for (blank
# prompt, dedup, inline stop). A caller that IS a terminal output overrides
# it: the dispatcher passes ``SETTLED_BY_TURN_ONLY_RESULT`` when the output
# completes the turn but deliberately leaves the run to another owner (a
# requeued Claude Activity), which must NOT be settled here (Codex P1).
sink.setdefault("settled_by", settled_by or SETTLED_BY_NO_TERMINAL_RESULT)
done = sink.get("done_event")
if done is not None:
done.set()
# ----- Live agent-runtime status (workbench sidebar dot) -------------
#
# ``agent_sessions.agent_status`` is a projection of durable Turn ownership.
# Admission projects running; the terminal transaction projects the exact
# successor state. Legacy non-durable paths use this writer directly.
@staticmethod
def _session_id_from_context(context: Optional[MessageContext]) -> Optional[str]:
spec = getattr(context, "platform_specific", None) or {}
sid = spec.get("agent_session_id")
return sid if isinstance(sid, str) and sid else None
def set_agent_status(self, session_id: Optional[str], status: str) -> None:
"""Persist a session's agent_status and broadcast ``session.status``.
Best-effort + idempotent: a no-op when the value is unchanged (the
service reports it), when ``session_id`` is empty, or when the DB write
fails. The realtime event rides the same controller→browser bus as
``turn.start`` / ``turn.end`` so the sidebar dot updates without a refetch.
"""
if not session_id:
return
try:
from core.services import sessions as workbench_sessions_service
from storage.db import create_sqlite_engine
engine = create_sqlite_engine()
try:
with engine.begin() as conn:
changed = workbench_sessions_service.set_agent_status(conn, session_id, status)
finally:
# Dispose the per-turn engine promptly: this fires on every
# workbench turn start/end, so leaking it would pin SQLite
# connections/FDs until GC under active Chat use (Codex P3).
engine.dispose()
if changed:
from core.inbox_events import bus
bus.publish("session.status", {"session_id": session_id, "agent_status": status})
except Exception:
logger.debug("set_agent_status failed for session=%s", session_id, exc_info=True)
def get_settings_manager_for_context(self, context: Optional[MessageContext] = None) -> SettingsManager:
if context is None:
return self.platform_settings_managers[self.primary_platform]
platform = context.platform or (context.platform_specific or {}).get("platform") or self.primary_platform
return self.platform_settings_managers.get(platform, self.platform_settings_managers[self.primary_platform])
async def archive_session(
self,
raw_session_id: str,
*,
deadline_seconds: float = 5.0,
) -> dict[str, Any]:
"""Archive one Workbench session through its serialized lifecycle."""
from core.services import sessions as workbench_sessions_service
from storage.agent_session_rows import WORKSPACE_NOTICE_SESSION_ID, is_bot_session, is_bot_session_id
from storage.db import create_sqlite_engine
if (
not isinstance(raw_session_id, str)
or not raw_session_id
or raw_session_id != raw_session_id.strip()
):
raise ValueError("invalid Workbench session ID")
if raw_session_id == WORKSPACE_NOTICE_SESSION_ID:
raise workbench_sessions_service.ReservedSessionError(raw_session_id)
if is_bot_session_id(raw_session_id):
def owned_by_bot() -> bool:
engine = create_sqlite_engine()
try:
with engine.connect() as conn:
return is_bot_session(conn, raw_session_id)
finally:
engine.dispose()
if await asyncio.to_thread(owned_by_bot):
raise workbench_sessions_service.ReservedSessionError(raw_session_id)
async def retire_computer_use_session() -> None:
js_kernels = getattr(self, "computer_js_kernels", None)
if js_kernels is not None:
js_kernels.discard_session(raw_session_id)
computer_server = getattr(self, "computer_use_server", None)
retire_session = getattr(computer_server, "retire_session", None)
if not callable(retire_session):
return
try:
cleanup = await retire_session(raw_session_id)
if isinstance(cleanup, dict) and cleanup.get("isError") is True:
logger.warning(
"Computer Use session retirement returned an error for %s: %s",
raw_session_id,
cleanup,
)
except Exception:
logger.warning(
"Computer Use session retirement failed for %s",
raw_session_id,
exc_info=True,
)
def read_session() -> dict[str, Any]:
engine = create_sqlite_engine()
try:
with engine.connect() as conn:
return workbench_sessions_service.get_session(
conn,
raw_session_id,
)
finally:
engine.dispose()
existing = await asyncio.to_thread(read_session)
if existing.get("status") == "archived":
turn_manager = getattr(self, "session_turns", None)
mark_non_admitting = getattr(
turn_manager,
"mark_session_non_admitting",
None,
)
if callable(mark_non_admitting):
mark_non_admitting(raw_session_id)
await retire_computer_use_session()
return existing
def archive_session() -> dict[str, Any]:
engine = create_sqlite_engine()
try:
with engine.begin() as conn:
return workbench_sessions_service.archive_session(
conn,
raw_session_id,
)
finally:
engine.dispose()
async def archive_operation() -> dict[str, Any]:
return await run_blocking(archive_session)
turn_manager = getattr(self, "session_turns", None)
turn_lifecycle = getattr(turn_manager, "run_session_lifecycle", None)
if callable(turn_lifecycle):
result = await turn_lifecycle(
raw_session_id,
archive_operation,
deadline_seconds=deadline_seconds,
)
else:
result = await archive_operation()
await retire_computer_use_session()
return result
def update_thread_message_id(self, context: MessageContext) -> None:
"""Run real-turn-start hooks after the runtime gate is acquired."""
self.message_dispatcher.update_thread_message_id(context)
async def clear_consolidated_message_id(
self, context: MessageContext, trigger_message_id: Optional[str] = None
) -> None:
"""Clear consolidated message anchor so next log chunk starts fresh."""
await self.message_dispatcher.clear_consolidated_message_id(context, trigger_message_id)
def resolve_agent_for_context(self, context: MessageContext) -> str:
"""Unified agent resolution with dynamic override support.
Priority:
1. explicit/session Vibe Agent target
2. existing session backend snapshot
3. default Vibe Agent route
4. AgentService.default_agent / first registered backend compatibility fallback
"""
target = self._agent_run_target_payload(context)
payload = context.platform_specific or {}
target_agent_id = payload.get("vibe_agent_id") or (target.get("agent_id") if target else None)
target_agent_name = target.get("agent_name") if target else None
target_backend = target.get("agent_backend") if target else None
if target_agent_id or target_agent_name:
vibe_agent = self.resolve_vibe_agent_for_context(
context,
override_agent_id=str(target_agent_id) if target_agent_id else None,
override_agent_name=str(target_agent_name) if target_agent_name else None,
required=False,
)
if vibe_agent:
return vibe_agent.backend
if target_backend and str(target_backend) in AGENT_BACKENDS:
return str(target_backend)
vibe_agent = self.resolve_vibe_agent_for_context(context, required=False)
if vibe_agent:
return vibe_agent.backend
return self._fallback_registered_agent_backend()
def _fallback_registered_agent_backend(self) -> str:
default_agent = getattr(self.agent_service, "default_agent", None)
registered = getattr(self.agent_service, "agents", {})
if default_agent in registered:
return str(default_agent)
if registered:
return next(iter(registered))
return DEFAULT_AGENT_BACKEND
def resolve_vibe_agent_for_context(
self,
context: MessageContext,
*,
override_agent_id: Optional[str] = None,
override_agent_name: Optional[str] = None,
required: bool = True,
) -> Optional[VibeAgent]:
target = self._agent_run_target_payload(context)
platform = context.platform or (context.platform_specific or {}).get("platform") or self.primary_platform
if platform == "avibe" and target:
routing = None
else:
settings_key = self._get_settings_key(context)
settings_manager = self.get_settings_manager_for_context(context)
routing = settings_manager.get_channel_routing(settings_key)
agent_name = override_agent_name or (target.get("agent_name") if target else None) or (
routing.agent_name if routing else None
)
agent_id = override_agent_id or (target.get("agent_id") if target else None)
try:
if agent_id:
return self.vibe_agent_store.require_reference_by_id(str(agent_id))
if agent_name:
return self.vibe_agent_store.require_reference(agent_name)
default_agent = self.vibe_agent_store.get_default_agent()
if default_agent is not None:
return default_agent
if required:
return self.vibe_agent_store.require("default")
return None
except Exception as exc:
if required:
raise
logger.warning(
"Scope references Vibe Agent '%s' but it cannot be resolved: %s",
agent_id or agent_name or "default",
exc,
)
return None
@staticmethod
def _agent_run_target_payload(context: MessageContext) -> dict[str, Any]:
payload = context.platform_specific or {}
target = payload.get("agent_run_target")
if isinstance(target, dict):
return target
session_target = payload.get("agent_session_target")
return session_target if isinstance(session_target, dict) else {}
def get_opencode_overrides(self, context: MessageContext) -> tuple[Optional[str], Optional[str], Optional[str]]:
"""Get OpenCode agent, model, and reasoning effort overrides for this channel.
Returns:
Tuple of (opencode_agent, opencode_model, opencode_reasoning_effort)
or (None, None, None) if no overrides.
"""
target = self._agent_run_target_payload(context)
platform = context.platform or (context.platform_specific or {}).get("platform") or self.primary_platform
if platform == "avibe" and target:
return (
_target_agent_variant(
target.get("agent_variant"),
target.get("agent_backend"),
target.get("agent_name"),
),
_optional_target_str(target.get("model")),
_optional_target_str(target.get("reasoning_effort")),
)
settings_key = self._get_settings_key(context)
settings_manager = self.get_settings_manager_for_context(context)
routing = settings_manager.get_channel_routing(settings_key)
if routing:
from config.v2_settings import routing_model_for_backend, routing_reasoning_effort_for_backend
return (
routing.opencode_agent,
routing_model_for_backend(routing, "opencode"),
routing_reasoning_effort_for_backend(routing, "opencode"),
)
return None, None, None
def get_codex_overrides(self, context: MessageContext) -> tuple[Optional[str], Optional[str], Optional[str]]:
"""Get Codex agent, model, and reasoning effort overrides for this channel."""
target = self._agent_run_target_payload(context)
platform = context.platform or (context.platform_specific or {}).get("platform") or self.primary_platform
if platform == "avibe" and target:
return (
_target_agent_variant(
target.get("agent_variant"),
target.get("agent_backend"),
target.get("agent_name"),
),
_optional_target_str(target.get("model")),
_optional_target_str(target.get("reasoning_effort")),
)
settings_key = self._get_settings_key(context)
settings_manager = self.get_settings_manager_for_context(context)
routing = settings_manager.get_channel_routing(settings_key)
if routing:
from config.v2_settings import routing_model_for_backend, routing_reasoning_effort_for_backend
return (
routing.codex_agent,
routing_model_for_backend(routing, "codex"),
routing_reasoning_effort_for_backend(routing, "codex"),
)
return None, None, None
async def emit_agent_message(
self,
context: MessageContext,
message_type: str,
text: str,
parse_mode: Optional[str] = "markdown",
*,
is_error: bool = False,
level: str = "normal",
status_label: Optional[str] = None,
result_footer: Optional[str] = None,
output: MessageOutput | None = None,
terminal_error: Optional[str] = None,
delivery: Any = None,
citations: Optional[CitationBundle] = None,
hosted: Optional[dict] = None,
):
"""Backward-compatible entrypoint; delegated to message dispatcher."""
result = await self.message_dispatcher.emit_agent_message(
context=context,
message_type=message_type,
text=text,
parse_mode=parse_mode,
is_error=is_error,
level=level,
status_label=status_label,
result_footer=result_footer,
output=output,
terminal_error=terminal_error,
# Forwarded ONLY when a caller asked for it, for the same reason
# ``emit_backend_failure`` does: ``message_dispatcher`` is a
# substitutable collaborator (six test suites replace it), so passing
# an optional diagnostic unconditionally would change the required
# signature of every stand-in. The citation sidecar rides the same way.
**({"delivery": delivery} if delivery is not None else {}),
**({"citations": citations} if citations else {}),
**({"hosted": hosted} if hosted else {}),
)
# The dispatcher's terminal commit already handed off this write beside
# the Turn's signal (#2714 / #2706), which makes this call a no-op even if
# its completion is pending. Still cover a dispatcher that skipped it.
manager = getattr(self, "session_turns", None)
complete = getattr(manager, "on_terminal_delivery_complete", None)
if callable(complete):
complete(context)
return result
def note_session_tokens(self, context: MessageContext, *, total: int) -> None:
"""Report the session's current context-window occupancy for the status
footer (backend-agnostic). SETs an absolute snapshot; the next footer render
shows it. No-op if the dispatcher is unavailable (partially-wired test
controllers)."""
dispatcher = getattr(self, "message_dispatcher", None)
if dispatcher is None:
return
dispatcher.note_session_tokens(context, total=total)
def session_token_field(self, context: MessageContext) -> str:
"""The compact ``{n} tok`` footer field for the session's current
context-window occupancy, or "" when unknown/zero or the dispatcher is
unavailable (partially-wired test controllers)."""
dispatcher = getattr(self, "message_dispatcher", None)
if dispatcher is None:
return ""
return dispatcher.session_token_field(context)
# Main run method
@property
def service_lock_safe_to_release(self) -> bool:
return bool(getattr(self, "_service_lock_safe_to_release", False))
@property
def shutdown_requested(self) -> bool:
return bool(getattr(self, "_shutdown_requested", False))
def request_shutdown(self, reason: str = "requested") -> None:
"""Schedule shutdown on the controller loop without blocking its owner."""
if getattr(self, "_shutdown_requested", False):
return
self._shutdown_requested = True
self._service_lock_safe_to_release = False
loop = getattr(self, "_loop", None)
if loop is None or loop.is_closed() or not loop.is_running():
return
try:
loop.call_soon_threadsafe(self._ensure_shutdown_task, reason)
except RuntimeError:
logger.exception("Failed to schedule controller shutdown")
self._shutdown_tainted = True
def _ensure_shutdown_task(self, reason: str = "requested") -> None:
task = getattr(self, "_shutdown_task", None)
if task is not None and not task.done():
return
self._shutdown_task = asyncio.create_task(
self._shutdown_on_loop(reason),
name="controller-shutdown",
)
async def _shutdown_on_loop(self, reason: str) -> None:
"""Join passive recovery owners before allowing the loop to stop."""
logger.info("Controller shutdown started: %s", reason)
try:
stop_task = self._begin_runtime_work_stack_shutdown()
grace = max(
0.0,
float(
getattr(
self,
"_runtime_work_shutdown_grace_seconds",
_RUNTIME_WORK_SHUTDOWN_GRACE_SECONDS,
)
),
)
done, _ = await asyncio.wait({stop_task}, timeout=grace)
if not done:
self._shutdown_tainted = True
logger.critical(
"Runtime work shutdown exceeded %.1fs; retaining the "
"service lease until exact workers join",
grace,
)
await asyncio.shield(stop_task)
except Exception:
self._shutdown_tainted = True
logger.exception("Controller shutdown could not join runtime work")
finally:
loop = self._loop
if loop is not None and loop.is_running():
loop.call_soon(loop.stop)
def _begin_runtime_work_stack_shutdown(self) -> asyncio.Task[None]:
task = getattr(self, "_runtime_work_shutdown_task", None)
if task is None:
task = asyncio.create_task(
self._stop_runtime_work_stack(),
name="controller-runtime-work-stack-stop",
)
self._runtime_work_shutdown_task = task
return task
async def _join_runtime_work_stack_shutdown(self) -> None:
await asyncio.shield(self._begin_runtime_work_stack_shutdown())
async def _stop_runtime_work_stack(self) -> None:
"""Stop lane consumers before disposing their shared executor."""
supervisor = getattr(self, "runtime_work_supervisor", None)
quiesce = getattr(supervisor, "quiesce", None)
if callable(quiesce):
quiesce()
# Controller-generation lanes can still be finishing work after
# quiesce. Join them before ScheduledTaskService releases durable Turn
# owners; otherwise a delivery-recovery worker can admit a new Turn
# immediately after the final owner snapshot.
controller_tokens = tuple(getattr(self, "_runtime_work_tokens", ()))
if controller_tokens:
begin_unregister = getattr(supervisor, "begin_unregister", None)
if not callable(begin_unregister):
self._shutdown_tainted = True
raise RuntimeError(
"runtime work supervisor cannot join controller lanes"
)
controller_lane_joins = [
begin_unregister(token) for token in controller_tokens
]
self._runtime_work_tokens = []
controller_results = await asyncio.gather(
*controller_lane_joins,
return_exceptions=True,
)
controller_errors = [
result
for result in controller_results
if isinstance(result, BaseException)
]
if controller_errors:
self._shutdown_tainted = True
raise RuntimeError(
"controller runtime work lane shutdown failed"
) from controller_errors[0]
dispatcher = getattr(self, "message_dispatcher", None)
drain_activity = getattr(dispatcher, "drain_agent_run_activity", None)
if callable(drain_activity):
await drain_activity()
await self._stop_model_hub_snapshot_reconciliation()
service_stops: list[tuple[str, asyncio.Task[None]]] = []
for service_name in (
"model_hub_service",
"scheduled_task_service",
"watch_service",
):
service = getattr(self, service_name, None)
stop = getattr(service, "stop", None)
if callable(stop):
service_stops.append(
(
service_name,
asyncio.create_task(
stop(),
name=f"controller-{service_name}-stop",
),
)
)
results = await asyncio.gather(
*(task for _, task in service_stops), return_exceptions=True
)
errors = [result for result in results if isinstance(result, BaseException)]
# Stopping the task service settles its in-flight Runs. Disposable
# runtimes waiting on those rows must then finish before the loop exits,
# especially adopted OpenCode servers that survive normal shutdown. If
# that settlement owner failed, the waiting tasks cannot safely finish.
task_service_failed = any(
service_name == "scheduled_task_service"
and isinstance(result, BaseException)
for (service_name, _), result in zip(service_stops, results)
)
drain_close_after = getattr(dispatcher, "drain_close_after_runtime", None)
if callable(drain_close_after) and not task_service_failed:
try:
await drain_close_after()
except BaseException as exc: # noqa: BLE001
errors.append(exc)
stop_supervisor = getattr(supervisor, "stop", None)
if callable(stop_supervisor):
try:
await stop_supervisor()
except BaseException as exc: # noqa: BLE001
errors.append(exc)
if errors:
self._shutdown_tainted = True
raise RuntimeError("runtime work stack shutdown failed") from errors[0]
def run(self):
"""Run the controller"""
logger.info("Starting Claude Proxy Controller with platforms: %s", ", ".join(self.enabled_platforms))
try:
self._loop = asyncio.new_event_loop()
asyncio.set_event_loop(self._loop)
# This thread runs the loop: the one thread an activation commit may hold
# the runtime boundary across its I/O on.
self.runtime_activation.bind_loop_thread()
# The thread whose SQLite write transactions are counted (#2710).
loop_writers.bind_loop_thread()
# Names whatever blocks this loop while it blocks; see core/loop_watchdog.py.
self._loop_watchdog = EventLoopWatchdog(self._loop)
self._loop_watchdog.start()
self._schedule_model_hub_snapshot_reconcile()
self.show_git_checkpoint_service.start()
# Internal control-IPC ASGI server for the Web UI / future
# ``vibe agent run --sync`` cross-process callers. Lives on
# the same loop as the IM dispatch path so they share one
# asyncio scheduler. See core/internal_server.py.
try:
from core import internal_server as _internal_server
self._internal_server_task = _internal_server.start(self)
except Exception:
logger.exception("internal dispatch server failed to schedule; UI fallback will use the queue path")
self._internal_server_task = None
self._im_thread = threading.Thread(target=self._run_im_runtime, name="im-runtime", daemon=True)
self._im_thread.start()
if self._shutdown_requested:
self._ensure_shutdown_task("pre-loop request")
# Readiness is NOT published here, and where it is published is the
# whole point rather than a placement detail. This function starts
# startup; it does not finish it. Announcing from a line in the
# middle means every step written after that line is outside the
# claim by accident, which is the mistake this loop has now made
# four times at four different steps -- the caller, the IM thread,
# the UI probe, and the durable-owner recovery. The announcement
# belongs at a stated boundary inside the function that runs the
# startup steps, so a step added later is inside or outside it by
# decision. See `_on_runtime_ready`.
self._loop.run_forever()
if self._im_run_exception and not isinstance(self._im_run_exception, (KeyboardInterrupt, SystemExit)):
raise self._im_run_exception
except KeyboardInterrupt:
logger.info("Received keyboard interrupt, shutting down...")
except Exception as e:
logger.error(f"Error in main run loop: {e}", exc_info=True)
finally:
# The loop no longer serves Sessions; cleanup that reuses its thread is
# not a loop writer (#2710).
loop_writers.unbind_loop_thread()
self.cleanup_sync()
if not getattr(self, "_shutdown_tainted", False):
self._service_lock_safe_to_release = True
# Let the transport owner close its listener and remove only the
# endpoint it published. This is required for the Windows descriptor:
# a stale Controller must not unlink a successor's endpoint.
try:
internal_task = getattr(self, "_internal_server_task", None)
if internal_task is not None and not internal_task.done():
internal_task.cancel()
self._loop.run_until_complete(internal_task)
except asyncio.CancelledError:
pass
except Exception as exc:
logger.debug("Internal control IPC cleanup skipped: %s", exc)
self._internal_server_task = None
loop_watchdog = getattr(self, "_loop_watchdog", None)
if loop_watchdog is not None:
loop_watchdog.stop()
self._loop_watchdog = None
if self._loop is not None:
try:
self._loop.stop()
except Exception:
pass
self._loop.close()
self._loop = None
def _get_idle_cleanup_timeouts(self) -> tuple[int, int]:
"""Return normalized idle cleanup timeouts for Claude and Codex."""
claude_config = getattr(self.config, "claude", None)
codex_config = getattr(self.config, "codex", None)
claude_timeout = int(
max(0, getattr(claude_config, "idle_timeout_seconds", DEFAULT_AGENT_IDLE_TIMEOUT_SECONDS) or 0)
)
codex_timeout = (
int(max(0, getattr(codex_config, "idle_timeout_seconds", DEFAULT_AGENT_IDLE_TIMEOUT_SECONDS) or 0))
if codex_config is not None
else 0
)
return claude_timeout, codex_timeout
_AGENT_EVENTS_RETENTION_CHECK_INTERVAL_SECONDS = 3600
def _agent_events_retention_config(self) -> dict[str, Any] | None:
"""Read the retention window from the persisted V2 config, failing closed.
The controller's ``self.config`` is an ``AppCompatConfig`` shim without
the runtime section, so this reloads ``V2Config`` directly. Deletion is
irreversible, so every ambiguous state disables the automatic pass:
config recovery defaults (``load_warnings`` set — the persisted policy
is unknown), a malformed opt-out, or a malformed window (a shorter
default must never silently override a longer persisted policy).
"""
from storage import agent_events_retention
try:
from config.v2_config import V2Config
config = V2Config.load()
except Exception:
logger.warning("Agent trace-event retention: config unreadable; disabling automatic pass", exc_info=True)
return None
policy = agent_events_retention.resolve_policy(config)
if policy.recovered:
logger.warning(
"Agent trace-event retention: persisted policy is malformed or recovered; "
"disabling automatic pass until the config is fixed"
)
return None
if not policy.enabled:
return None
return {"days": policy.days}
def _run_agent_events_retention_pass(
self, cancel_event: Optional[threading.Event] = None
) -> dict[str, Any]:
"""One maintenance pass on the worker thread (no VACUUM: manual-only).
Full ``VACUUM`` holds SQLite's sole write lock for the entire rewrite;
on a large live database that can exceed every writer's busy timeout.
The automatic path therefore deletes rows only and reports compaction
as not attempted — ``vibe data retention --run`` owns compaction.
"""
from storage import agent_events_retention
from storage.db import get_cached_sqlite_engine
self._run_sqlite_backup_expiry_pass(cancel_event)
# Skill privacy retention remains active when collection/tool retention
# is disabled; both policies share this bounded maintenance worker.
engine = get_cached_sqlite_engine()
agent_events_retention.run_skill_retention(engine, cancel_event=cancel_event)
config = self._agent_events_retention_config()
if config is None:
return {"status": "disabled"}
return agent_events_retention.run_once(
engine,
retention_days=int(config["days"]),
compact=False,
cancel_event=cancel_event,
)
def _run_sqlite_backup_expiry_pass(self, cancel_event=None) -> dict:
import os
from config import paths
from storage.backups import expire_sqlite_migration_backups
from vibe.runtime import (
confirm_pending_sqlite_backup_readiness,
current_process_owns_service_instance,
service_instance_started,
)
if cancel_event is not None and cancel_event.is_set():
return {"status": "cancelled", "removed": []}
if not current_process_owns_service_instance() or not service_instance_started(os.getpid()):
return {"status": "not_owner", "removed": []}
try:
confirm_pending_sqlite_backup_readiness()
result = expire_sqlite_migration_backups(
paths.get_sqlite_state_path(), cancel_event=cancel_event
)
if result["status"] == "unconfirmed":
logger.info("SQLite backup expiry deferred: no verified ready migration lifecycle")
return result
except Exception:
logger.warning("SQLite backup expiry deferred; retaining rollback files", exc_info=True)
return {"status": "deferred", "removed": []}
async def _join_trace_retention_future(self) -> None:
"""Wait for the worker's current batch to observe cooperative stop."""
future = getattr(self, "_trace_retention_future", None)
if future is None:
return
# The worker checks the event between transactions. Keep waiting even
# if the outer task receives another cancellation so shutdown never
# disposes the executor while SQLite work is still executing.
while not future.done():
try:
await asyncio.shield(future)
except asyncio.CancelledError:
continue
except Exception:
break
async def _agent_events_retention_loop(self) -> None:
"""Bounded retention for internal agent trace events (avibe#1506).
Checks immediately and then hourly; the storage marker inside
``agent_events_retention.run_once`` owns the once-per-day cadence, so
short-lived sessions still get their first pass on startup. Work runs
on a single-worker executor so shutdown can join it.
"""
import concurrent.futures
executor = concurrent.futures.ThreadPoolExecutor(
max_workers=1, thread_name_prefix="trace-retention"
)
cancel_event = threading.Event()
self._trace_retention_executor = executor
self._trace_retention_cancel_event = cancel_event
logger.info(
"Agent trace-event retention loop started (check interval=%ss)",
self._AGENT_EVENTS_RETENTION_CHECK_INTERVAL_SECONDS,
)
loop = asyncio.get_running_loop()
try:
while True:
try:
future = loop.run_in_executor(
executor,
self._run_agent_events_retention_pass,
cancel_event,
)
self._trace_retention_future = future
try:
summary = await asyncio.shield(future)
finally:
if self._trace_retention_future is future and future.done():
self._trace_retention_future = None
status = str((summary or {}).get("status") or "unknown")
if status not in {"not_due", "disabled"}:
logger.info(
"Agent trace-event retention %s: deleted=%s compaction=%s",
status,
(summary or {}).get("deleted_rows"),
((summary or {}).get("compaction") or {}).get("status"),
)
except asyncio.CancelledError:
cancel_event.set()
await self._join_trace_retention_future()
raise
except Exception:
logger.error("Agent trace-event retention failed", exc_info=True)
await asyncio.sleep(self._AGENT_EVENTS_RETENTION_CHECK_INTERVAL_SECONDS)
finally:
cancel_event.set()
await self._join_trace_retention_future()
try:
executor.shutdown(wait=True)
finally:
if self._trace_retention_executor is executor:
self._trace_retention_executor = None
if self._trace_retention_cancel_event is cancel_event:
self._trace_retention_cancel_event = None
if self._trace_retention_future is not None and self._trace_retention_future.done():
self._trace_retention_future = None
async def periodic_cleanup(self):
"""Sweep idle backend runtime state without interrupting active work.
Timeouts are read on every sweep, and the sweep interval does not depend
on them, so a saved change applies within one interval without a
backend restart.
"""
try:
while True:
await asyncio.sleep(_IDLE_SWEEP_INTERVAL_SECONDS)
claude_timeout, codex_timeout = self._get_idle_cleanup_timeouts()
try:
# A disabled backend's processes whose stop failed.
await self.agent_service.retry_pending()
except Exception as e:
logger.error("Teardown retry sweep failed: %s", e, exc_info=True)
# Retiring runtime generations stop once drained, whatever the
# idle timeouts say.
for agent in list(getattr(self.agent_service, "agents", {}).values()):
reap = getattr(agent, "reap_runtime_generations", None)
if callable(reap):
try:
await reap()
except Exception as e:
logger.error("Runtime generation sweep failed for %s: %s", agent.name, e, exc_info=True)
try:
# A client from before the latest renewal is reclaimed once
# idle, even with idle eviction off.
await self.session_handler.evict_idle_sessions(claude_timeout)
except Exception as e:
logger.error("Claude idle cleanup failed: %s", e, exc_info=True)
if claude_timeout > 0:
try:
# Defense-in-depth: reconcile live claude subprocesses
# against tracked sessions and reap orphans (no-owner /
# cross-restart) the idle-eviction path cannot see.
await self.session_handler.reap_orphaned_claude_sessions()
except Exception as e:
logger.error("Claude orphan reaper failed: %s", e, exc_info=True)
if codex_timeout > 0:
codex_agent = self.agent_service.agents.get("codex")
if codex_agent and hasattr(codex_agent, "evict_idle_transports"):
try:
await codex_agent.evict_idle_transports(codex_timeout)
except Exception as e:
logger.error("Codex idle cleanup failed: %s", e, exc_info=True)
except asyncio.CancelledError:
logger.info("Idle cleanup loop stopped.")
raise
def cleanup_sync(self):
"""Best-effort synchronous cleanup without cross-loop awaits"""
logger.info("Cleaning up controller resources (sync, best-effort)...")
def _stop_loop_coroutine(coro, label: str, *, timeout: float | None = 5) -> None:
try:
loop = self._loop
if not loop or loop.is_closed():
return
if loop.is_running():
future = asyncio.run_coroutine_threadsafe(coro, loop)
future.result(timeout=timeout)
return
loop.run_until_complete(coro)
except Exception as e:
logger.debug(f"{label} cleanup skipped: {e}")
mcp_runtime = getattr(self, "mcp_runtime", None)
if mcp_runtime is not None:
mcp_runtime.stop()
# Stop update checker
try:
update_task = self.update_checker.stop()
if update_task and not update_task.done():
loop = self._loop
if loop and not loop.is_running() and not loop.is_closed():
try:
loop.run_until_complete(self.update_checker.wait_stopped(update_task))
except Exception:
pass
except Exception as e:
logger.debug(f"Update checker cleanup skipped: {e}")
async def _cancel_cleanup_task() -> None:
if self.cleanup_task and not self.cleanup_task.done():
self.cleanup_task.cancel()
try:
await self.cleanup_task
except asyncio.CancelledError:
pass
self.cleanup_task = None
async def _cancel_vibey_model_supply_task() -> None:
task = getattr(self, "vibey_model_supply_task", None)
if task is not None and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
self.vibey_model_supply_task = None
async def _cancel_mcp_discovery_task() -> None:
task = getattr(self, "mcp_discovery_task", None)
if task is not None and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
self.mcp_discovery_task = None
async def _cancel_web_ui_watchdog_task() -> None:
task = self.web_ui_watchdog_task
if task and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
self.web_ui_watchdog_task = None
async def _cancel_trace_retention_task() -> None:
cancel_event = getattr(self, "_trace_retention_cancel_event", None)
if cancel_event is not None:
cancel_event.set()
if self.trace_retention_task and not self.trace_retention_task.done():
self.trace_retention_task.cancel()
try:
await self.trace_retention_task
except asyncio.CancelledError:
pass
self.trace_retention_task = None
# The loop normally joins this future in its finally block. Keep
# the fallback here for partial startup/shutdown states where the
# task was never fully scheduled.
await self._join_trace_retention_future()
executor = getattr(self, "_trace_retention_executor", None)
if executor is not None:
executor.shutdown(wait=True)
self._trace_retention_executor = None
self._trace_retention_cancel_event = None
self._trace_retention_future = None
async def _cancel_internal_server_task() -> None:
task = getattr(self, "_internal_server_task", None)
if task is not None and not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
self._internal_server_task = None
async def _close_computer_use_server() -> None:
js_kernels = getattr(self, "computer_js_kernels", None)
self.computer_js_kernels = None
if js_kernels is not None:
js_kernels.close_all()
script_interactions = getattr(
self,
"computer_script_interactions",
None,
)
self.computer_script_interactions = None
if script_interactions is not None:
script_interactions.close_all()
script_sessions = getattr(self, "computer_script_sessions", None)
self.computer_script_sessions = None
if script_sessions is not None:
script_sessions.close_all()
server = getattr(self, "computer_use_server", None)
self.computer_use_server = None
if server is not None:
await server.close()
# Without this the task is never settled, so the done callback that
# records "stopped" never runs and internal-server.json keeps saying
# "ready" after the service exits.
_stop_loop_coroutine(_cancel_internal_server_task(), "Internal dispatch server")
_stop_loop_coroutine(_close_computer_use_server(), "Computer Use server")
try:
from core import internal_server as _internal_server
_internal_server.note_stopped()
except Exception as e:
logger.debug(f"Internal dispatch server status write skipped: {e}")
_stop_loop_coroutine(_cancel_cleanup_task(), "Idle cleanup task")
_stop_loop_coroutine(_cancel_vibey_model_supply_task(), "Vibey model supply seed")
_stop_loop_coroutine(_cancel_mcp_discovery_task(), "MCP Hub discovery")
_stop_loop_coroutine(_cancel_web_ui_watchdog_task(), "Web UI watchdog task")
# Retention cancellation is cooperative at a delete-batch boundary;
# wait for that bounded join instead of abandoning the worker after
# the generic five-second cleanup timeout.
_stop_loop_coroutine(
_cancel_trace_retention_task(),
"Agent trace retention task",
timeout=None,
)
_stop_loop_coroutine(
self._join_runtime_work_stack_shutdown(),
"Runtime work stack",
)
_stop_loop_coroutine(self.runtime_command_watcher.stop(), "Runtime command watcher")
# Reconciliation, capture cancellation, accepted destructive work, and
# runtime close share one deadline so no stage can block service exit or
# starve the stages behind it.
model_hub_turn_gateway = getattr(self, "model_hub_turn_gateway", None)
if model_hub_turn_gateway is not None:
_stop_loop_coroutine(model_hub_turn_gateway.close(), "Model Hub turn gateway")
# Turns settled up to here queued their mode stamps on the stamp writer.
# Wait at most 2 s, then abandon what is left; the writer is a daemon
# thread, so even a stamp blocked on a locked database cannot hold exit.
from modules.agents.model_hub import drain_turn_mode_stamps
drain_turn_mode_stamps(timeout=2.0)
show_git_checkpoint_service = getattr(self, "show_git_checkpoint_service", None)
if show_git_checkpoint_service is not None:
show_git_checkpoint_service.stop()
# Vibey's recovery retries end with the service; the next start recovers again.
vibey_recovery = getattr(self, "vibey_recovery", None)
if vibey_recovery is not None:
_stop_loop_coroutine(vibey_recovery.stop(), "Vibey recovery")
try:
codex_agent = self.agent_service.agents.get("codex")
if codex_agent and hasattr(codex_agent, "shutdown_runtime"):
_stop_loop_coroutine(codex_agent.shutdown_runtime(), "Codex runtime")
except Exception as e:
logger.debug(f"Codex runtime cleanup skipped: {e}")
try:
# A disabled backend whose stop failed has no other owner: its
# agent is out of routing and the idle sweep that retries it is
# already cancelled, so shutdown makes the last attempt.
_stop_loop_coroutine(self.agent_service.retry_pending(), "Pending backend teardowns")
except Exception as e:
logger.debug(f"Pending backend teardowns skipped: {e}")
# Cancel receiver tasks without awaiting (they may belong to other loops)
try:
for session_id, task in list(self.receiver_tasks.items()):
if not task.done():
task.cancel()
# Remove from registry regardless
del self.receiver_tasks[session_id]
except Exception as e:
logger.debug(f"Receiver tasks cleanup skipped due to: {e}")
# Do not attempt to await SessionHandler cleanup here to avoid cross-loop issues.
# Active connections will be closed by process exit; mappings are persisted separately.
# Attempt to call stop if it's a plain function; skip if coroutine to avoid cross-loop awaits
try:
stop_attr = getattr(self.im_client, "stop", None)
if callable(stop_attr):
import inspect
if not inspect.iscoroutinefunction(stop_attr):
stop_attr()
except Exception as e:
logger.warning("Failed to stop IM client: %s", e)
# Best-effort async shutdown for IM clients
try:
shutdown_attr = getattr(self.im_client, "shutdown", None)
if callable(shutdown_attr):
import inspect
if inspect.iscoroutinefunction(shutdown_attr):
loop = self._loop
if loop and loop.is_running():
try:
future = asyncio.run_coroutine_threadsafe(shutdown_attr(), loop)
future.result(timeout=5)
except Exception:
pass
else:
shutdown_attr()
except Exception as e:
logger.warning("Failed to shutdown IM client: %s", e)
if self._im_thread and self._im_thread.is_alive():
self._im_thread.join(timeout=5)
self._im_thread = None
# An explicit Avibe stop/restart must not leave any OpenCode generation
# of this controller behind; another desktop Runtime's are not ours.
# Active turns have already reached shutdown cleanup.
try:
from modules.agents.opencode.server import stop_owned_generations_sync
stop_owned_generations_sync()
except Exception as e:
logger.warning("OpenCode server cleanup failed: %s", e)
loop_writers.log_summary(final=True)
logger.info("Controller cleanup (sync) complete")
def _model_hub_snapshot_refresh_completed(self) -> None:
"""Move a worker completion onto the controller's event loop."""
if getattr(self, "_shutdown_requested", False) or getattr(
self,
"_model_hub_snapshot_reconcile_stopping",
False,
):
return
pending = getattr(self, "_model_hub_snapshot_refresh_pending", None)
if pending is None:
return
pending.set()
loop = getattr(self, "_loop", None)
if loop is None or loop.is_closed() or not loop.is_running():
return
loop.call_soon_threadsafe(self._schedule_model_hub_snapshot_reconcile)
def _schedule_model_hub_snapshot_reconcile(self) -> None:
if getattr(self, "_shutdown_requested", False) or getattr(
self,
"_model_hub_snapshot_reconcile_stopping",
False,
):
return
pending = getattr(self, "_model_hub_snapshot_refresh_pending", None)
service = getattr(self, "model_hub_service", None)
if pending is None or not pending.is_set() or service is None:
return
task = getattr(self, "_model_hub_snapshot_reconcile_task", None)
if task is not None and not task.done():
return
pending.clear()
async def reconcile() -> None:
try:
try:
await service.reconcile_builtin_models()
except Exception:
logger.warning(
"Model Hub built-in reconciliation failed after snapshot refresh",
exc_info=True,
)
try:
await service.backfill_models_dev_limits()
except Exception:
logger.warning(
"Model Hub models.dev limit backfill failed after snapshot refresh",
exc_info=True,
)
try:
await self._start_vibey_without_a_model()
except Exception:
logger.warning(
"Vibey's starting model failed after snapshot refresh",
exc_info=True,
)
finally:
self._model_hub_snapshot_reconcile_task = None
if (
pending.is_set()
and not getattr(self, "_shutdown_requested", False)
and not getattr(
self,
"_model_hub_snapshot_reconcile_stopping",
False,
)
):
self._schedule_model_hub_snapshot_reconcile()
loop = getattr(self, "_loop", None)
if loop is None:
loop = asyncio.get_running_loop()
self._model_hub_snapshot_reconcile_task = loop.create_task(
reconcile(),
name="model-hub-snapshot-refresh-reconcile",
)
async def _model_hub_snapshot_reconcile_loop(self) -> None:
"""Re-read cross-process snapshot inputs on the controller cadence."""
try:
while True:
interval = max(
0.01,
float(
getattr(
self,
"_model_hub_snapshot_reconcile_interval_seconds",
_MODEL_HUB_SNAPSHOT_RECONCILE_INTERVAL_SECONDS,
)
),
)
await asyncio.sleep(interval)
if getattr(self, "_shutdown_requested", False) or getattr(
self,
"_model_hub_snapshot_reconcile_stopping",
False,
):
return
pending = getattr(
self,
"_model_hub_snapshot_refresh_pending",
None,
)
if pending is None:
return
pending.set()
self._schedule_model_hub_snapshot_reconcile()
finally:
if getattr(
self,
"_model_hub_snapshot_reconcile_loop_task",
None,
) is asyncio.current_task():
self._model_hub_snapshot_reconcile_loop_task = None
def _start_model_hub_snapshot_reconcile_loop(self) -> None:
if (
getattr(self, "model_hub_service", None) is None
or getattr(self, "_shutdown_requested", False)
or getattr(self, "_model_hub_snapshot_reconcile_stopping", False)
):
return
task = getattr(self, "_model_hub_snapshot_reconcile_loop_task", None)
if task is not None and not task.done():
return
self._model_hub_snapshot_reconcile_loop_task = asyncio.create_task(
self._model_hub_snapshot_reconcile_loop(),
name="model-hub-snapshot-reconcile-loop",
)
async def _stop_model_hub_snapshot_reconciliation(self) -> None:
"""Quiesce snapshot tasks before the Model Hub service is stopped."""
self._model_hub_snapshot_reconcile_stopping = True
pending = getattr(self, "_model_hub_snapshot_refresh_pending", None)
if pending is not None:
pending.clear()
loop_task = getattr(
self,
"_model_hub_snapshot_reconcile_loop_task",
None,
)
if loop_task is not None and not loop_task.done():
loop_task.cancel()
await asyncio.gather(loop_task, return_exceptions=True)
self._model_hub_snapshot_reconcile_loop_task = None
reconcile_task = getattr(
self,
"_model_hub_snapshot_reconcile_task",
None,
)
if reconcile_task is not None and not reconcile_task.done():
await asyncio.gather(reconcile_task, return_exceptions=True)
self._model_hub_snapshot_reconcile_task = None
if pending is not None:
pending.clear()
def _init_model_hub(self) -> None:
"""Own the CPA dependency and the default-on Model Hub aggregate."""
from config.v2_config import V2Config, is_model_hub_enabled
from vibe.model_hub_runtime import get_model_hub_engine_adapter
self.model_hub_service = None
self.model_hub_turn_gateway = None
self.model_hub_runtime = None
self.model_hub_engine_adapter = get_model_hub_engine_adapter()
self._model_hub_snapshot_refresh_pending = threading.Event()
self._model_hub_snapshot_reconcile_task = None
self._model_hub_snapshot_reconcile_loop_task = None
self._model_hub_snapshot_reconcile_stopping = False
if not is_model_hub_enabled():
return
# The controller is the single Model Hub aggregate and engine owner.
# The UI process reaches this instance through the internal Unix socket.
from core.handlers.model_hub import create_default_service
from core.handlers.model_hub.turn_gateway import ModelHubTurnGateway
from modules.agents.model_hub import ModelHubRuntimeRouter
from vibe.api import resolve_cli_paths
from vibe.backend_model_catalog import set_remote_catalog_refresh_completed
from vibe.models_dev_catalog import set_models_dev_copy_arrived
def default_vibe_agent_model(backend: str) -> Optional[str]:
agent = self.vibe_agent_store.get_default_agent()
if agent is None or agent.backend != backend:
return None
return agent.model
def default_vibe_agent_name(backend: str) -> Optional[str]:
agent = self.vibe_agent_store.get_default_agent()
if agent is None or agent.backend != backend or not str(agent.model or "").strip():
return None
return agent.name
def named_vibe_agents(backend: str) -> list[tuple[str, Optional[str]]]:
return [
(agent.name, agent.model)
for agent in self.vibe_agent_store.list_agents(include_disabled=False)
if agent.backend == backend
]
cli_presence: dict[str, bool] = {}
cli_presence_lock = threading.Lock()
cli_presence_generation: dict[str, int] = {}
next_cli_presence_generation = 0
def cli_present(backend: str) -> bool:
# Payload assembly runs on the controller loop. Read only the last
# complete worker-produced snapshot here.
return cli_presence.get(backend, False)
def refresh_cli_presence(
include_npm_global: bool,
backends: tuple[str, ...] | None = None,
) -> None:
nonlocal cli_presence, next_cli_presence_generation
selected_backends = backends or tuple(sorted(NATIVE_CLI_BACKENDS))
with cli_presence_lock:
next_cli_presence_generation += 1
generation = next_cli_presence_generation
for backend in selected_backends:
cli_presence_generation[backend] = generation
try:
v2_config = V2Config.load()
except FileNotFoundError:
v2_config = None
except Exception:
logger.warning("Model Hub CLI config probe failed", exc_info=True)
v2_config = None
configured_paths: dict[str, str] = {}
for backend in selected_backends:
backend_config = getattr(getattr(v2_config, "agents", None), backend, None)
configured_paths[backend] = str(
getattr(backend_config, "cli_path", None) or backend
)
try:
resolved_paths = resolve_cli_paths(
list(configured_paths.values()),
include_npm_global=include_npm_global,
)
except Exception:
logger.warning("Model Hub CLI presence probe failed", exc_info=True)
return
refreshed = {
backend: resolved_paths.get(configured_path) is not None
for backend, configured_path in configured_paths.items()
}
with cli_presence_lock:
cli_presence = {
**cli_presence,
**{
backend: present
for backend, present in refreshed.items()
if cli_presence_generation.get(backend) == generation
},
}
# Seed only filesystem and PATH facts before the internal RPC surface
# exists. The page publishes npm-only installs through an explicit
# post-paint refresh, so controller readiness never waits on npm.
refresh_cli_presence(False, None)
async def repair_model_selections(addresses: frozenset[str]) -> int:
# A Vibe Agent's model, a channel's routing override, and a
# session's pin are each a copy of an id the Model Hub menu once
# offered. The hub proves which addresses are addresses; this
# process owns the rows that copied them.
from storage.model_selection_addresses import (
remove_credential_addresses_from_selections,
)
return await asyncio.to_thread(
remove_credential_addresses_from_selections, addresses
)
self.model_hub_service = create_default_service(
adapter=self.model_hub_engine_adapter,
requested_model_override=default_vibe_agent_model,
selected_agent_override=default_vibe_agent_name,
named_agents_override=named_vibe_agents,
builtin_agent_models_override=self._builtin_agent_models,
cli_present_override=cli_present,
cli_presence_refresh=refresh_cli_presence,
backend_catalog_changed=self._model_hub_catalog_changed,
repair_model_selections=repair_model_selections,
)
set_remote_catalog_refresh_completed(
self._model_hub_snapshot_refresh_completed
)
# A models.dev copy is another snapshot input: rows written before one
# was cached take their limits from it (backfill_models_dev_limits).
set_models_dev_copy_arrived(self._model_hub_snapshot_refresh_completed)
try:
asyncio.run(
self.model_hub_service.reconcile_builtin_models(notify=False)
)
except Exception:
logger.warning(
"Model Hub built-in reconciliation failed during startup",
exc_info=True,
)
try:
asyncio.run(self.model_hub_service.reconcile_source_reasoning_tiers())
except Exception:
logger.warning(
"Model Hub could not derive saved Source tiers again during startup",
exc_info=True,
)
self.model_hub_turn_gateway = ModelHubTurnGateway(
self.model_hub_service,
language_provider=lambda: self.config.language,
)
from core.model_hub_progress import publish_recovery_changed
self.model_hub_turn_gateway.correlation.on_recovery_changed = (
lambda turn_id: publish_recovery_changed(self, turn_id)
)
self.model_hub_runtime = ModelHubRuntimeRouter(
service=self.model_hub_service,
turn_gateway=self.model_hub_turn_gateway,
)