Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(activity): end turns left running by a host that went away
raullopez-sandbox4a10a550c6ff
diff --git a/core/switch_core/db/stores/session_activity_store.py b/core/switch_core/db/stores/session_activity_store.pyindex c365ac31d..534138631 100644--- a/core/switch_core/db/stores/session_activity_store.py+++ b/core/switch_core/db/stores/session_activity_store.py@@ -6,13 +6,20 @@ transaction, and the bound tenant scopes every read through row-level security.from __future__ import annotations-from datetime import datetime+from datetime import datetime, timedeltafrom typing import TYPE_CHECKING-from sqlalchemy import delete, func, or_, select, update+from sqlalchemy import delete, exists, func, literal, or_, select, updatefrom sqlalchemy.dialects.postgresql import insert-from switch_core.db.models import Agent, AgentSessionActivityItem, ApprovalRequest+from switch_core.db.models import (+ Agent,+ AgentConnectionRecord,+ AgentSession,+ AgentSessionActivityItem,+ ApprovalRequest,+ SwitchCoreProcess,+)if TYPE_CHECKING:from sqlalchemy.ext.asyncio import AsyncSession@@ -138,6 +145,63 @@ class AgentSessionActivityStore:)return status+ async def end_orphaned_turns(+ self,+ session: AsyncSession,+ *,+ quiet_for: timedelta,+ lease_ttl: timedelta,+ heartbeat_ttl: timedelta,+ text: str,+ ) -> list[tuple[str, str, str]]:+ """End, as interrupted, the turns still queued or running that no+ host can finish: nothing in the turn has moved for `quiet_for`, and+ its agent has no live connection (open, on a process holding its+ lease) and no fresh heartbeat. The revision moves on, so the change is+ announced like a host's. Returns (agent, session, turn) of each."""+ other = AgentSessionActivityItem.__table__.alias("other")+ turn = AgentSessionActivityItem+ quiet = ~exists().where(+ other.c.tenant_id == turn.tenant_id,+ other.c.agent_id == turn.agent_id,+ other.c.session_id == turn.session_id,+ other.c.turn_id == turn.turn_id,+ other.c.updated_at > func.now() - literal(quiet_for),+ )+ connected = exists().where(+ AgentConnectionRecord.agent_id == turn.agent_id,+ AgentConnectionRecord.closed_at.is_(None),+ SwitchCoreProcess.id == AgentConnectionRecord.process_id,+ SwitchCoreProcess.stopped_at.is_(None),+ SwitchCoreProcess.beat_at > func.now() - literal(lease_ttl),+ )+ beating = exists().where(+ AgentSession.agent_id == turn.agent_id,+ AgentSession.lifecycle == "heartbeat",+ AgentSession.last_seen_at > func.now() - literal(heartbeat_ttl),+ )+ result = await session.execute(+ update(turn)+ .where(+ turn.item_id == TURN_ITEM_ID,+ turn.status.in_(("queued", "running")),+ quiet,+ ~connected,+ ~beating,+ )+ .values(+ status="interrupted",+ text=text,+ revision=turn.revision + 1,+ updated_at=func.now(),+ )+ .returning(turn.agent_id, turn.session_id, turn.turn_id)+ )+ return [+ (agent_id, session_id, turn_id)+ for agent_id, session_id, turn_id in result.all()+ ]+async def upsert(self, session: AsyncSession, item: AgentSessionActivityItem) -> bool:diff --git a/core/switch_core/session_activity/maintenance.py b/core/switch_core/session_activity/maintenance.pyindex c34518b19..50068aa40 100644--- a/core/switch_core/session_activity/maintenance.py+++ b/core/switch_core/session_activity/maintenance.py@@ -25,6 +25,7 @@ logger = logging.getLogger(__name__)EXPIRY_INTERVAL_SECONDS = 5.0PRUNE_EVERY_SECONDS = 3600.0+ORPHANS_EVERY_SECONDS = 60.0ACTIVITY_RETENTION = timedelta(days=7)@@ -52,12 +53,39 @@ async def maintain_once()+async def end_orphaned_turns_once(+ session_factory: async_sessionmaker[AsyncSession],+) -> None:+ """One pass over every tenant: end the turns no host is left to finish."""+ service = AgentSessionActivityService(session_factory)+ for tenant_id in await all_tenant_ids(session_factory):+ with tenant_scope(tenant_id):+ try:+ ended = await service.end_orphaned_turns()+ if ended:+ logger.warning(+ "Ended %d turn(s) in tenant %s whose agent went away "+ "before reporting their end",+ len(ended),+ tenant_id,+ )+ except Exception:+ logger.exception(+ "Ending orphaned turns failed for tenant %s", tenant_id+ )++async def maintenance_loop(session_factory: async_sessionmaker[AsyncSession]) -> None:- """Session-activity upkeep every 5 s."""+ """Session-activity upkeep every 5 s; orphaned turns every minute."""with no_tenant():since_prune = PRUNE_EVERY_SECONDS+ since_orphans = 0.0while True:await asyncio.sleep(EXPIRY_INTERVAL_SECONDS)+ since_orphans += EXPIRY_INTERVAL_SECONDS+ if since_orphans >= ORPHANS_EVERY_SECONDS:+ since_orphans = 0.0+ await end_orphaned_turns_once(session_factory)since_prune += EXPIRY_INTERVAL_SECONDSprune = since_prune >= PRUNE_EVERY_SECONDStry:diff --git a/core/switch_core/session_activity/service.py b/core/switch_core/session_activity/service.pyindex 992db9a51..ce57dea8e 100644--- a/core/switch_core/session_activity/service.py+++ b/core/switch_core/session_activity/service.py@@ -30,6 +30,7 @@ from switch_core.db.models import (require_tenant_id,)from switch_core.db.session_scope import tenant_session+from switch_core.db.stores.agent_session_store import AgentSessionStorefrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.client_store import ClientStorefrom switch_core.db.stores.external_user_store import ExternalUserStore@@ -42,6 +43,7 @@ from switch_core.db.stores.session_activity_store import ()from switch_core.db.stores.usage_store import UsageStorefrom switch_core.delivery.addressing import AddressingResolver+from switch_core.process_lease import LEASE_TTLfrom switch_core.sessions.contract import (TURN_ENDED,ApprovalResult,@@ -71,6 +73,14 @@ _OPTION_DESCRIPTION_CHARS = 1000# outlives a working day is no longer the question anyone is looking at.MAX_APPROVAL_LIFETIME = timedelta(hours=24)NOTICE_PREFIX = "notice:"+# A turn left queued or running by a host that went away (it crashed, or its+# provider failed before it reported the turn's end) is ended once its agent is+# offline and nothing in it has moved for this long, so no one is shown it as+# still working. A live session's turn is never ended here.+ORPHANED_TURN_QUIET = timedelta(minutes=10)+ORPHANED_TURN_TEXT = (+ "Ended by Switch: the agent's session went away before this turn finished."+)ITEM_STATUSES: dict[str, frozenset[str]] = {"turn": frozenset({"queued", "running", "completed", "interrupted", "error"}),@@ -549,6 +559,22 @@ class AgentSessionActivityService:row.delivered_at = datetime.now(UTC)return row+ async def end_orphaned_turns(self) -> list[tuple[str, str, str]]:+ """End the bound tenant's turns whose agent is gone and that nothing+ has moved for `ORPHANED_TURN_QUIET` (`end_orphaned_turns` on the+ store). Run on a timer."""+ async with (+ tenant_session(self._sessions, require_tenant_id()) as db,+ db.begin(),+ ):+ return await self._activity.end_orphaned_turns(+ db,+ quiet_for=ORPHANED_TURN_QUIET,+ lease_ttl=LEASE_TTL,+ heartbeat_ttl=AgentSessionStore.ALWAYS_ON_TTL,+ text=ORPHANED_TURN_TEXT,+ )+async def expire_due(self) -> list[ApprovalRequest]:"""Expire the bound tenant's overdue requests. Run on a timer."""async with (diff --git a/core/tests/switch_core/session_activity/test_orphaned_turns.py b/core/tests/switch_core/session_activity/test_orphaned_turns.pynew file mode 100644index 000000000..7879cd09d--- /dev/null+++ b/core/tests/switch_core/session_activity/test_orphaned_turns.py@@ -0,0 +1,93 @@+"""A turn left queued or running by a host that went away is ended once its+agent is offline and nothing in it has moved for a while; a turn of an agent+that is still connected, or still moving, is left alone."""++from __future__ import annotations++import uuid+from datetime import UTC, datetime, timedelta++from sqlalchemy import update++from switch_core.db.models import (+ AgentConnectionRecord,+ AgentSessionActivityItem,+ SwitchCoreProcess,+)+from switch_core.session_activity.maintenance import end_orphaned_turns_once+from switch_core.session_activity.service import ORPHANED_TURN_TEXT++from .conftest import AGENT+++async def _turn(service, session_id: str, status: str) -> None:+ await service.report_item(+ AGENT,+ session_id,+ turn_id="turn-1",+ item_id="turn",+ kind="turn",+ revision=3,+ status=status,+ title="Working",+ text="",+ command_id=None,+ room_id=None,+ thread_id=None,+ message_id=None,+ occurred_at=datetime.now(UTC),+ usage=[],+ )+++async def _quiet(session_factory, session_id: str) -> None:+ async with session_factory() as db, db.begin():+ await db.execute(+ update(AgentSessionActivityItem)+ .where(AgentSessionActivityItem.session_id == session_id)+ .values(updated_at=datetime.now(UTC) - timedelta(minutes=30))+ )+++async def _status(service, session_id: str) -> tuple[str, int, str]:+ [row] = await service.turn_items(AGENT, session_id, "turn-1")+ return row.status, row.revision, row.text+++async def test_an_offline_agents_quiet_turn_is_ended(service, session_factory) -> None:+ await _turn(service, "gone", "running")+ await _turn(service, "moving", "running")+ await _turn(service, "done", "completed")+ await _quiet(session_factory, "gone")+ await _quiet(session_factory, "done")++ await end_orphaned_turns_once(session_factory)+ assert await _status(service, "gone") == ("interrupted", 4, ORPHANED_TURN_TEXT)+ assert (await _status(service, "moving"))[0] == "running"+ assert (await _status(service, "done"))[0] == "completed"+++async def test_a_connected_agents_turn_is_left_running(+ service, session_factory+) -> None:+ await _turn(service, "live", "running")+ await _quiet(session_factory, "live")+ process_id = f"proc-{uuid.uuid4().hex[:8]}"+ async with session_factory() as db, db.begin():+ db.add(SwitchCoreProcess(id=process_id))+ await db.flush()+ db.add(+ AgentConnectionRecord(+ id=f"conn-{uuid.uuid4().hex[:8]}",+ agent_id=AGENT,+ process_id=process_id,+ incarnation=1,+ scope="all",+ delivery_filter="addressed",+ spawn_capable=False,+ transport="sse",+ )+ )++ await end_orphaned_turns_once(session_factory)+ assert (await _status(service, "live"))[0] == "running"