Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(activity): end turns left running by a host that went away

raullopez-sandbox4a10a550c6ff
diff --git a/core/switch_core/db/stores/session_activity_store.py b/core/switch_core/db/stores/session_activity_store.py
index c365ac31d..534138631 100644
--- a/core/switch_core/db/stores/session_activity_store.py
+++ b/core/switch_core/db/stores/session_activity_store.py
@@ -6,13 +6,20 @@ transaction, and the bound tenant scopes every read through row-level security.
from __future__ import annotations
-from datetime import datetime
+from datetime import datetime, timedelta
from typing import TYPE_CHECKING
-from sqlalchemy import delete, func, or_, select, update
+from sqlalchemy import delete, exists, func, literal, or_, select, update
from sqlalchemy.dialects.postgresql import insert
-from switch_core.db.models import Agent, AgentSessionActivityItem, ApprovalRequest
+from switch_core.db.models import (
+ Agent,
+ AgentConnectionRecord,
+ AgentSession,
+ AgentSessionActivityItem,
+ ApprovalRequest,
+ SwitchCoreProcess,
+)
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession
@@ -138,6 +145,63 @@ class AgentSessionActivityStore:
)
return status
+ async def end_orphaned_turns(
+ self,
+ session: AsyncSession,
+ *,
+ quiet_for: timedelta,
+ lease_ttl: timedelta,
+ heartbeat_ttl: timedelta,
+ text: str,
+ ) -> list[tuple[str, str, str]]:
+ """End, as interrupted, the turns still queued or running that no
+ host can finish: nothing in the turn has moved for `quiet_for`, and
+ its agent has no live connection (open, on a process holding its
+ lease) and no fresh heartbeat. The revision moves on, so the change is
+ announced like a host's. Returns (agent, session, turn) of each."""
+ other = AgentSessionActivityItem.__table__.alias("other")
+ turn = AgentSessionActivityItem
+ quiet = ~exists().where(
+ other.c.tenant_id == turn.tenant_id,
+ other.c.agent_id == turn.agent_id,
+ other.c.session_id == turn.session_id,
+ other.c.turn_id == turn.turn_id,
+ other.c.updated_at > func.now() - literal(quiet_for),
+ )
+ connected = exists().where(
+ AgentConnectionRecord.agent_id == turn.agent_id,
+ AgentConnectionRecord.closed_at.is_(None),
+ SwitchCoreProcess.id == AgentConnectionRecord.process_id,
+ SwitchCoreProcess.stopped_at.is_(None),
+ SwitchCoreProcess.beat_at > func.now() - literal(lease_ttl),
+ )
+ beating = exists().where(
+ AgentSession.agent_id == turn.agent_id,
+ AgentSession.lifecycle == "heartbeat",
+ AgentSession.last_seen_at > func.now() - literal(heartbeat_ttl),
+ )
+ result = await session.execute(
+ update(turn)
+ .where(
+ turn.item_id == TURN_ITEM_ID,
+ turn.status.in_(("queued", "running")),
+ quiet,
+ ~connected,
+ ~beating,
+ )
+ .values(
+ status="interrupted",
+ text=text,
+ revision=turn.revision + 1,
+ updated_at=func.now(),
+ )
+ .returning(turn.agent_id, turn.session_id, turn.turn_id)
+ )
+ return [
+ (agent_id, session_id, turn_id)
+ for agent_id, session_id, turn_id in result.all()
+ ]
+
async def upsert(
self, session: AsyncSession, item: AgentSessionActivityItem
) -> bool:
diff --git a/core/switch_core/session_activity/maintenance.py b/core/switch_core/session_activity/maintenance.py
index c34518b19..50068aa40 100644
--- a/core/switch_core/session_activity/maintenance.py
+++ b/core/switch_core/session_activity/maintenance.py
@@ -25,6 +25,7 @@ logger = logging.getLogger(__name__)
EXPIRY_INTERVAL_SECONDS = 5.0
PRUNE_EVERY_SECONDS = 3600.0
+ORPHANS_EVERY_SECONDS = 60.0
ACTIVITY_RETENTION = timedelta(days=7)
@@ -52,12 +53,39 @@ async def maintain_once(
)
+async def end_orphaned_turns_once(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+ """One pass over every tenant: end the turns no host is left to finish."""
+ service = AgentSessionActivityService(session_factory)
+ for tenant_id in await all_tenant_ids(session_factory):
+ with tenant_scope(tenant_id):
+ try:
+ ended = await service.end_orphaned_turns()
+ if ended:
+ logger.warning(
+ "Ended %d turn(s) in tenant %s whose agent went away "
+ "before reporting their end",
+ len(ended),
+ tenant_id,
+ )
+ except Exception:
+ logger.exception(
+ "Ending orphaned turns failed for tenant %s", tenant_id
+ )
+
+
async def maintenance_loop(session_factory: async_sessionmaker[AsyncSession]) -> None:
- """Session-activity upkeep every 5 s."""
+ """Session-activity upkeep every 5 s; orphaned turns every minute."""
with no_tenant():
since_prune = PRUNE_EVERY_SECONDS
+ since_orphans = 0.0
while True:
await asyncio.sleep(EXPIRY_INTERVAL_SECONDS)
+ since_orphans += EXPIRY_INTERVAL_SECONDS
+ if since_orphans >= ORPHANS_EVERY_SECONDS:
+ since_orphans = 0.0
+ await end_orphaned_turns_once(session_factory)
since_prune += EXPIRY_INTERVAL_SECONDS
prune = since_prune >= PRUNE_EVERY_SECONDS
try:
diff --git a/core/switch_core/session_activity/service.py b/core/switch_core/session_activity/service.py
index 992db9a51..ce57dea8e 100644
--- a/core/switch_core/session_activity/service.py
+++ b/core/switch_core/session_activity/service.py
@@ -30,6 +30,7 @@ from switch_core.db.models import (
require_tenant_id,
)
from switch_core.db.session_scope import tenant_session
+from switch_core.db.stores.agent_session_store import AgentSessionStore
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.external_user_store import ExternalUserStore
@@ -42,6 +43,7 @@ from switch_core.db.stores.session_activity_store import (
)
from switch_core.db.stores.usage_store import UsageStore
from switch_core.delivery.addressing import AddressingResolver
+from switch_core.process_lease import LEASE_TTL
from switch_core.sessions.contract import (
TURN_ENDED,
ApprovalResult,
@@ -71,6 +73,14 @@ _OPTION_DESCRIPTION_CHARS = 1000
# outlives a working day is no longer the question anyone is looking at.
MAX_APPROVAL_LIFETIME = timedelta(hours=24)
NOTICE_PREFIX = "notice:"
+# A turn left queued or running by a host that went away (it crashed, or its
+# provider failed before it reported the turn's end) is ended once its agent is
+# offline and nothing in it has moved for this long, so no one is shown it as
+# still working. A live session's turn is never ended here.
+ORPHANED_TURN_QUIET = timedelta(minutes=10)
+ORPHANED_TURN_TEXT = (
+ "Ended by Switch: the agent's session went away before this turn finished."
+)
ITEM_STATUSES: dict[str, frozenset[str]] = {
"turn": frozenset({"queued", "running", "completed", "interrupted", "error"}),
@@ -549,6 +559,22 @@ class AgentSessionActivityService:
row.delivered_at = datetime.now(UTC)
return row
+ async def end_orphaned_turns(self) -> list[tuple[str, str, str]]:
+ """End the bound tenant's turns whose agent is gone and that nothing
+ has moved for `ORPHANED_TURN_QUIET` (`end_orphaned_turns` on the
+ store). Run on a timer."""
+ async with (
+ tenant_session(self._sessions, require_tenant_id()) as db,
+ db.begin(),
+ ):
+ return await self._activity.end_orphaned_turns(
+ db,
+ quiet_for=ORPHANED_TURN_QUIET,
+ lease_ttl=LEASE_TTL,
+ heartbeat_ttl=AgentSessionStore.ALWAYS_ON_TTL,
+ text=ORPHANED_TURN_TEXT,
+ )
+
async def expire_due(self) -> list[ApprovalRequest]:
"""Expire the bound tenant's overdue requests. Run on a timer."""
async with (
diff --git a/core/tests/switch_core/session_activity/test_orphaned_turns.py b/core/tests/switch_core/session_activity/test_orphaned_turns.py
new file mode 100644
index 000000000..7879cd09d
--- /dev/null
+++ b/core/tests/switch_core/session_activity/test_orphaned_turns.py
@@ -0,0 +1,93 @@
+"""A turn left queued or running by a host that went away is ended once its
+agent is offline and nothing in it has moved for a while; a turn of an agent
+that is still connected, or still moving, is left alone."""
+
+from __future__ import annotations
+
+import uuid
+from datetime import UTC, datetime, timedelta
+
+from sqlalchemy import update
+
+from switch_core.db.models import (
+ AgentConnectionRecord,
+ AgentSessionActivityItem,
+ SwitchCoreProcess,
+)
+from switch_core.session_activity.maintenance import end_orphaned_turns_once
+from switch_core.session_activity.service import ORPHANED_TURN_TEXT
+
+from .conftest import AGENT
+
+
+async def _turn(service, session_id: str, status: str) -> None:
+ await service.report_item(
+ AGENT,
+ session_id,
+ turn_id="turn-1",
+ item_id="turn",
+ kind="turn",
+ revision=3,
+ status=status,
+ title="Working",
+ text="",
+ command_id=None,
+ room_id=None,
+ thread_id=None,
+ message_id=None,
+ occurred_at=datetime.now(UTC),
+ usage=[],
+ )
+
+
+async def _quiet(session_factory, session_id: str) -> None:
+ async with session_factory() as db, db.begin():
+ await db.execute(
+ update(AgentSessionActivityItem)
+ .where(AgentSessionActivityItem.session_id == session_id)
+ .values(updated_at=datetime.now(UTC) - timedelta(minutes=30))
+ )
+
+
+async def _status(service, session_id: str) -> tuple[str, int, str]:
+ [row] = await service.turn_items(AGENT, session_id, "turn-1")
+ return row.status, row.revision, row.text
+
+
+async def test_an_offline_agents_quiet_turn_is_ended(service, session_factory) -> None:
+ await _turn(service, "gone", "running")
+ await _turn(service, "moving", "running")
+ await _turn(service, "done", "completed")
+ await _quiet(session_factory, "gone")
+ await _quiet(session_factory, "done")
+
+ await end_orphaned_turns_once(session_factory)
+ assert await _status(service, "gone") == ("interrupted", 4, ORPHANED_TURN_TEXT)
+ assert (await _status(service, "moving"))[0] == "running"
+ assert (await _status(service, "done"))[0] == "completed"
+
+
+async def test_a_connected_agents_turn_is_left_running(
+ service, session_factory
+) -> None:
+ await _turn(service, "live", "running")
+ await _quiet(session_factory, "live")
+ process_id = f"proc-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as db, db.begin():
+ db.add(SwitchCoreProcess(id=process_id))
+ await db.flush()
+ db.add(
+ AgentConnectionRecord(
+ id=f"conn-{uuid.uuid4().hex[:8]}",
+ agent_id=AGENT,
+ process_id=process_id,
+ incarnation=1,
+ scope="all",
+ delivery_filter="addressed",
+ spawn_capable=False,
+ transport="sse",
+ )
+ )
+
+ await end_orphaned_turns_once(session_factory)
+ assert (await _status(service, "live"))[0] == "running"