Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(activity): end a turn when its session is gone, not only its agent

raullopez-sandboxf81179635199
diff --git a/core/switch_core/db/stores/session_activity_store.py b/core/switch_core/db/stores/session_activity_store.py
index 39bfb53a1..d63b5f612 100644
--- a/core/switch_core/db/stores/session_activity_store.py
+++ b/core/switch_core/db/stores/session_activity_store.py
@@ -7,9 +7,19 @@ transaction, and the bound tenant scopes every read through row-level security.
from __future__ import annotations
from datetime import datetime, timedelta
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
-from sqlalchemy import delete, exists, func, literal, or_, select, update
+from sqlalchemy import (
+ ColumnElement,
+ and_,
+ delete,
+ exists,
+ func,
+ literal,
+ or_,
+ select,
+ update,
+)
from sqlalchemy.dialects.postgresql import insert
from switch_core.db.models import (
@@ -17,6 +27,7 @@ from switch_core.db.models import (
AgentConnectionRecord,
AgentSession,
AgentSessionActivityItem,
+ AgentSessionPlacement,
ApprovalRequest,
SwitchCoreProcess,
)
@@ -151,31 +162,63 @@ class AgentSessionActivityStore:
self,
session: AsyncSession,
*,
- quiet_for: timedelta,
+ offline_quiet: timedelta,
+ session_quiet: timedelta,
lease_ttl: timedelta,
heartbeat_ttl: timedelta,
text: str,
) -> list[tuple[str, str, str]]:
- """End, as interrupted, the turns still queued or running that no
- host can finish: nothing in the turn has moved for `quiet_for`, and
- its agent has no live connection (open, on a process holding its
- lease) and no fresh heartbeat. The revision moves on, so the change is
- announced like a host's. Returns (agent, session, turn) of each."""
+ """End, as interrupted, the turns still queued or running that their
+ session can no longer finish, keyed on the session rather than the
+ agent:
+
+ - its session is live when it is placed (`AgentSessionPlacement`) over
+ a connection whose process holds its lease, or when a heartbeat for
+ the turn's room is fresh. A live session's turn is never ended;
+ - otherwise the turn is ended once nothing in it has moved for
+ `offline_quiet` if the agent has no live connection and no fresh
+ heartbeat at all, or for the longer `session_quiet` if the agent is
+ still connected (its controller, or another session, is up while
+ this session went quiet).
+
+ The revision moves on, so the change is announced like a host's, and a
+ host that was in fact still working overrides it with its next report.
+ Returns (agent, session, turn) of each."""
other = AgentSessionActivityItem.__table__.alias("other")
turn = AgentSessionActivityItem
- quiet = ~exists().where(
- other.c.tenant_id == turn.tenant_id,
- other.c.agent_id == turn.agent_id,
- other.c.session_id == turn.session_id,
- other.c.turn_id == turn.turn_id,
- other.c.updated_at > func.now() - literal(quiet_for),
+
+ def quiet(for_: timedelta) -> ColumnElement[bool]:
+ return ~exists().where(
+ other.c.tenant_id == turn.tenant_id,
+ other.c.agent_id == turn.agent_id,
+ other.c.session_id == turn.session_id,
+ other.c.turn_id == turn.turn_id,
+ other.c.updated_at > func.now() - literal(for_),
+ )
+
+ def held(process_id: Any) -> ColumnElement[bool]:
+ return exists().where(
+ SwitchCoreProcess.id == process_id,
+ SwitchCoreProcess.stopped_at.is_(None),
+ SwitchCoreProcess.beat_at > func.now() - literal(lease_ttl),
+ )
+
+ placed = exists().where(
+ AgentSessionPlacement.agent_id == turn.agent_id,
+ AgentSessionPlacement.session_id == turn.session_id,
+ AgentConnectionRecord.id == AgentSessionPlacement.connection_id,
+ held(AgentConnectionRecord.process_id),
+ )
+ beating_here = exists().where(
+ AgentSession.agent_id == turn.agent_id,
+ AgentSession.room_id == turn.room_id,
+ AgentSession.lifecycle == "heartbeat",
+ AgentSession.last_seen_at > func.now() - literal(heartbeat_ttl),
)
connected = exists().where(
AgentConnectionRecord.agent_id == turn.agent_id,
AgentConnectionRecord.closed_at.is_(None),
- SwitchCoreProcess.id == AgentConnectionRecord.process_id,
- SwitchCoreProcess.stopped_at.is_(None),
- SwitchCoreProcess.beat_at > func.now() - literal(lease_ttl),
+ held(AgentConnectionRecord.process_id),
)
beating = exists().where(
AgentSession.agent_id == turn.agent_id,
@@ -187,9 +230,12 @@ class AgentSessionActivityStore:
.where(
turn.item_id == TURN_ITEM_ID,
turn.status.in_(("queued", "running")),
- quiet,
- ~connected,
- ~beating,
+ ~placed,
+ ~beating_here,
+ or_(
+ and_(quiet(offline_quiet), ~connected, ~beating),
+ quiet(session_quiet),
+ ),
)
.values(
status="interrupted",
diff --git a/core/switch_core/session_activity/service.py b/core/switch_core/session_activity/service.py
index ce57dea8e..4c1cdefaf 100644
--- a/core/switch_core/session_activity/service.py
+++ b/core/switch_core/session_activity/service.py
@@ -78,6 +78,9 @@ NOTICE_PREFIX = "notice:"
# offline and nothing in it has moved for this long, so no one is shown it as
# still working. A live session's turn is never ended here.
ORPHANED_TURN_QUIET = timedelta(minutes=10)
+# Longer when the agent is still connected and only the turn's own session
+# went quiet: a tool can run that long without a step to report.
+ORPHANED_SESSION_QUIET = timedelta(minutes=30)
ORPHANED_TURN_TEXT = (
"Ended by Switch: the agent's session went away before this turn finished."
)
@@ -560,16 +563,16 @@ class AgentSessionActivityService:
return row
async def end_orphaned_turns(self) -> list[tuple[str, str, str]]:
- """End the bound tenant's turns whose agent is gone and that nothing
- has moved for `ORPHANED_TURN_QUIET` (`end_orphaned_turns` on the
- store). Run on a timer."""
+ """End the bound tenant's turns whose session is gone
+ (`end_orphaned_turns` on the store). Run on a timer."""
async with (
tenant_session(self._sessions, require_tenant_id()) as db,
db.begin(),
):
return await self._activity.end_orphaned_turns(
db,
- quiet_for=ORPHANED_TURN_QUIET,
+ offline_quiet=ORPHANED_TURN_QUIET,
+ session_quiet=ORPHANED_SESSION_QUIET,
lease_ttl=LEASE_TTL,
heartbeat_ttl=AgentSessionStore.ALWAYS_ON_TTL,
text=ORPHANED_TURN_TEXT,
diff --git a/core/tests/switch_core/session_activity/test_orphaned_turns.py b/core/tests/switch_core/session_activity/test_orphaned_turns.py
index 7879cd09d..3e0776ae4 100644
--- a/core/tests/switch_core/session_activity/test_orphaned_turns.py
+++ b/core/tests/switch_core/session_activity/test_orphaned_turns.py
@@ -12,12 +12,13 @@ from sqlalchemy import update
from switch_core.db.models import (
AgentConnectionRecord,
AgentSessionActivityItem,
+ AgentSessionPlacement,
SwitchCoreProcess,
)
from switch_core.session_activity.maintenance import end_orphaned_turns_once
from switch_core.session_activity.service import ORPHANED_TURN_TEXT
-from .conftest import AGENT
+from .conftest import AGENT, make_room
async def _turn(service, session_id: str, status: str) -> None:
@@ -40,15 +41,37 @@ async def _turn(service, session_id: str, status: str) -> None:
)
-async def _quiet(session_factory, session_id: str) -> None:
+async def _quiet(session_factory, session_id: str, minutes: int = 40) -> None:
async with session_factory() as db, db.begin():
await db.execute(
update(AgentSessionActivityItem)
.where(AgentSessionActivityItem.session_id == session_id)
- .values(updated_at=datetime.now(UTC) - timedelta(minutes=30))
+ .values(updated_at=datetime.now(UTC) - timedelta(minutes=minutes))
)
+async def _connect(session_factory) -> str:
+ """A live connection for the agent, on a process holding its lease."""
+ process_id = f"proc-{uuid.uuid4().hex[:8]}"
+ connection_id = f"conn-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as db, db.begin():
+ db.add(SwitchCoreProcess(id=process_id))
+ await db.flush()
+ db.add(
+ AgentConnectionRecord(
+ id=connection_id,
+ agent_id=AGENT,
+ process_id=process_id,
+ incarnation=1,
+ scope="all",
+ delivery_filter="addressed",
+ spawn_capable=False,
+ transport="sse",
+ )
+ )
+ return connection_id
+
+
async def _status(service, session_id: str) -> tuple[str, int, str]:
[row] = await service.turn_items(AGENT, session_id, "turn-1")
return row.status, row.revision, row.text
@@ -67,27 +90,36 @@ async def test_an_offline_agents_quiet_turn_is_ended(service, session_factory) -
assert (await _status(service, "done"))[0] == "completed"
-async def test_a_connected_agents_turn_is_left_running(
+async def test_a_connected_agent_keeps_a_turn_its_session_is_still_on(
service, session_factory
) -> None:
- await _turn(service, "live", "running")
- await _quiet(session_factory, "live")
- process_id = f"proc-{uuid.uuid4().hex[:8]}"
+ connection_id = await _connect(session_factory)
async with session_factory() as db, db.begin():
- db.add(SwitchCoreProcess(id=process_id))
- await db.flush()
+ room_id = await make_room(db, member=AGENT)
db.add(
- AgentConnectionRecord(
- id=f"conn-{uuid.uuid4().hex[:8]}",
+ AgentSessionPlacement(
agent_id=AGENT,
- process_id=process_id,
- incarnation=1,
- scope="all",
- delivery_filter="addressed",
- spawn_capable=False,
- transport="sse",
+ session_id="placed",
+ room_id=room_id,
+ connection_id=connection_id,
)
)
+ await _turn(service, "placed", "running")
+ await _turn(service, "recent", "running")
+ await _quiet(session_factory, "placed")
+ await _quiet(session_factory, "recent", minutes=20)
+
+ await end_orphaned_turns_once(session_factory)
+ assert (await _status(service, "placed"))[0] == "running"
+ assert (await _status(service, "recent"))[0] == "running"
+
+
+async def test_a_connected_agents_dead_session_has_its_turn_ended(
+ service, session_factory
+) -> None:
+ await _connect(session_factory)
+ await _turn(service, "faulted", "running")
+ await _quiet(session_factory, "faulted")
await end_orphaned_turns_once(session_factory)
- assert (await _status(service, "live"))[0] == "running"
+ assert (await _status(service, "faulted"))[0] == "interrupted"