Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(databases): recover sessions after harness crashes (#7417)

Wolf Mermelstein0b322c3f1b92
diff --git a/.sqlx/query-1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577.json b/.sqlx/query-1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577.json
new file mode 100644
index 0000000000..e1752b3d8d
--- /dev/null
+++ b/.sqlx/query-1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE harness_replica SET last_heartbeat_at = now() WHERE id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577"
+}
diff --git a/.sqlx/query-2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66.json b/.sqlx/query-2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66.json
new file mode 100644
index 0000000000..7c691a3b02
--- /dev/null
+++ b/.sqlx/query-2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE agent_session_log SET created_at = created_at + interval '1 second' WHERE agent_session_id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66"
+}
diff --git a/.sqlx/query-371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8.json b/.sqlx/query-371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8.json
new file mode 100644
index 0000000000..735d41a923
--- /dev/null
+++ b/.sqlx/query-371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE agent_session AS session\n SET manager_replica_id = NULL,\n manager_fence = manager_fence + 1,\n status = $4, status_event_name = $5, turn_state = $6,\n modified_at = now()\n WHERE session.id = $1\n AND session.manager_replica_id = $2 AND session.manager_fence = $3\n AND EXISTS (\n SELECT 1 FROM harness_replica AS replica\n WHERE replica.id = session.manager_replica_id\n AND replica.last_heartbeat_at <= now() - make_interval(secs => $7)\n )\n RETURNING session.id\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Int8",
+ "Text",
+ "Text",
+ "Text",
+ "Float8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8"
+}
diff --git a/.sqlx/query-4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580.json b/.sqlx/query-4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580.json
new file mode 100644
index 0000000000..8263e98949
--- /dev/null
+++ b/.sqlx/query-4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT id FROM harness_replica WHERE id = $1 FOR UPDATE",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580"
+}
diff --git a/.sqlx/query-6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c.json b/.sqlx/query-6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c.json
new file mode 100644
index 0000000000..0287ac34b0
--- /dev/null
+++ b/.sqlx/query-6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c.json
@@ -0,0 +1,24 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n RETURNING id\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n FROM replica\n WHERE agent_session.id = $1 AND replica.id = $2\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n AND live.draining_at IS NULL\n )\n )\n RETURNING manager_fence\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "manager_fence",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Float8"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c"
+}
diff --git a/.sqlx/query-8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108.json b/.sqlx/query-8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108.json
new file mode 100644
index 0000000000..64144677a0
--- /dev/null
+++ b/.sqlx/query-8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108.json
@@ -0,0 +1,35 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT session.id, session.manager_replica_id AS \"replica!\", session.manager_fence\n FROM agent_session AS session\n JOIN harness_replica AS replica ON replica.id = session.manager_replica_id\n WHERE session.manager_replica_id IS NOT NULL\n AND replica.last_heartbeat_at <= now() - make_interval(secs => $1)\n ORDER BY session.manager_replica_id, session.id\n LIMIT $2\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "replica!",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 2,
+ "name": "manager_fence",
+ "type_info": "Int8"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Float8",
+ "Int8"
+ ]
+ },
+ "nullable": [
+ false,
+ true,
+ false
+ ]
+ },
+ "hash": "8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108"
+}
diff --git a/.sqlx/query-bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee.json b/.sqlx/query-bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee.json
new file mode 100644
index 0000000000..f41ad95457
--- /dev/null
+++ b/.sqlx/query-bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee.json
@@ -0,0 +1,32 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)\n SELECT frame.id, $1, frame.user_id, frame.direction, frame.content,\n stamp.created_at + (frame.ordinality - 1) * interval '1 microsecond'\n FROM UNNEST($2::uuid[], $3::text[], $4::text[], $5::jsonb[])\n WITH ORDINALITY AS frame(id, user_id, direction, content, ordinality)\n CROSS JOIN (\n SELECT GREATEST(statement_timestamp(), (\n SELECT created_at + interval '1 microsecond'\n FROM agent_session_log WHERE agent_session_id = $1\n ORDER BY created_at DESC, id DESC LIMIT 1\n )) AS created_at\n ) AS stamp\n RETURNING id, created_at\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "id",
+ "type_info": "Uuid"
+ },
+ {
+ "ordinal": 1,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "UuidArray",
+ "TextArray",
+ "TextArray",
+ "JsonbArray"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee"
+}
diff --git a/.sqlx/query-c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274.json b/.sqlx/query-c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274.json
new file mode 100644
index 0000000000..df78d2d982
--- /dev/null
+++ b/.sqlx/query-c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274.json
@@ -0,0 +1,15 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "UPDATE agent_session SET manager_replica_id = $2, manager_fence = manager_fence + 1 WHERE id = $1",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274"
+}
diff --git a/.sqlx/query-eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764.json b/.sqlx/query-eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764.json
new file mode 100644
index 0000000000..303a27c997
--- /dev/null
+++ b/.sqlx/query-eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764.json
@@ -0,0 +1,26 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)\n VALUES ($1, $2, $3, $4, $5, GREATEST(clock_timestamp(), (\n SELECT created_at + interval '1 microsecond'\n FROM agent_session_log WHERE agent_session_id = $2\n ORDER BY created_at DESC, id DESC LIMIT 1\n )))\n RETURNING created_at\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Text",
+ "Text",
+ "Jsonb"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764"
+}
diff --git a/.sqlx/query-f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45.json b/.sqlx/query-f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45.json
new file mode 100644
index 0000000000..de5b20be53
--- /dev/null
+++ b/.sqlx/query-f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45.json
@@ -0,0 +1,26 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)\n VALUES ($1, $2, $3, $4, $5, GREATEST(statement_timestamp(), (\n SELECT created_at + interval '1 microsecond'\n FROM agent_session_log WHERE agent_session_id = $2\n ORDER BY created_at DESC, id DESC LIMIT 1\n )))\n RETURNING created_at\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "created_at",
+ "type_info": "Timestamptz"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Uuid",
+ "Uuid",
+ "Text",
+ "Text",
+ "Jsonb"
+ ]
+ },
+ "nullable": [
+ false
+ ]
+ },
+ "hash": "f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45"
+}
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
index 0f5235a15c..07546f47c8 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
@@ -32,6 +32,21 @@ const socket = vi.hoisted(() => ({
}),
}));
+const updates = vi.hoisted(() => ({
+ listeners: new Set<(event: { agentSessionId: string }) => void>(),
+}));
+vi.mock('@queries/agent-session/session-metadata-sync', () => ({
+ subscribeAgentSessionUpdated: (
+ listener: (event: { agentSessionId: string }) => void
+ ) => {
+ updates.listeners.add(listener);
+ return () => updates.listeners.delete(listener);
+ },
+}));
+function invalidate(id = SESSION) {
+ for (const listener of updates.listeners) listener({ agentSessionId: id });
+}
+
vi.mock('@core/agent-fold/client', () => fold);
vi.mock('@service-agent-harness/client', () => ({
agentHarnessServiceClient: harness,
@@ -84,6 +99,7 @@ beforeEach(() => {
vi.clearAllMocks();
resetSessionTurns();
socket.listeners.clear();
+ updates.listeners.clear();
// Instances are shared and refcounted, so a test that fails before its
// `release()` would hand the next one a session that is already loaded.
for (
@@ -316,6 +332,200 @@ describe('AgentSession', () => {
expect(AgentSession.get(SESSION)).toBeUndefined();
});
+ it('reloads committed recovery state and preserves frames received during the read', async () => {
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+ const log = deferred<LogResult>();
+ harness.getLog.mockReturnValueOnce(log.promise);
+ invalidate('another-session');
+ expect(harness.getLog).toHaveBeenCalledTimes(1);
+ invalidate();
+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(3)] });
+ expect(inputs().filter((input) => input.kind === 'confirmed')).toEqual([]);
+ log.resolve(logOf([row(1), row(2)]));
+ await vi.waitFor(() =>
+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(3) })
+ );
+ expect(inputs().slice(-2)).toEqual([
+ { kind: 'snapshot', rows: [row(1), row(2)] },
+ { kind: 'confirmed', row: row(3) },
+ ]);
+ live.release();
+ expect(updates.listeners.size).toBe(0);
+ });
+
+ it('coalesces repeated invalidations and catches one received during initial loading', async () => {
+ const first = deferred<LogResult>();
+ const refresh = deferred<LogResult>();
+ harness.getLog
+ .mockReturnValueOnce(first.promise)
+ .mockReturnValueOnce(refresh.promise)
+ .mockResolvedValueOnce(logOf([row(1), row(2), row(3)]));
+ const live = AgentSession.acquire(SESSION);
+ invalidate();
+ first.resolve(logOf([row(1)]));
+ await live.load();
+ expect(harness.getLog).toHaveBeenCalledTimes(2);
+ invalidate();
+ invalidate();
+ expect(harness.getLog).toHaveBeenCalledTimes(2);
+ refresh.resolve(logOf([row(1), row(2)]));
+ await vi.waitFor(() =>
+ expect(inputs().at(-1)).toEqual({
+ kind: 'snapshot',
+ rows: [row(1), row(2), row(3)],
+ })
+ );
+ expect(harness.getLog).toHaveBeenCalledTimes(3);
+ live.release();
+ });
+
+ it('releases buffered live frames after a failed refresh', async () => {
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+ const refresh = deferred<LogResult>();
+ harness.getLog.mockReturnValueOnce(refresh.promise);
+ invalidate();
+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(2)] });
+ refresh.resolve(err([{ code: 'INTERNAL' }]));
+ await vi.waitFor(() =>
+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(2) })
+ );
+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(3)] });
+ await live.snapshot();
+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(3) });
+ live.release();
+ });
+
+ it.each(['result', 'transport'])(
+ 'retries a failed refresh without blocking live frames (%s)',
+ async (failure) => {
+ vi.useFakeTimers();
+ const live = AgentSession.acquire(SESSION);
+ try {
+ await live.load();
+ if (failure === 'result')
+ harness.getLog.mockResolvedValueOnce(err([{ code: 'INTERNAL' }]));
+ else harness.getLog.mockRejectedValueOnce(new Error('network down'));
+ harness.getLog.mockResolvedValueOnce(logOf([row(1), row(2), row(3)]));
+ invalidate();
+ await vi.advanceTimersByTimeAsync(0);
+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(2)] });
+ await live.snapshot();
+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(2) });
+ expect(harness.getLog).toHaveBeenCalledTimes(2);
+ await vi.advanceTimersByTimeAsync(1_000);
+ expect(inputs().at(-1)).toEqual({
+ kind: 'snapshot',
+ rows: [row(1), row(2), row(3)],
+ });
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(harness.getLog).toHaveBeenCalledTimes(3);
+ } finally {
+ live.release();
+ vi.useRealTimers();
+ }
+ }
+ );
+
+ it('bounds failed refresh retries and gives a new invalidation a fresh budget', async () => {
+ vi.useFakeTimers();
+ const live = AgentSession.acquire(SESSION);
+ try {
+ await live.load();
+ harness.getLog.mockResolvedValue(err([{ code: 'INTERNAL' }]));
+ invalidate();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(harness.getLog).toHaveBeenCalledTimes(5);
+ invalidate();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(harness.getLog).toHaveBeenCalledTimes(9);
+ } finally {
+ live.release();
+ vi.useRealTimers();
+ }
+ });
+
+ it('cancels a scheduled refresh retry when the last view releases it', async () => {
+ vi.useFakeTimers();
+ const live = AgentSession.acquire(SESSION);
+ try {
+ await live.load();
+ harness.getLog.mockResolvedValue(err([{ code: 'INTERNAL' }]));
+ invalidate();
+ await vi.advanceTimersByTimeAsync(0);
+ live.release();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(harness.getLog).toHaveBeenCalledTimes(2);
+ } finally {
+ if (AgentSession.get(SESSION)) live.release();
+ vi.useRealTimers();
+ }
+ });
+
+ it('does not retry an access-denied refresh', async () => {
+ vi.useFakeTimers();
+ const live = AgentSession.acquire(SESSION);
+ try {
+ await live.load();
+ harness.getLog.mockResolvedValue(err([{ code: 'FORBIDDEN' }]));
+ invalidate();
+ await vi.advanceTimersByTimeAsync(60_000);
+ expect(harness.getLog).toHaveBeenCalledTimes(2);
+ } finally {
+ live.release();
+ vi.useRealTimers();
+ }
+ });
+
+ it.each([false, true])(
+ 'retains optimistic actions during refresh (replacement ID=%s)',
+ async (replacement) => {
+ if (replacement)
+ harness.control.mockResolvedValue(
+ ok({ actionId: 'server-id', status: 'sent' })
+ );
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+ const refresh = deferred<LogResult>();
+ harness.getLog.mockReturnValueOnce(refresh.promise);
+ invalidate();
+ await live.issue({ type: 'prompt', prompt: 'Continue' });
+ expect(inputs().filter((input) => input.kind === 'speculated')).toEqual(
+ []
+ );
+ refresh.resolve(logOf([row(1)]));
+ await vi.waitFor(() =>
+ expect(inputs().at(-1)).toMatchObject({
+ kind: 'speculated',
+ action: { type: 'prompt', prompt: 'Continue' },
+ })
+ );
+ if (replacement) {
+ const issued = harness.control.mock.calls.at(-1)![1].actionId;
+ expect(inputs().slice(-2)).toMatchObject([
+ { kind: 'retracted', actionId: issued },
+ { kind: 'speculated', actionId: 'server-id' },
+ ]);
+ }
+ live.release();
+ }
+ );
+
+ it('does not recreate a released session when a refresh completes', async () => {
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+ const refresh = deferred<LogResult>();
+ harness.getLog.mockReturnValueOnce(refresh.promise);
+ invalidate();
+ live.release();
+ const before = inputs();
+ refresh.resolve(logOf([row(1), row(2)]));
+ await settle();
+ expect(inputs()).toEqual(before);
+ expect(updates.listeners.size).toBe(0);
+ });
+
it('re-snapshots when the socket reopens', async () => {
const live = AgentSession.acquire(SESSION);
await live.load();
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.ts
index 8948223088..72a8182621 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts
@@ -20,8 +20,10 @@ import {
readSession,
type SessionFoldSnapshot,
} from '@core/agent-fold/client';
+import { fetchAgentSessionLog } from '@queries/agent-session/log';
import { subscribeSocketSessionStarted } from '@queries/agent-session/queue-sync';
import type { AgentSessionLogEvent } from '@queries/agent-session/realtime-protocol';
+import { subscribeAgentSessionUpdated } from '@queries/agent-session/session-metadata-sync';
import type {
FoldedStreamEvent,
TurnState,
@@ -90,6 +92,8 @@ function occupiesTurn(action: AgentAction): boolean {
return action.type === 'prompt' || action.type === 'compact';
}
+const RESYNC_RETRY_DELAYS_MS = [1_000, 3_000, 10_000];
+
export class AgentSession {
private static readonly open = new Map<string, AgentSession>();
@@ -134,6 +138,11 @@ export class AgentSession {
private buffered: FoldInput[] = [];
private readonly listeners = new Set<AgentSessionListener>();
private readonly unsubscribeSocket: () => void;
+ private readonly unsubscribeUpdated: () => void;
+ private syncing = false;
+ private resyncRequested = false;
+ private resyncRetry = 0;
+ private resyncTimer?: ReturnType<typeof setTimeout>;
/**
* Serializes worker pushes so inputs reach the machine in the order this
* class saw them, even though each push is its own await.
@@ -163,6 +172,9 @@ export class AgentSession {
this.unsubscribeSocket = subscribeSocketSessionStarted(() => {
void this.resync();
});
+ this.unsubscribeUpdated = subscribeAgentSessionUpdated((event) => {
+ if (event.agentSessionId === this.id) void this.resync();
+ });
this.trace = new SessionLoadTrace(id);
this.loading = this.startLoad();
}
@@ -238,7 +250,7 @@ export class AgentSession {
// confirms it by content whatever id the harness accepted it under.
const accepted = result.value.actionId;
if (accepted !== actionId && action.type !== 'stop') {
- void this.apply([
+ void this.enqueueAll([
{ kind: 'retracted', actionId },
{
kind: 'speculated',
@@ -343,6 +355,8 @@ export class AgentSession {
this.trace.end('released');
this.listeners.clear();
this.unsubscribeSocket();
+ this.unsubscribeUpdated();
+ clearTimeout(this.resyncTimer);
closeSession(this.id);
}
@@ -366,7 +380,7 @@ export class AgentSession {
private async fetchAndFold(): Promise<AgentSessionRecord> {
const [session, log] = await Promise.all([
agentHarnessServiceClient.get(this.id),
- agentHarnessServiceClient.getLog(this.id),
+ fetchAgentSessionLog(this.id),
]);
if (session.isErr()) {
if (accessDenied(session.error)) {
@@ -385,14 +399,10 @@ export class AgentSession {
await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);
// Inputs can keep arriving while each push is in flight; drain until a
// check finds nothing, then flip ready so the next one goes straight in.
- while (this.buffered.length > 0) {
- const inputs = this.buffered;
- this.buffered = [];
- await this.apply(inputs);
- }
- this.ready = true;
+ await this.drainBuffered();
this.trace.folded(foldStartedAt);
this.setTurn((await readSession(this.id)).metadata.turn);
+ if (this.resyncRequested) void this.resync();
return { session: session.value, bot: log.value.bot };
}
@@ -401,11 +411,54 @@ export class AgentSession {
* while it was down. Refetch, and let the machine reconcile the overlap
* and settle any speculation the log confirmed meanwhile.
*/
- private async resync(): Promise<void> {
- if (!this.ready || this.closed) return;
- const log = await agentHarnessServiceClient.getLog(this.id);
- if (log.isErr() || this.closed) return;
- await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);
+ private async resync(retrying = false): Promise<void> {
+ if (this.closed) return;
+ if (!retrying) {
+ clearTimeout(this.resyncTimer);
+ this.resyncRetry = 0;
+ }
+ this.resyncRequested = true;
+ if (!this.ready || this.syncing) return;
+ this.syncing = true;
+ this.ready = false;
+ let failed = false;
+ try {
+ do {
+ this.resyncRequested = false;
+ const log = await fetchAgentSessionLog(this.id);
+ if (this.closed) return;
+ failed = log.isErr() && !accessDenied(log.error);
+ if (log.isOk()) {
+ await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);
+ this.resyncRetry = 0;
+ }
+ } while (this.resyncRequested);
+ } catch (error: unknown) {
+ failed = true;
+ console.warn('[agent-session] log could not be refreshed', error);
+ } finally {
+ // A snapshot may predate frames received during its fetch. Replay those
+ // frames after it, retaining speculation and the fold's overlap deduping.
+ await this.drainBuffered();
+ this.syncing = false;
+ if (!this.closed && this.resyncRequested) {
+ void this.resync(true);
+ } else if (!this.closed && failed) {
+ const delay = RESYNC_RETRY_DELAYS_MS[this.resyncRetry++];
+ if (delay !== undefined)
+ this.resyncTimer = setTimeout(() => void this.resync(true), delay);
+ }
+ }
+ }
+
+ private async drainBuffered(): Promise<void> {
+ while (!this.closed && this.buffered.length > 0) {
+ const inputs = this.buffered;
+ this.buffered = [];
+ await this.apply(inputs);
+ }
+ // No await between observing an empty buffer and accepting live inputs.
+ if (!this.closed) this.ready = true;
}
private enqueue(input: FoldInput): Promise<void> {
diff --git a/apps/web/src/lib/queries/agent-session/log.ts b/apps/web/src/lib/queries/agent-session/log.ts
new file mode 100644
index 0000000000..8b90f72be9
--- /dev/null
+++ b/apps/web/src/lib/queries/agent-session/log.ts
@@ -0,0 +1,6 @@
+import { agentHarnessServiceClient } from '@service-agent-harness/client';
+
+/** Read the authoritative log; the shared fold owns its in-memory projection. */
+export function fetchAgentSessionLog(sessionId: string) {
+ return agentHarnessServiceClient.getLog(sessionId);
+}
diff --git a/crates/agent_session/src/domain/mod.rs b/crates/agent_session/src/domain/mod.rs
index e2095d4366..0e71a176e2 100644
--- a/crates/agent_session/src/domain/mod.rs
+++ b/crates/agent_session/src/domain/mod.rs
@@ -31,3 +31,6 @@ pub mod turn_state;
/// Repository-matched working branches reported by a session's runtime.
pub mod working_branch;
+
+/// Recovery of sessions abandoned by crashed replicas.
+pub mod recovery;
diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rs
index b921590f3a..5519de607f 100644
--- a/crates/agent_session/src/domain/ports.rs
+++ b/crates/agent_session/src/domain/ports.rs
@@ -827,7 +827,7 @@ pub trait AgentSessionRealtime {
event: LogAppended,
) -> impl Future<Output = Result<(), rootcause::Report>> + Send;
- /// Tell viewers to refetch changed session metadata.
+ /// Tell viewers to refetch changed session metadata and the durable log.
fn publish_updated(
&self,
_session: AgentSessionId,
diff --git a/crates/agent_session/src/domain/recovery.rs b/crates/agent_session/src/domain/recovery.rs
new file mode 100644
index 0000000000..6d3630fdf5
--- /dev/null
+++ b/crates/agent_session/src/domain/recovery.rs
@@ -0,0 +1,60 @@
+//! Close abandoned runtime connections without replaying their work.
+
+use std::num::NonZeroUsize;
+
+use super::error::Result;
+use super::model::{SessionClaim, StoredAgentSessionLog};
+use super::ports::AgentSessionRealtime;
+
+/// Atomic persistence boundary for recovery after a replica stops heartbeating.
+pub trait StaleSessionRepo: Send + Sync {
+ /// Return at most `limit` claims whose holder has missed the lease deadline.
+ /// A draining but heartbeating holder is still alive and must be excluded.
+ fn stale_claims(
+ &self,
+ limit: NonZeroUsize,
+ ) -> impl Future<Output = Result<Vec<SessionClaim>>> + Send;
+
+ /// Recheck the exact holder, fence, and heartbeat, then atomically revoke
+ /// the claim, append `Disconnected`, and project its turn state. Return
+ /// `None` if a heartbeat, takeover, release, or another recovery won.
+ /// Revocation increments the fence so the old actor can never append again.
+ fn disconnect_stale_claim(
+ &self,
+ claim: SessionClaim,
+ ) -> impl Future<Output = Result<Option<StoredAgentSessionLog>>> + Send;
+}
+
+/// Reconcile one bounded batch and invalidate viewers after durable recovery.
+/// This only closes runtime state: it never starts a runtime or retries work.
+#[tracing::instrument(err, skip(repo, realtime))]
+pub async fn recover_stale_sessions<R: StaleSessionRepo, Rt: AgentSessionRealtime>(
+ repo: &R,
+ realtime: &Rt,
+ limit: NonZeroUsize,
+) -> Result<usize> {
+ let mut recovered = 0;
+ for claim in repo.stale_claims(limit).await? {
+ match repo.disconnect_stale_claim(claim).await {
+ Ok(Some(_)) => {}
+ Ok(None) => continue,
+ Err(error) => {
+ tracing::warn!(error = ?error, session_id = %claim.session, recovered, "failed to disconnect abandoned agent session");
+ continue;
+ }
+ };
+ recovered += 1;
+ // A successor can append AcpReady immediately after the commit. Never
+ // publish a delayed Disconnected live frame over that newer state:
+ // viewers must refetch the log in durable order instead.
+ realtime
+ .publish_updated(claim.session)
+ .await
+ .inspect_err(|error| {
+ tracing::warn!(error = ?error, session_id = %claim.session, "failed to publish recovered session disconnect");
+ })
+ .ok();
+ tracing::info!(session_id = %claim.session, replica_id = %claim.replica, fence = claim.fence.0, "disconnected abandoned agent session");
+ }
+ Ok(recovered)
+}
diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rs
index 97f6bf3b8f..308ab3c842 100644
--- a/crates/agent_session/src/outbound/postgres.rs
+++ b/crates/agent_session/src/outbound/postgres.rs
@@ -11,6 +11,7 @@ mod test;
mod pull_request;
mod queue;
+mod recovery;
mod sharing;
mod turn_state;
mod working_branch;
@@ -1251,17 +1252,24 @@ impl<B: BotFacts + 'static> AgentSessionLogRepo for PgAgentSessionRepo<B> {
return Err(AgentSessionError::FencedOut(session));
}
- // One transaction means one `now()`, so the batch is spread over
- // consecutive microseconds in append order: readers order by
- // `(created_at, id)`, and the ids are v7 without a monotonic
- // counter, so same-instant rows would otherwise interleave.
+ // Start after the durable tail while holding the session lock, then
+ // spread the batch over consecutive microseconds. Transaction start
+ // time can predate a lock wait or a previous batch's synthetic tail;
+ // readers must see append order rather than UUID tie-breaking.
let stamped = sqlx::query!(
r#"
INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)
SELECT frame.id, $1, frame.user_id, frame.direction, frame.content,
- now() + (frame.ordinality - 1) * interval '1 microsecond'
+ stamp.created_at + (frame.ordinality - 1) * interval '1 microsecond'
FROM UNNEST($2::uuid[], $3::text[], $4::text[], $5::jsonb[])
WITH ORDINALITY AS frame(id, user_id, direction, content, ordinality)
+ CROSS JOIN (
+ SELECT GREATEST(statement_timestamp(), (
+ SELECT created_at + interval '1 microsecond'
+ FROM agent_session_log WHERE agent_session_id = $1
+ ORDER BY created_at DESC, id DESC LIMIT 1
+ )) AS created_at
+ ) AS stamp
RETURNING id, created_at
"#,
session.as_uuid(),
@@ -1357,8 +1365,12 @@ impl<B: BotFacts + 'static> AgentSessionLogRepo for PgAgentSessionRepo<B> {
let id = macro_uuid::generate_uuid_v7();
let created_at = sqlx::query_scalar!(
r#"
- INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content)
- VALUES ($1, $2, $3, $4, $5)
+ INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)
+ VALUES ($1, $2, $3, $4, $5, GREATEST(statement_timestamp(), (
+ SELECT created_at + interval '1 microsecond'
+ FROM agent_session_log WHERE agent_session_id = $2
+ ORDER BY created_at DESC, id DESC LIMIT 1
+ )))
RETURNING created_at
"#,
id,
@@ -1543,12 +1555,14 @@ impl<B: BotFacts + 'static> SessionOwnership for PgAgentSessionRepo<B> {
INSERT INTO harness_replica (id, last_heartbeat_at)
VALUES ($2, now())
ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()
+ RETURNING id
)
UPDATE agent_session
SET manager_replica_id = $2,
manager_fence = manager_fence + 1,
modified_at = now()
- WHERE id = $1
+ FROM replica
+ WHERE agent_session.id = $1 AND replica.id = $2
AND (
manager_replica_id IS NULL
OR manager_replica_id = $2
diff --git a/crates/agent_session/src/outbound/postgres/recovery.rs b/crates/agent_session/src/outbound/postgres/recovery.rs
new file mode 100644
index 0000000000..c08dbd7217
--- /dev/null
+++ b/crates/agent_session/src/outbound/postgres/recovery.rs
@@ -0,0 +1,125 @@
+use super::*;
+use crate::domain::recovery::StaleSessionRepo;
+use agent_fold::domain::model::TurnState;
+
+impl<B: BotFacts + 'static> StaleSessionRepo for PgAgentSessionRepo<B> {
+ async fn stale_claims(&self, limit: NonZeroUsize) -> Result<Vec<SessionClaim>> {
+ let rows = sqlx::query!(
+ r#"
+ SELECT session.id, session.manager_replica_id AS "replica!", session.manager_fence
+ FROM agent_session AS session
+ JOIN harness_replica AS replica ON replica.id = session.manager_replica_id
+ WHERE session.manager_replica_id IS NOT NULL
+ AND replica.last_heartbeat_at <= now() - make_interval(secs => $1)
+ ORDER BY session.manager_replica_id, session.id
+ LIMIT $2
+ "#,
+ REPLICA_STALE_AFTER.as_secs_f64(),
+ i64::try_from(limit.get()).unwrap_or(i64::MAX),
+ )
+ .fetch_all(&self.pool)
+ .await
+ .context("list abandoned session claims")?;
+ Ok(rows
+ .into_iter()
+ .map(|row| SessionClaim {
+ session: AgentSessionId::new_from_uuid(row.id),
+ replica: ReplicaId::from_uuid(row.replica),
+ fence: ManagerFence(row.manager_fence),
+ })
+ .collect())
+ }
+
+ async fn disconnect_stale_claim(
+ &self,
+ claim: SessionClaim,
+ ) -> Result<Option<StoredAgentSessionLog>> {
+ let log = AgentSessionLog {
+ agent_session_id: claim.session,
+ user_id: None,
+ content: Message::ToServer(ToServerMessage::Event {
+ event: SystemEvent::Disconnected,
+ }),
+ };
+ let (direction, content) = message_columns(&log.content)?;
+ let status = SessionStatus::Event(SystemEvent::Disconnected);
+ let (status, status_event_name) = status_columns(&status);
+ let mut transaction = self
+ .pool
+ .begin()
+ .await
+ .context("begin abandoned session recovery")?;
+ // Match claim's replica-before-session lock order. A heartbeat that
+ // commits while this waits must be seen by the next statement; one
+ // arriving after this lock waits until revocation is durable.
+ sqlx::query_scalar!(
+ "SELECT id FROM harness_replica WHERE id = $1 FOR UPDATE",
+ claim.replica.as_uuid(),
+ )
+ .fetch_optional(&mut *transaction)
+ .await
+ .context("lock abandoned replica heartbeat")?;
+ // Claim changes and live appends lock this same row. An intervening
+ // owner/fence change fails the predicate even after waiting for its
+ // transaction; an old process that wakes later is fenced out.
+ let revoked = sqlx::query_scalar!(
+ r#"
+ UPDATE agent_session AS session
+ SET manager_replica_id = NULL,
+ manager_fence = manager_fence + 1,
+ status = $4, status_event_name = $5, turn_state = $6,
+ modified_at = now()
+ WHERE session.id = $1
+ AND session.manager_replica_id = $2 AND session.manager_fence = $3
+ AND EXISTS (
+ SELECT 1 FROM harness_replica AS replica
+ WHERE replica.id = session.manager_replica_id
+ AND replica.last_heartbeat_at <= now() - make_interval(secs => $7)
+ )
+ RETURNING session.id
+ "#,
+ claim.session.as_uuid(),
+ claim.replica.as_uuid(),
+ claim.fence.0,
+ status,
+ status_event_name,
+ TurnState::Disconnected.as_ref(),
+ REPLICA_STALE_AFTER.as_secs_f64(),
+ )
+ .fetch_optional(&mut *transaction)
+ .await
+ .context("revoke abandoned session claim")?;
+ if revoked.is_none() {
+ return Ok(None);
+ }
+ let id = macro_uuid::generate_uuid_v7();
+ let created_at = sqlx::query_scalar!(
+ r#"
+ INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)
+ VALUES ($1, $2, $3, $4, $5, GREATEST(clock_timestamp(), (
+ SELECT created_at + interval '1 microsecond'
+ FROM agent_session_log WHERE agent_session_id = $2
+ ORDER BY created_at DESC, id DESC LIMIT 1
+ )))
+ RETURNING created_at
+ "#,
+ id,
+ claim.session.as_uuid(),
+ Option::<&str>::None,
+ direction,
+ content,
+ )
+ .fetch_one(&mut *transaction)
+ .await
+ .context("append abandoned session disconnect")?;
+ transaction
+ .commit()
+ .await
+ .context("commit abandoned session recovery")?;
+ Ok(Some(StoredAgentSessionLog {
+ id,
+ created_at,
+ entry: log,
+ }))
+ }
+}
diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rs
index 1842d952c5..17d98b71b8 100644
--- a/crates/agent_session/src/outbound/postgres/test.rs
+++ b/crates/agent_session/src/outbound/postgres/test.rs
@@ -1,5 +1,6 @@
use super::*;
mod queue;
+mod recovery;
mod search;
mod user_cleanup;
mod working_branch;
diff --git a/crates/agent_session/src/outbound/postgres/test/recovery.rs b/crates/agent_session/src/outbound/postgres/test/recovery.rs
new file mode 100644
index 0000000000..4dfa63eed9
--- /dev/null
+++ b/crates/agent_session/src/outbound/postgres/test/recovery.rs
@@ -0,0 +1,255 @@
+use super::*;
+use crate::domain::ports::AgentSessionRealtime;
+use crate::domain::recovery::{StaleSessionRepo, recover_stale_sessions};
+use agent_fold::domain::lifecycle::LifecycleFold;
+use agent_fold::domain::model::TurnState;
+use agent_fold::testing::{TURN, parse_log_as};
+use std::sync::Mutex;
+
+#[derive(Default)]
+struct Realtime(Mutex<Vec<AgentSessionId>>);
+
+impl AgentSessionRealtime for Realtime {
+ async fn publish(
+ &self,
+ _event: crate::domain::model::LogAppended,
+ ) -> std::result::Result<(), rootcause::Report> {
+ panic!("recovery must invalidate snapshots, never send stale live frames");
+ }
+
+ async fn publish_updated(
+ &self,
+ session: AgentSessionId,
+ ) -> std::result::Result<(), rootcause::Report> {
+ self.0.lock().unwrap().push(session);
+ Ok(())
+ }
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn recovery_preserves_history_closes_stale_turn_and_fences_old_writer(pool: PgPool) {
+ let repo = test_repo(&pool);
+ let bot = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot, None, None)).await;
+ let replica = ReplicaId::mint();
+ let claim = claimed(repo.claim(session.id, replica).await.unwrap());
+ let mut fold = LifecycleFold::new();
+ for frame in parse_log_as(session.id, TURN) {
+ let prompt = matches!(&frame.content, Message::ToRuntime(ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request)))) if request.method.as_ref() == "session/prompt");
+ let _ = fold.push(frame.clone());
+ repo.create_projected(
+ frame,
+ Some(&claim),
+ None,
+ Some(fold.inner().metadata().turn),
+ )
+ .await
+ .unwrap();
+ if prompt {
+ break;
+ }
+ }
+ assert_eq!(fold.inner().metadata().turn, TurnState::Running);
+ // Batched writes can leave their final microseconds ahead of wall time.
+ // Both recovery and the successor must remain after that durable tail.
+ sqlx::query!(
+ "UPDATE agent_session_log SET created_at = created_at + interval '1 second' WHERE agent_session_id = $1",
+ session.id.as_uuid()
+ )
+ .execute(&pool)
+ .await
+ .unwrap();
+ let before = repo.list_by_session(session.id).await.unwrap();
+ let_heartbeat_go_stale(&pool, replica).await;
+ let realtime = Realtime::default();
+ assert_eq!(
+ recover_stale_sessions(&repo, &realtime, NonZeroUsize::new(10).unwrap())
+ .await
+ .unwrap(),
+ 1
+ );
+ let after = repo.list_by_session(session.id).await.unwrap();
+ assert_eq!(after.len(), before.len() + 1);
+ assert_eq!(
+ before.iter().map(|v| v.id).collect::<Vec<_>>(),
+ after[..before.len()]
+ .iter()
+ .map(|v| v.id)
+ .collect::<Vec<_>>()
+ );
+ let _ = fold.push(after.last().unwrap().entry.clone());
+ assert_eq!(fold.inner().metadata().turn, TurnState::Disconnected);
+ assert!(after.last().unwrap().created_at > before.last().unwrap().created_at);
+ let projected = sqlx::query_scalar!(
+ "SELECT turn_state FROM agent_session WHERE id = $1",
+ session.id.as_uuid()
+ )
+ .fetch_one(&pool)
+ .await
+ .unwrap();
+ assert_eq!(projected.as_deref(), Some("disconnected"));
+ assert_eq!(realtime.0.lock().unwrap()[0], session.id);
+ assert!(matches!(
+ repo.create_fenced(fenced_log(session.id), &claim).await,
+ Err(AgentSessionError::FencedOut(_))
+ ));
+ assert_eq!(
+ recover_stale_sessions(&repo, &realtime, NonZeroUsize::new(10).unwrap())
+ .await
+ .unwrap(),
+ 0
+ );
+ assert_eq!(realtime.0.lock().unwrap().len(), 1);
+ let resumed = claimed(repo.claim(session.id, ReplicaId::mint()).await.unwrap());
+ assert_eq!(resumed.fence.0, claim.fence.0 + 2);
+ let ready = repo
+ .create_fenced(
+ AgentSessionLog {
+ agent_session_id: session.id,
+ user_id: None,
+ content: Message::ToServer(ToServerMessage::Event {
+ event: SystemEvent::AcpReady,
+ }),
+ },
+ &resumed,
+ )
+ .await
+ .unwrap();
+ assert!(ready.created_at > after.last().unwrap().created_at);
+ let batch = repo
+ .create_batch_fenced(
+ vec![StoredAgentSessionLog {
+ id: macro_uuid::generate_uuid_v7(),
+ created_at: chrono::Utc::now(),
+ entry: fenced_log(session.id),
+ }],
+ &resumed,
+ )
+ .await
+ .unwrap();
+ assert!(batch[0].created_at > ready.created_at);
+ let mut replay = LifecycleFold::new();
+ for row in repo.list_by_session(session.id).await.unwrap() {
+ let _ = replay.push(row.entry);
+ }
+ assert_ne!(replay.inner().metadata().turn, TurnState::Disconnected);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn recovery_is_bounded_and_leaves_live_and_draining_holders_alone(pool: PgPool) {
+ let repo = test_repo(&pool);
+ let bot = create_test_bot(&pool).await;
+ let live = create_session(&repo, new_session(bot, None, None)).await;
+ let live_replica = ReplicaId::mint();
+ let live_claim = claimed(repo.claim(live.id, live_replica).await.unwrap());
+ repo.begin_draining(live_replica).await.unwrap();
+ assert!(
+ repo.disconnect_stale_claim(live_claim)
+ .await
+ .unwrap()
+ .is_none()
+ );
+ for _ in 0..3 {
+ let session = create_session(&repo, new_session(bot, None, None)).await;
+ let replica = ReplicaId::mint();
+ claimed(repo.claim(session.id, replica).await.unwrap());
+ let_heartbeat_go_stale(&pool, replica).await;
+ }
+ let realtime = Realtime::default();
+ let limit = NonZeroUsize::new(2).unwrap();
+ assert_eq!(
+ recover_stale_sessions(&repo, &realtime, limit)
+ .await
+ .unwrap(),
+ 2
+ );
+ assert_eq!(
+ recover_stale_sessions(&repo, &realtime, limit)
+ .await
+ .unwrap(),
+ 1
+ );
+ assert!(repo.list_by_session(live.id).await.unwrap().is_empty());
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn refreshed_heartbeat_wins_while_recovery_waits_for_its_lock(pool: PgPool) {
+ let repo = test_repo(&pool);
+ let bot = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot, None, None)).await;
+ let replica = ReplicaId::mint();
+ let claim = claimed(repo.claim(session.id, replica).await.unwrap());
+ let_heartbeat_go_stale(&pool, replica).await;
+ let mut heartbeat = pool.begin().await.unwrap();
+ sqlx::query!(
+ "UPDATE harness_replica SET last_heartbeat_at = now() WHERE id = $1",
+ replica.as_uuid()
+ )
+ .execute(&mut *heartbeat)
+ .await
+ .unwrap();
+ let recovering = repo.clone();
+ let mut pending = tokio::spawn(async move { recovering.disconnect_stale_claim(claim).await });
+ assert!(
+ tokio::time::timeout(std::time::Duration::from_millis(50), &mut pending)
+ .await
+ .is_err()
+ );
+ heartbeat.commit().await.unwrap();
+ assert!(pending.await.unwrap().unwrap().is_none());
+ repo.create_fenced(fenced_log(session.id), &claim)
+ .await
+ .unwrap();
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn successor_claim_wins_while_recovery_waits_for_the_session_lock(pool: PgPool) {
+ let repo = test_repo(&pool);
+ let bot = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot, None, None)).await;
+ let old = ReplicaId::mint();
+ let claim = claimed(repo.claim(session.id, old).await.unwrap());
+ let_heartbeat_go_stale(&pool, old).await;
+ let successor = ReplicaId::mint();
+ repo.heartbeat(successor, None).await.unwrap();
+ let mut takeover = pool.begin().await.unwrap();
+ sqlx::query!("UPDATE agent_session SET manager_replica_id = $2, manager_fence = manager_fence + 1 WHERE id = $1", session.id.as_uuid(), successor.as_uuid()).execute(&mut *takeover).await.unwrap();
+ let recovering = repo.clone();
+ let mut pending = tokio::spawn(async move { recovering.disconnect_stale_claim(claim).await });
+ assert!(
+ tokio::time::timeout(std::time::Duration::from_millis(50), &mut pending)
+ .await
+ .is_err()
+ );
+ takeover.commit().await.unwrap();
+ assert!(pending.await.unwrap().unwrap().is_none());
+ assert!(repo.list_by_session(session.id).await.unwrap().is_empty());
+ assert_eq!(
+ repo.lease_view(session.id, successor)
+ .await
+ .unwrap()
+ .holder
+ .unwrap()
+ .replica,
+ successor
+ );
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn simultaneous_recoveries_append_only_one_disconnect(pool: PgPool) {
+ let repo = test_repo(&pool);
+ let bot = create_test_bot(&pool).await;
+ let session = create_session(&repo, new_session(bot, None, None)).await;
+ let replica = ReplicaId::mint();
+ let claim = claimed(repo.claim(session.id, replica).await.unwrap());
+ let_heartbeat_go_stale(&pool, replica).await;
+ let (left, right) = tokio::join!(
+ repo.disconnect_stale_claim(claim),
+ repo.disconnect_stale_claim(claim),
+ );
+ assert_eq!(
+ usize::from(left.unwrap().is_some()) + usize::from(right.unwrap().is_some()),
+ 1
+ );
+ assert_eq!(repo.list_by_session(session.id).await.unwrap().len(), 1);
+}
diff --git a/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.down.sql b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.down.sql
new file mode 100644
index 0000000000..261c9f780d
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.down.sql
@@ -0,0 +1,2 @@
+-- no-transaction
+DROP INDEX CONCURRENTLY IF EXISTS agent_session_manager_replica_idx;
diff --git a/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.up.sql b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.up.sql
new file mode 100644
index 0000000000..3f69196f15
--- /dev/null
+++ b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.up.sql
@@ -0,0 +1,5 @@
+-- no-transaction
+-- Recovery only scans claimed sessions; released history is not part of the sweep.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS agent_session_manager_replica_idx
+ ON agent_session (manager_replica_id, id)
+ WHERE manager_replica_id IS NOT NULL;
diff --git a/docs/AGENT_GUIDE/ai-chat.md b/docs/AGENT_GUIDE/ai-chat.md
index f0081f114d..abd96bee68 100644
--- a/docs/AGENT_GUIDE/ai-chat.md
+++ b/docs/AGENT_GUIDE/ai-chat.md
@@ -1015,6 +1015,14 @@ scroll several screens up and confirm output does not pull you down. Scroll down
to reveal the overlay and return to latest. Repeat with a short session and on a
physical phone while opening/dismissing the keyboard, both at latest and in history.
+After a harness server crashes and its lease expires, the session stops showing
+Working and becomes disconnected without requiring a new message. Its transcript
+is preserved; recovery does not replay the prompt or retry tool calls. Open chats
+refresh the durable history while preserving live events arriving during the read,
+so a session that has already resumed stays active. Verify that a transient history
+read failure retries automatically while live messages continue, and that closing
+the chat cancels pending retries.
+
When a session reconnects using ACP load, the last committed conversation stays
visible while history is reconstructed. A successful load replaces the transcript
once, including prompts, thoughts, and tool results; it does not append another
diff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rs
index 54eee2a5bc..5f84446fe1 100644
--- a/services/agent_harness_service/src/main.rs
+++ b/services/agent_harness_service/src/main.rs
@@ -1081,7 +1081,10 @@ async fn run() -> anyhow::Result<()> {
AgentSessionServiceImpl::new(
session_repo.clone(),
FoldedMessageService::new(session_repo.clone()),
- ConnectionGatewayAgentSessionRealtime::new(connection_gateway, session_audience),
+ ConnectionGatewayAgentSessionRealtime::new(
+ connection_gateway.clone(),
+ session_audience.clone(),
+ ),
NoOpAgentSessionNameGenerator,
Arc::new(NoOpTurnObserver),
lifecycle_publisher,
@@ -1215,6 +1218,31 @@ async fn run() -> anyhow::Result<()> {
}
});
+ // A hard abort cannot emit a disconnect. Reconcile expired leases so
+ // viewers stop showing a turn as working without issuing another command.
+ let recovery_repo = session_repo.clone();
+ let recovery_realtime = ConnectionGatewayAgentSessionRealtime::new(
+ connection_gateway.clone(),
+ session_audience.clone(),
+ );
+ let recovery = tokio::spawn(async move {
+ let mut ticker =
+ tokio::time::interval(agent_session::domain::ports::REPLICA_HEARTBEAT_INTERVAL);
+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);
+ loop {
+ ticker.tick().await;
+ if let Err(error) = agent_session::domain::recovery::recover_stale_sessions(
+ &recovery_repo,
+ &recovery_realtime,
+ std::num::NonZeroUsize::new(100).expect("nonzero recovery batch"),
+ )
+ .await
+ {
+ tracing::warn!(error = ?error, "failed to recover abandoned agent sessions");
+ }
+ }
+ });
+
// Keep trigger generation in this deployment while retaining Kafka as the
// boundary between channel events and harness commands.
let mut trigger = tokio::spawn(trigger::supervise(
@@ -1434,6 +1462,7 @@ async fn run() -> anyhow::Result<()> {
trigger.abort();
egress_http.abort();
heartbeat.abort();
+ recovery.abort();
runtime_commands.abort();
let stop_failures = container_shutdown.shutdown_all().await;
if stop_failures > 0 {