Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(databases): recover sessions after harness crashes (#7417)
Wolf Mermelstein0b322c3f1b92
diff --git a/.sqlx/query-1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577.json b/.sqlx/query-1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577.jsonnew file mode 100644index 0000000000..e1752b3d8d--- /dev/null+++ b/.sqlx/query-1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE harness_replica SET last_heartbeat_at = now() WHERE id = $1",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "1f6ed68aaa85344fb41877eed44c812033c50618fdbf219db80a117728d55577"+}diff --git a/.sqlx/query-2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66.json b/.sqlx/query-2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66.jsonnew file mode 100644index 0000000000..7c691a3b02--- /dev/null+++ b/.sqlx/query-2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE agent_session_log SET created_at = created_at + interval '1 second' WHERE agent_session_id = $1",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "2ea2a143eab94461f62be76320578f818dd75d5e15944b8775a07e48c43edb66"+}diff --git a/.sqlx/query-371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8.json b/.sqlx/query-371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8.jsonnew file mode 100644index 0000000000..735d41a923--- /dev/null+++ b/.sqlx/query-371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE agent_session AS session\n SET manager_replica_id = NULL,\n manager_fence = manager_fence + 1,\n status = $4, status_event_name = $5, turn_state = $6,\n modified_at = now()\n WHERE session.id = $1\n AND session.manager_replica_id = $2 AND session.manager_fence = $3\n AND EXISTS (\n SELECT 1 FROM harness_replica AS replica\n WHERE replica.id = session.manager_replica_id\n AND replica.last_heartbeat_at <= now() - make_interval(secs => $7)\n )\n RETURNING session.id\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Int8",+ "Text",+ "Text",+ "Text",+ "Float8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "371919c2f02efd5c97e1e6984b71befa1b08b98324811df8690ab8cfb93b4db8"+}diff --git a/.sqlx/query-4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580.json b/.sqlx/query-4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580.jsonnew file mode 100644index 0000000000..8263e98949--- /dev/null+++ b/.sqlx/query-4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT id FROM harness_replica WHERE id = $1 FOR UPDATE",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "4d39d272b9f8678f700d610d709d23f58be0641f0fdb1423f4c5ee05d2ff2580"+}diff --git a/.sqlx/query-6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c.json b/.sqlx/query-6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c.jsonnew file mode 100644index 0000000000..0287ac34b0--- /dev/null+++ b/.sqlx/query-6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "\n WITH replica AS (\n INSERT INTO harness_replica (id, last_heartbeat_at)\n VALUES ($2, now())\n ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()\n RETURNING id\n )\n UPDATE agent_session\n SET manager_replica_id = $2,\n manager_fence = manager_fence + 1,\n modified_at = now()\n FROM replica\n WHERE agent_session.id = $1 AND replica.id = $2\n AND (\n manager_replica_id IS NULL\n OR manager_replica_id = $2\n OR NOT EXISTS (\n SELECT 1 FROM harness_replica live\n WHERE live.id = agent_session.manager_replica_id\n AND live.last_heartbeat_at > now() - make_interval(secs => $3)\n AND live.draining_at IS NULL\n )\n )\n RETURNING manager_fence\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "manager_fence",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Float8"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "6e63c4aee5b0f3ac09cfa9ffa74822979b9683912ff2e8e533ed3d500b5a470c"+}diff --git a/.sqlx/query-8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108.json b/.sqlx/query-8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108.jsonnew file mode 100644index 0000000000..64144677a0--- /dev/null+++ b/.sqlx/query-8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108.json@@ -0,0 +1,35 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT session.id, session.manager_replica_id AS \"replica!\", session.manager_fence\n FROM agent_session AS session\n JOIN harness_replica AS replica ON replica.id = session.manager_replica_id\n WHERE session.manager_replica_id IS NOT NULL\n AND replica.last_heartbeat_at <= now() - make_interval(secs => $1)\n ORDER BY session.manager_replica_id, session.id\n LIMIT $2\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "replica!",+ "type_info": "Uuid"+ },+ {+ "ordinal": 2,+ "name": "manager_fence",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Float8",+ "Int8"+ ]+ },+ "nullable": [+ false,+ true,+ false+ ]+ },+ "hash": "8f52bf06163f2e1faf97cf5fe59f7b1931138ca4496e88f33c6e1f11b2cfb108"+}diff --git a/.sqlx/query-bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee.json b/.sqlx/query-bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee.jsonnew file mode 100644index 0000000000..f41ad95457--- /dev/null+++ b/.sqlx/query-bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee.json@@ -0,0 +1,32 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)\n SELECT frame.id, $1, frame.user_id, frame.direction, frame.content,\n stamp.created_at + (frame.ordinality - 1) * interval '1 microsecond'\n FROM UNNEST($2::uuid[], $3::text[], $4::text[], $5::jsonb[])\n WITH ORDINALITY AS frame(id, user_id, direction, content, ordinality)\n CROSS JOIN (\n SELECT GREATEST(statement_timestamp(), (\n SELECT created_at + interval '1 microsecond'\n FROM agent_session_log WHERE agent_session_id = $1\n ORDER BY created_at DESC, id DESC LIMIT 1\n )) AS created_at\n ) AS stamp\n RETURNING id, created_at\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "created_at",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "UuidArray",+ "TextArray",+ "TextArray",+ "JsonbArray"+ ]+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "bb979ea2a2e956068325620d27daa7d28f34613dd595c17bf9d2081087701aee"+}diff --git a/.sqlx/query-c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274.json b/.sqlx/query-c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274.jsonnew file mode 100644index 0000000000..df78d2d982--- /dev/null+++ b/.sqlx/query-c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274.json@@ -0,0 +1,15 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE agent_session SET manager_replica_id = $2, manager_fence = manager_fence + 1 WHERE id = $1",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "c5dfb8f0e980af93d0bcf1074cbdb0f51245d49823426957bba536448decb274"+}diff --git a/.sqlx/query-eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764.json b/.sqlx/query-eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764.jsonnew file mode 100644index 0000000000..303a27c997--- /dev/null+++ b/.sqlx/query-eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764.json@@ -0,0 +1,26 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)\n VALUES ($1, $2, $3, $4, $5, GREATEST(clock_timestamp(), (\n SELECT created_at + interval '1 microsecond'\n FROM agent_session_log WHERE agent_session_id = $2\n ORDER BY created_at DESC, id DESC LIMIT 1\n )))\n RETURNING created_at\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "created_at",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Text",+ "Text",+ "Jsonb"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "eb60dd19b82fe337de7932ee2b6562f97f2c87c69a471dd2121e9de03bbe0764"+}diff --git a/.sqlx/query-f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45.json b/.sqlx/query-f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45.jsonnew file mode 100644index 0000000000..de5b20be53--- /dev/null+++ b/.sqlx/query-f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45.json@@ -0,0 +1,26 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)\n VALUES ($1, $2, $3, $4, $5, GREATEST(statement_timestamp(), (\n SELECT created_at + interval '1 microsecond'\n FROM agent_session_log WHERE agent_session_id = $2\n ORDER BY created_at DESC, id DESC LIMIT 1\n )))\n RETURNING created_at\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "created_at",+ "type_info": "Timestamptz"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid",+ "Text",+ "Text",+ "Jsonb"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "f9b31bd7f8a1e12e7e547c9921e8f0ce8042a5dcbad0378ced090d5f88395b45"+}diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.tsindex 0f5235a15c..07546f47c8 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts@@ -32,6 +32,21 @@ const socket = vi.hoisted(() => ({}),}));+const updates = vi.hoisted(() => ({+ listeners: new Set<(event: { agentSessionId: string }) => void>(),+}));+vi.mock('@queries/agent-session/session-metadata-sync', () => ({+ subscribeAgentSessionUpdated: (+ listener: (event: { agentSessionId: string }) => void+ ) => {+ updates.listeners.add(listener);+ return () => updates.listeners.delete(listener);+ },+}));+function invalidate(id = SESSION) {+ for (const listener of updates.listeners) listener({ agentSessionId: id });+}+vi.mock('@core/agent-fold/client', () => fold);vi.mock('@service-agent-harness/client', () => ({agentHarnessServiceClient: harness,@@ -84,6 +99,7 @@ beforeEach(() => {vi.clearAllMocks();resetSessionTurns();socket.listeners.clear();+ updates.listeners.clear();// Instances are shared and refcounted, so a test that fails before its// `release()` would hand the next one a session that is already loaded.for (@@ -316,6 +332,200 @@ describe('AgentSession', () => {expect(AgentSession.get(SESSION)).toBeUndefined();});+ it('reloads committed recovery state and preserves frames received during the read', async () => {+ const live = AgentSession.acquire(SESSION);+ await live.load();+ const log = deferred<LogResult>();+ harness.getLog.mockReturnValueOnce(log.promise);+ invalidate('another-session');+ expect(harness.getLog).toHaveBeenCalledTimes(1);+ invalidate();+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(3)] });+ expect(inputs().filter((input) => input.kind === 'confirmed')).toEqual([]);+ log.resolve(logOf([row(1), row(2)]));+ await vi.waitFor(() =>+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(3) })+ );+ expect(inputs().slice(-2)).toEqual([+ { kind: 'snapshot', rows: [row(1), row(2)] },+ { kind: 'confirmed', row: row(3) },+ ]);+ live.release();+ expect(updates.listeners.size).toBe(0);+ });++ it('coalesces repeated invalidations and catches one received during initial loading', async () => {+ const first = deferred<LogResult>();+ const refresh = deferred<LogResult>();+ harness.getLog+ .mockReturnValueOnce(first.promise)+ .mockReturnValueOnce(refresh.promise)+ .mockResolvedValueOnce(logOf([row(1), row(2), row(3)]));+ const live = AgentSession.acquire(SESSION);+ invalidate();+ first.resolve(logOf([row(1)]));+ await live.load();+ expect(harness.getLog).toHaveBeenCalledTimes(2);+ invalidate();+ invalidate();+ expect(harness.getLog).toHaveBeenCalledTimes(2);+ refresh.resolve(logOf([row(1), row(2)]));+ await vi.waitFor(() =>+ expect(inputs().at(-1)).toEqual({+ kind: 'snapshot',+ rows: [row(1), row(2), row(3)],+ })+ );+ expect(harness.getLog).toHaveBeenCalledTimes(3);+ live.release();+ });++ it('releases buffered live frames after a failed refresh', async () => {+ const live = AgentSession.acquire(SESSION);+ await live.load();+ const refresh = deferred<LogResult>();+ harness.getLog.mockReturnValueOnce(refresh.promise);+ invalidate();+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(2)] });+ refresh.resolve(err([{ code: 'INTERNAL' }]));+ await vi.waitFor(() =>+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(2) })+ );+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(3)] });+ await live.snapshot();+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(3) });+ live.release();+ });++ it.each(['result', 'transport'])(+ 'retries a failed refresh without blocking live frames (%s)',+ async (failure) => {+ vi.useFakeTimers();+ const live = AgentSession.acquire(SESSION);+ try {+ await live.load();+ if (failure === 'result')+ harness.getLog.mockResolvedValueOnce(err([{ code: 'INTERNAL' }]));+ else harness.getLog.mockRejectedValueOnce(new Error('network down'));+ harness.getLog.mockResolvedValueOnce(logOf([row(1), row(2), row(3)]));+ invalidate();+ await vi.advanceTimersByTimeAsync(0);+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(2)] });+ await live.snapshot();+ expect(inputs().at(-1)).toEqual({ kind: 'confirmed', row: row(2) });+ expect(harness.getLog).toHaveBeenCalledTimes(2);+ await vi.advanceTimersByTimeAsync(1_000);+ expect(inputs().at(-1)).toEqual({+ kind: 'snapshot',+ rows: [row(1), row(2), row(3)],+ });+ await vi.advanceTimersByTimeAsync(60_000);+ expect(harness.getLog).toHaveBeenCalledTimes(3);+ } finally {+ live.release();+ vi.useRealTimers();+ }+ }+ );++ it('bounds failed refresh retries and gives a new invalidation a fresh budget', async () => {+ vi.useFakeTimers();+ const live = AgentSession.acquire(SESSION);+ try {+ await live.load();+ harness.getLog.mockResolvedValue(err([{ code: 'INTERNAL' }]));+ invalidate();+ await vi.advanceTimersByTimeAsync(60_000);+ expect(harness.getLog).toHaveBeenCalledTimes(5);+ invalidate();+ await vi.advanceTimersByTimeAsync(60_000);+ expect(harness.getLog).toHaveBeenCalledTimes(9);+ } finally {+ live.release();+ vi.useRealTimers();+ }+ });++ it('cancels a scheduled refresh retry when the last view releases it', async () => {+ vi.useFakeTimers();+ const live = AgentSession.acquire(SESSION);+ try {+ await live.load();+ harness.getLog.mockResolvedValue(err([{ code: 'INTERNAL' }]));+ invalidate();+ await vi.advanceTimersByTimeAsync(0);+ live.release();+ await vi.advanceTimersByTimeAsync(60_000);+ expect(harness.getLog).toHaveBeenCalledTimes(2);+ } finally {+ if (AgentSession.get(SESSION)) live.release();+ vi.useRealTimers();+ }+ });++ it('does not retry an access-denied refresh', async () => {+ vi.useFakeTimers();+ const live = AgentSession.acquire(SESSION);+ try {+ await live.load();+ harness.getLog.mockResolvedValue(err([{ code: 'FORBIDDEN' }]));+ invalidate();+ await vi.advanceTimersByTimeAsync(60_000);+ expect(harness.getLog).toHaveBeenCalledTimes(2);+ } finally {+ live.release();+ vi.useRealTimers();+ }+ });++ it.each([false, true])(+ 'retains optimistic actions during refresh (replacement ID=%s)',+ async (replacement) => {+ if (replacement)+ harness.control.mockResolvedValue(+ ok({ actionId: 'server-id', status: 'sent' })+ );+ const live = AgentSession.acquire(SESSION);+ await live.load();+ const refresh = deferred<LogResult>();+ harness.getLog.mockReturnValueOnce(refresh.promise);+ invalidate();+ await live.issue({ type: 'prompt', prompt: 'Continue' });+ expect(inputs().filter((input) => input.kind === 'speculated')).toEqual(+ []+ );+ refresh.resolve(logOf([row(1)]));+ await vi.waitFor(() =>+ expect(inputs().at(-1)).toMatchObject({+ kind: 'speculated',+ action: { type: 'prompt', prompt: 'Continue' },+ })+ );+ if (replacement) {+ const issued = harness.control.mock.calls.at(-1)![1].actionId;+ expect(inputs().slice(-2)).toMatchObject([+ { kind: 'retracted', actionId: issued },+ { kind: 'speculated', actionId: 'server-id' },+ ]);+ }+ live.release();+ }+ );++ it('does not recreate a released session when a refresh completes', async () => {+ const live = AgentSession.acquire(SESSION);+ await live.load();+ const refresh = deferred<LogResult>();+ harness.getLog.mockReturnValueOnce(refresh.promise);+ invalidate();+ live.release();+ const before = inputs();+ refresh.resolve(logOf([row(1), row(2)]));+ await settle();+ expect(inputs()).toEqual(before);+ expect(updates.listeners.size).toBe(0);+ });+it('re-snapshots when the socket reopens', async () => {const live = AgentSession.acquire(SESSION);await live.load();diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.tsindex 8948223088..72a8182621 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts@@ -20,8 +20,10 @@ import {readSession,type SessionFoldSnapshot,} from '@core/agent-fold/client';+import { fetchAgentSessionLog } from '@queries/agent-session/log';import { subscribeSocketSessionStarted } from '@queries/agent-session/queue-sync';import type { AgentSessionLogEvent } from '@queries/agent-session/realtime-protocol';+import { subscribeAgentSessionUpdated } from '@queries/agent-session/session-metadata-sync';import type {FoldedStreamEvent,TurnState,@@ -90,6 +92,8 @@ function occupiesTurn(action: AgentAction): boolean {return action.type === 'prompt' || action.type === 'compact';}+const RESYNC_RETRY_DELAYS_MS = [1_000, 3_000, 10_000];+export class AgentSession {private static readonly open = new Map<string, AgentSession>();@@ -134,6 +138,11 @@ export class AgentSession {private buffered: FoldInput[] = [];private readonly listeners = new Set<AgentSessionListener>();private readonly unsubscribeSocket: () => void;+ private readonly unsubscribeUpdated: () => void;+ private syncing = false;+ private resyncRequested = false;+ private resyncRetry = 0;+ private resyncTimer?: ReturnType<typeof setTimeout>;/*** Serializes worker pushes so inputs reach the machine in the order this* class saw them, even though each push is its own await.@@ -163,6 +172,9 @@ export class AgentSession {this.unsubscribeSocket = subscribeSocketSessionStarted(() => {void this.resync();});+ this.unsubscribeUpdated = subscribeAgentSessionUpdated((event) => {+ if (event.agentSessionId === this.id) void this.resync();+ });this.trace = new SessionLoadTrace(id);this.loading = this.startLoad();}@@ -238,7 +250,7 @@ export class AgentSession {// confirms it by content whatever id the harness accepted it under.const accepted = result.value.actionId;if (accepted !== actionId && action.type !== 'stop') {- void this.apply([+ void this.enqueueAll([{ kind: 'retracted', actionId },{kind: 'speculated',@@ -343,6 +355,8 @@ export class AgentSession {this.trace.end('released');this.listeners.clear();this.unsubscribeSocket();+ this.unsubscribeUpdated();+ clearTimeout(this.resyncTimer);closeSession(this.id);}@@ -366,7 +380,7 @@ export class AgentSession {private async fetchAndFold(): Promise<AgentSessionRecord> {const [session, log] = await Promise.all([agentHarnessServiceClient.get(this.id),- agentHarnessServiceClient.getLog(this.id),+ fetchAgentSessionLog(this.id),]);if (session.isErr()) {if (accessDenied(session.error)) {@@ -385,14 +399,10 @@ export class AgentSession {await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);// Inputs can keep arriving while each push is in flight; drain until a// check finds nothing, then flip ready so the next one goes straight in.- while (this.buffered.length > 0) {- const inputs = this.buffered;- this.buffered = [];- await this.apply(inputs);- }- this.ready = true;+ await this.drainBuffered();this.trace.folded(foldStartedAt);this.setTurn((await readSession(this.id)).metadata.turn);+ if (this.resyncRequested) void this.resync();return { session: session.value, bot: log.value.bot };}@@ -401,11 +411,54 @@ export class AgentSession {* while it was down. Refetch, and let the machine reconcile the overlap* and settle any speculation the log confirmed meanwhile.*/- private async resync(): Promise<void> {- if (!this.ready || this.closed) return;- const log = await agentHarnessServiceClient.getLog(this.id);- if (log.isErr() || this.closed) return;- await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);+ private async resync(retrying = false): Promise<void> {+ if (this.closed) return;+ if (!retrying) {+ clearTimeout(this.resyncTimer);+ this.resyncRetry = 0;+ }+ this.resyncRequested = true;+ if (!this.ready || this.syncing) return;+ this.syncing = true;+ this.ready = false;+ let failed = false;+ try {+ do {+ this.resyncRequested = false;+ const log = await fetchAgentSessionLog(this.id);+ if (this.closed) return;+ failed = log.isErr() && !accessDenied(log.error);+ if (log.isOk()) {+ await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);+ this.resyncRetry = 0;+ }+ } while (this.resyncRequested);+ } catch (error: unknown) {+ failed = true;+ console.warn('[agent-session] log could not be refreshed', error);+ } finally {+ // A snapshot may predate frames received during its fetch. Replay those+ // frames after it, retaining speculation and the fold's overlap deduping.+ await this.drainBuffered();+ this.syncing = false;+ if (!this.closed && this.resyncRequested) {+ void this.resync(true);+ } else if (!this.closed && failed) {+ const delay = RESYNC_RETRY_DELAYS_MS[this.resyncRetry++];+ if (delay !== undefined)+ this.resyncTimer = setTimeout(() => void this.resync(true), delay);+ }+ }+ }++ private async drainBuffered(): Promise<void> {+ while (!this.closed && this.buffered.length > 0) {+ const inputs = this.buffered;+ this.buffered = [];+ await this.apply(inputs);+ }+ // No await between observing an empty buffer and accepting live inputs.+ if (!this.closed) this.ready = true;}private enqueue(input: FoldInput): Promise<void> {diff --git a/apps/web/src/lib/queries/agent-session/log.ts b/apps/web/src/lib/queries/agent-session/log.tsnew file mode 100644index 0000000000..8b90f72be9--- /dev/null+++ b/apps/web/src/lib/queries/agent-session/log.ts@@ -0,0 +1,6 @@+import { agentHarnessServiceClient } from '@service-agent-harness/client';++/** Read the authoritative log; the shared fold owns its in-memory projection. */+export function fetchAgentSessionLog(sessionId: string) {+ return agentHarnessServiceClient.getLog(sessionId);+}diff --git a/crates/agent_session/src/domain/mod.rs b/crates/agent_session/src/domain/mod.rsindex e2095d4366..0e71a176e2 100644--- a/crates/agent_session/src/domain/mod.rs+++ b/crates/agent_session/src/domain/mod.rs@@ -31,3 +31,6 @@ pub mod turn_state;/// Repository-matched working branches reported by a session's runtime.pub mod working_branch;++/// Recovery of sessions abandoned by crashed replicas.+pub mod recovery;diff --git a/crates/agent_session/src/domain/ports.rs b/crates/agent_session/src/domain/ports.rsindex b921590f3a..5519de607f 100644--- a/crates/agent_session/src/domain/ports.rs+++ b/crates/agent_session/src/domain/ports.rs@@ -827,7 +827,7 @@ pub trait AgentSessionRealtime {event: LogAppended,) -> impl Future<Output = Result<(), rootcause::Report>> + Send;- /// Tell viewers to refetch changed session metadata.+ /// Tell viewers to refetch changed session metadata and the durable log.fn publish_updated(&self,_session: AgentSessionId,diff --git a/crates/agent_session/src/domain/recovery.rs b/crates/agent_session/src/domain/recovery.rsnew file mode 100644index 0000000000..6d3630fdf5--- /dev/null+++ b/crates/agent_session/src/domain/recovery.rs@@ -0,0 +1,60 @@+//! Close abandoned runtime connections without replaying their work.++use std::num::NonZeroUsize;++use super::error::Result;+use super::model::{SessionClaim, StoredAgentSessionLog};+use super::ports::AgentSessionRealtime;++/// Atomic persistence boundary for recovery after a replica stops heartbeating.+pub trait StaleSessionRepo: Send + Sync {+ /// Return at most `limit` claims whose holder has missed the lease deadline.+ /// A draining but heartbeating holder is still alive and must be excluded.+ fn stale_claims(+ &self,+ limit: NonZeroUsize,+ ) -> impl Future<Output = Result<Vec<SessionClaim>>> + Send;++ /// Recheck the exact holder, fence, and heartbeat, then atomically revoke+ /// the claim, append `Disconnected`, and project its turn state. Return+ /// `None` if a heartbeat, takeover, release, or another recovery won.+ /// Revocation increments the fence so the old actor can never append again.+ fn disconnect_stale_claim(+ &self,+ claim: SessionClaim,+ ) -> impl Future<Output = Result<Option<StoredAgentSessionLog>>> + Send;+}++/// Reconcile one bounded batch and invalidate viewers after durable recovery.+/// This only closes runtime state: it never starts a runtime or retries work.+#[tracing::instrument(err, skip(repo, realtime))]+pub async fn recover_stale_sessions<R: StaleSessionRepo, Rt: AgentSessionRealtime>(+ repo: &R,+ realtime: &Rt,+ limit: NonZeroUsize,+) -> Result<usize> {+ let mut recovered = 0;+ for claim in repo.stale_claims(limit).await? {+ match repo.disconnect_stale_claim(claim).await {+ Ok(Some(_)) => {}+ Ok(None) => continue,+ Err(error) => {+ tracing::warn!(error = ?error, session_id = %claim.session, recovered, "failed to disconnect abandoned agent session");+ continue;+ }+ };+ recovered += 1;+ // A successor can append AcpReady immediately after the commit. Never+ // publish a delayed Disconnected live frame over that newer state:+ // viewers must refetch the log in durable order instead.+ realtime+ .publish_updated(claim.session)+ .await+ .inspect_err(|error| {+ tracing::warn!(error = ?error, session_id = %claim.session, "failed to publish recovered session disconnect");+ })+ .ok();+ tracing::info!(session_id = %claim.session, replica_id = %claim.replica, fence = claim.fence.0, "disconnected abandoned agent session");+ }+ Ok(recovered)+}diff --git a/crates/agent_session/src/outbound/postgres.rs b/crates/agent_session/src/outbound/postgres.rsindex 97f6bf3b8f..308ab3c842 100644--- a/crates/agent_session/src/outbound/postgres.rs+++ b/crates/agent_session/src/outbound/postgres.rs@@ -11,6 +11,7 @@ mod test;mod pull_request;mod queue;+mod recovery;mod sharing;mod turn_state;mod working_branch;@@ -1251,17 +1252,24 @@ impl<B: BotFacts + 'static> AgentSessionLogRepo for PgAgentSessionRepo<B> {return Err(AgentSessionError::FencedOut(session));}- // One transaction means one `now()`, so the batch is spread over- // consecutive microseconds in append order: readers order by- // `(created_at, id)`, and the ids are v7 without a monotonic- // counter, so same-instant rows would otherwise interleave.+ // Start after the durable tail while holding the session lock, then+ // spread the batch over consecutive microseconds. Transaction start+ // time can predate a lock wait or a previous batch's synthetic tail;+ // readers must see append order rather than UUID tie-breaking.let stamped = sqlx::query!(r#"INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)SELECT frame.id, $1, frame.user_id, frame.direction, frame.content,- now() + (frame.ordinality - 1) * interval '1 microsecond'+ stamp.created_at + (frame.ordinality - 1) * interval '1 microsecond'FROM UNNEST($2::uuid[], $3::text[], $4::text[], $5::jsonb[])WITH ORDINALITY AS frame(id, user_id, direction, content, ordinality)+ CROSS JOIN (+ SELECT GREATEST(statement_timestamp(), (+ SELECT created_at + interval '1 microsecond'+ FROM agent_session_log WHERE agent_session_id = $1+ ORDER BY created_at DESC, id DESC LIMIT 1+ )) AS created_at+ ) AS stampRETURNING id, created_at"#,session.as_uuid(),@@ -1357,8 +1365,12 @@ impl<B: BotFacts + 'static> AgentSessionLogRepo for PgAgentSessionRepo<B> {let id = macro_uuid::generate_uuid_v7();let created_at = sqlx::query_scalar!(r#"- INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content)- VALUES ($1, $2, $3, $4, $5)+ INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)+ VALUES ($1, $2, $3, $4, $5, GREATEST(statement_timestamp(), (+ SELECT created_at + interval '1 microsecond'+ FROM agent_session_log WHERE agent_session_id = $2+ ORDER BY created_at DESC, id DESC LIMIT 1+ )))RETURNING created_at"#,id,@@ -1543,12 +1555,14 @@ impl<B: BotFacts + 'static> SessionOwnership for PgAgentSessionRepo<B> {INSERT INTO harness_replica (id, last_heartbeat_at)VALUES ($2, now())ON CONFLICT (id) DO UPDATE SET last_heartbeat_at = now()+ RETURNING id)UPDATE agent_sessionSET manager_replica_id = $2,manager_fence = manager_fence + 1,modified_at = now()- WHERE id = $1+ FROM replica+ WHERE agent_session.id = $1 AND replica.id = $2AND (manager_replica_id IS NULLOR manager_replica_id = $2diff --git a/crates/agent_session/src/outbound/postgres/recovery.rs b/crates/agent_session/src/outbound/postgres/recovery.rsnew file mode 100644index 0000000000..c08dbd7217--- /dev/null+++ b/crates/agent_session/src/outbound/postgres/recovery.rs@@ -0,0 +1,125 @@+use super::*;+use crate::domain::recovery::StaleSessionRepo;+use agent_fold::domain::model::TurnState;++impl<B: BotFacts + 'static> StaleSessionRepo for PgAgentSessionRepo<B> {+ async fn stale_claims(&self, limit: NonZeroUsize) -> Result<Vec<SessionClaim>> {+ let rows = sqlx::query!(+ r#"+ SELECT session.id, session.manager_replica_id AS "replica!", session.manager_fence+ FROM agent_session AS session+ JOIN harness_replica AS replica ON replica.id = session.manager_replica_id+ WHERE session.manager_replica_id IS NOT NULL+ AND replica.last_heartbeat_at <= now() - make_interval(secs => $1)+ ORDER BY session.manager_replica_id, session.id+ LIMIT $2+ "#,+ REPLICA_STALE_AFTER.as_secs_f64(),+ i64::try_from(limit.get()).unwrap_or(i64::MAX),+ )+ .fetch_all(&self.pool)+ .await+ .context("list abandoned session claims")?;+ Ok(rows+ .into_iter()+ .map(|row| SessionClaim {+ session: AgentSessionId::new_from_uuid(row.id),+ replica: ReplicaId::from_uuid(row.replica),+ fence: ManagerFence(row.manager_fence),+ })+ .collect())+ }++ async fn disconnect_stale_claim(+ &self,+ claim: SessionClaim,+ ) -> Result<Option<StoredAgentSessionLog>> {+ let log = AgentSessionLog {+ agent_session_id: claim.session,+ user_id: None,+ content: Message::ToServer(ToServerMessage::Event {+ event: SystemEvent::Disconnected,+ }),+ };+ let (direction, content) = message_columns(&log.content)?;+ let status = SessionStatus::Event(SystemEvent::Disconnected);+ let (status, status_event_name) = status_columns(&status);+ let mut transaction = self+ .pool+ .begin()+ .await+ .context("begin abandoned session recovery")?;+ // Match claim's replica-before-session lock order. A heartbeat that+ // commits while this waits must be seen by the next statement; one+ // arriving after this lock waits until revocation is durable.+ sqlx::query_scalar!(+ "SELECT id FROM harness_replica WHERE id = $1 FOR UPDATE",+ claim.replica.as_uuid(),+ )+ .fetch_optional(&mut *transaction)+ .await+ .context("lock abandoned replica heartbeat")?;+ // Claim changes and live appends lock this same row. An intervening+ // owner/fence change fails the predicate even after waiting for its+ // transaction; an old process that wakes later is fenced out.+ let revoked = sqlx::query_scalar!(+ r#"+ UPDATE agent_session AS session+ SET manager_replica_id = NULL,+ manager_fence = manager_fence + 1,+ status = $4, status_event_name = $5, turn_state = $6,+ modified_at = now()+ WHERE session.id = $1+ AND session.manager_replica_id = $2 AND session.manager_fence = $3+ AND EXISTS (+ SELECT 1 FROM harness_replica AS replica+ WHERE replica.id = session.manager_replica_id+ AND replica.last_heartbeat_at <= now() - make_interval(secs => $7)+ )+ RETURNING session.id+ "#,+ claim.session.as_uuid(),+ claim.replica.as_uuid(),+ claim.fence.0,+ status,+ status_event_name,+ TurnState::Disconnected.as_ref(),+ REPLICA_STALE_AFTER.as_secs_f64(),+ )+ .fetch_optional(&mut *transaction)+ .await+ .context("revoke abandoned session claim")?;+ if revoked.is_none() {+ return Ok(None);+ }+ let id = macro_uuid::generate_uuid_v7();+ let created_at = sqlx::query_scalar!(+ r#"+ INSERT INTO agent_session_log (id, agent_session_id, user_id, direction, content, created_at)+ VALUES ($1, $2, $3, $4, $5, GREATEST(clock_timestamp(), (+ SELECT created_at + interval '1 microsecond'+ FROM agent_session_log WHERE agent_session_id = $2+ ORDER BY created_at DESC, id DESC LIMIT 1+ )))+ RETURNING created_at+ "#,+ id,+ claim.session.as_uuid(),+ Option::<&str>::None,+ direction,+ content,+ )+ .fetch_one(&mut *transaction)+ .await+ .context("append abandoned session disconnect")?;+ transaction+ .commit()+ .await+ .context("commit abandoned session recovery")?;+ Ok(Some(StoredAgentSessionLog {+ id,+ created_at,+ entry: log,+ }))+ }+}diff --git a/crates/agent_session/src/outbound/postgres/test.rs b/crates/agent_session/src/outbound/postgres/test.rsindex 1842d952c5..17d98b71b8 100644--- a/crates/agent_session/src/outbound/postgres/test.rs+++ b/crates/agent_session/src/outbound/postgres/test.rs@@ -1,5 +1,6 @@use super::*;mod queue;+mod recovery;mod search;mod user_cleanup;mod working_branch;diff --git a/crates/agent_session/src/outbound/postgres/test/recovery.rs b/crates/agent_session/src/outbound/postgres/test/recovery.rsnew file mode 100644index 0000000000..4dfa63eed9--- /dev/null+++ b/crates/agent_session/src/outbound/postgres/test/recovery.rs@@ -0,0 +1,255 @@+use super::*;+use crate::domain::ports::AgentSessionRealtime;+use crate::domain::recovery::{StaleSessionRepo, recover_stale_sessions};+use agent_fold::domain::lifecycle::LifecycleFold;+use agent_fold::domain::model::TurnState;+use agent_fold::testing::{TURN, parse_log_as};+use std::sync::Mutex;++#[derive(Default)]+struct Realtime(Mutex<Vec<AgentSessionId>>);++impl AgentSessionRealtime for Realtime {+ async fn publish(+ &self,+ _event: crate::domain::model::LogAppended,+ ) -> std::result::Result<(), rootcause::Report> {+ panic!("recovery must invalidate snapshots, never send stale live frames");+ }++ async fn publish_updated(+ &self,+ session: AgentSessionId,+ ) -> std::result::Result<(), rootcause::Report> {+ self.0.lock().unwrap().push(session);+ Ok(())+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn recovery_preserves_history_closes_stale_turn_and_fences_old_writer(pool: PgPool) {+ let repo = test_repo(&pool);+ let bot = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot, None, None)).await;+ let replica = ReplicaId::mint();+ let claim = claimed(repo.claim(session.id, replica).await.unwrap());+ let mut fold = LifecycleFold::new();+ for frame in parse_log_as(session.id, TURN) {+ let prompt = matches!(&frame.content, Message::ToRuntime(ToRuntimeMessage::Acp(AcpMessage(RawJsonRpcMessage::Request(request)))) if request.method.as_ref() == "session/prompt");+ let _ = fold.push(frame.clone());+ repo.create_projected(+ frame,+ Some(&claim),+ None,+ Some(fold.inner().metadata().turn),+ )+ .await+ .unwrap();+ if prompt {+ break;+ }+ }+ assert_eq!(fold.inner().metadata().turn, TurnState::Running);+ // Batched writes can leave their final microseconds ahead of wall time.+ // Both recovery and the successor must remain after that durable tail.+ sqlx::query!(+ "UPDATE agent_session_log SET created_at = created_at + interval '1 second' WHERE agent_session_id = $1",+ session.id.as_uuid()+ )+ .execute(&pool)+ .await+ .unwrap();+ let before = repo.list_by_session(session.id).await.unwrap();+ let_heartbeat_go_stale(&pool, replica).await;+ let realtime = Realtime::default();+ assert_eq!(+ recover_stale_sessions(&repo, &realtime, NonZeroUsize::new(10).unwrap())+ .await+ .unwrap(),+ 1+ );+ let after = repo.list_by_session(session.id).await.unwrap();+ assert_eq!(after.len(), before.len() + 1);+ assert_eq!(+ before.iter().map(|v| v.id).collect::<Vec<_>>(),+ after[..before.len()]+ .iter()+ .map(|v| v.id)+ .collect::<Vec<_>>()+ );+ let _ = fold.push(after.last().unwrap().entry.clone());+ assert_eq!(fold.inner().metadata().turn, TurnState::Disconnected);+ assert!(after.last().unwrap().created_at > before.last().unwrap().created_at);+ let projected = sqlx::query_scalar!(+ "SELECT turn_state FROM agent_session WHERE id = $1",+ session.id.as_uuid()+ )+ .fetch_one(&pool)+ .await+ .unwrap();+ assert_eq!(projected.as_deref(), Some("disconnected"));+ assert_eq!(realtime.0.lock().unwrap()[0], session.id);+ assert!(matches!(+ repo.create_fenced(fenced_log(session.id), &claim).await,+ Err(AgentSessionError::FencedOut(_))+ ));+ assert_eq!(+ recover_stale_sessions(&repo, &realtime, NonZeroUsize::new(10).unwrap())+ .await+ .unwrap(),+ 0+ );+ assert_eq!(realtime.0.lock().unwrap().len(), 1);+ let resumed = claimed(repo.claim(session.id, ReplicaId::mint()).await.unwrap());+ assert_eq!(resumed.fence.0, claim.fence.0 + 2);+ let ready = repo+ .create_fenced(+ AgentSessionLog {+ agent_session_id: session.id,+ user_id: None,+ content: Message::ToServer(ToServerMessage::Event {+ event: SystemEvent::AcpReady,+ }),+ },+ &resumed,+ )+ .await+ .unwrap();+ assert!(ready.created_at > after.last().unwrap().created_at);+ let batch = repo+ .create_batch_fenced(+ vec![StoredAgentSessionLog {+ id: macro_uuid::generate_uuid_v7(),+ created_at: chrono::Utc::now(),+ entry: fenced_log(session.id),+ }],+ &resumed,+ )+ .await+ .unwrap();+ assert!(batch[0].created_at > ready.created_at);+ let mut replay = LifecycleFold::new();+ for row in repo.list_by_session(session.id).await.unwrap() {+ let _ = replay.push(row.entry);+ }+ assert_ne!(replay.inner().metadata().turn, TurnState::Disconnected);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn recovery_is_bounded_and_leaves_live_and_draining_holders_alone(pool: PgPool) {+ let repo = test_repo(&pool);+ let bot = create_test_bot(&pool).await;+ let live = create_session(&repo, new_session(bot, None, None)).await;+ let live_replica = ReplicaId::mint();+ let live_claim = claimed(repo.claim(live.id, live_replica).await.unwrap());+ repo.begin_draining(live_replica).await.unwrap();+ assert!(+ repo.disconnect_stale_claim(live_claim)+ .await+ .unwrap()+ .is_none()+ );+ for _ in 0..3 {+ let session = create_session(&repo, new_session(bot, None, None)).await;+ let replica = ReplicaId::mint();+ claimed(repo.claim(session.id, replica).await.unwrap());+ let_heartbeat_go_stale(&pool, replica).await;+ }+ let realtime = Realtime::default();+ let limit = NonZeroUsize::new(2).unwrap();+ assert_eq!(+ recover_stale_sessions(&repo, &realtime, limit)+ .await+ .unwrap(),+ 2+ );+ assert_eq!(+ recover_stale_sessions(&repo, &realtime, limit)+ .await+ .unwrap(),+ 1+ );+ assert!(repo.list_by_session(live.id).await.unwrap().is_empty());+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn refreshed_heartbeat_wins_while_recovery_waits_for_its_lock(pool: PgPool) {+ let repo = test_repo(&pool);+ let bot = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot, None, None)).await;+ let replica = ReplicaId::mint();+ let claim = claimed(repo.claim(session.id, replica).await.unwrap());+ let_heartbeat_go_stale(&pool, replica).await;+ let mut heartbeat = pool.begin().await.unwrap();+ sqlx::query!(+ "UPDATE harness_replica SET last_heartbeat_at = now() WHERE id = $1",+ replica.as_uuid()+ )+ .execute(&mut *heartbeat)+ .await+ .unwrap();+ let recovering = repo.clone();+ let mut pending = tokio::spawn(async move { recovering.disconnect_stale_claim(claim).await });+ assert!(+ tokio::time::timeout(std::time::Duration::from_millis(50), &mut pending)+ .await+ .is_err()+ );+ heartbeat.commit().await.unwrap();+ assert!(pending.await.unwrap().unwrap().is_none());+ repo.create_fenced(fenced_log(session.id), &claim)+ .await+ .unwrap();+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn successor_claim_wins_while_recovery_waits_for_the_session_lock(pool: PgPool) {+ let repo = test_repo(&pool);+ let bot = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot, None, None)).await;+ let old = ReplicaId::mint();+ let claim = claimed(repo.claim(session.id, old).await.unwrap());+ let_heartbeat_go_stale(&pool, old).await;+ let successor = ReplicaId::mint();+ repo.heartbeat(successor, None).await.unwrap();+ let mut takeover = pool.begin().await.unwrap();+ sqlx::query!("UPDATE agent_session SET manager_replica_id = $2, manager_fence = manager_fence + 1 WHERE id = $1", session.id.as_uuid(), successor.as_uuid()).execute(&mut *takeover).await.unwrap();+ let recovering = repo.clone();+ let mut pending = tokio::spawn(async move { recovering.disconnect_stale_claim(claim).await });+ assert!(+ tokio::time::timeout(std::time::Duration::from_millis(50), &mut pending)+ .await+ .is_err()+ );+ takeover.commit().await.unwrap();+ assert!(pending.await.unwrap().unwrap().is_none());+ assert!(repo.list_by_session(session.id).await.unwrap().is_empty());+ assert_eq!(+ repo.lease_view(session.id, successor)+ .await+ .unwrap()+ .holder+ .unwrap()+ .replica,+ successor+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn simultaneous_recoveries_append_only_one_disconnect(pool: PgPool) {+ let repo = test_repo(&pool);+ let bot = create_test_bot(&pool).await;+ let session = create_session(&repo, new_session(bot, None, None)).await;+ let replica = ReplicaId::mint();+ let claim = claimed(repo.claim(session.id, replica).await.unwrap());+ let_heartbeat_go_stale(&pool, replica).await;+ let (left, right) = tokio::join!(+ repo.disconnect_stale_claim(claim),+ repo.disconnect_stale_claim(claim),+ );+ assert_eq!(+ usize::from(left.unwrap().is_some()) + usize::from(right.unwrap().is_some()),+ 1+ );+ assert_eq!(repo.list_by_session(session.id).await.unwrap().len(), 1);+}diff --git a/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.down.sql b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.down.sqlnew file mode 100644index 0000000000..261c9f780d--- /dev/null+++ b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.down.sql@@ -0,0 +1,2 @@+-- no-transaction+DROP INDEX CONCURRENTLY IF EXISTS agent_session_manager_replica_idx;diff --git a/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.up.sql b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.up.sqlnew file mode 100644index 0000000000..3f69196f15--- /dev/null+++ b/crates/macro_db_client/migrations/20261002221926_index_agent_session_manager.up.sql@@ -0,0 +1,5 @@+-- no-transaction+-- Recovery only scans claimed sessions; released history is not part of the sweep.+CREATE INDEX CONCURRENTLY IF NOT EXISTS agent_session_manager_replica_idx+ ON agent_session (manager_replica_id, id)+ WHERE manager_replica_id IS NOT NULL;diff --git a/docs/AGENT_GUIDE/ai-chat.md b/docs/AGENT_GUIDE/ai-chat.mdindex f0081f114d..abd96bee68 100644--- a/docs/AGENT_GUIDE/ai-chat.md+++ b/docs/AGENT_GUIDE/ai-chat.md@@ -1015,6 +1015,14 @@ scroll several screens up and confirm output does not pull you down. Scroll downto reveal the overlay and return to latest. Repeat with a short session and on aphysical phone while opening/dismissing the keyboard, both at latest and in history.+After a harness server crashes and its lease expires, the session stops showing+Working and becomes disconnected without requiring a new message. Its transcript+is preserved; recovery does not replay the prompt or retry tool calls. Open chats+refresh the durable history while preserving live events arriving during the read,+so a session that has already resumed stays active. Verify that a transient history+read failure retries automatically while live messages continue, and that closing+the chat cancels pending retries.+When a session reconnects using ACP load, the last committed conversation staysvisible while history is reconstructed. A successful load replaces the transcriptonce, including prompts, thoughts, and tool results; it does not append anotherdiff --git a/services/agent_harness_service/src/main.rs b/services/agent_harness_service/src/main.rsindex 54eee2a5bc..5f84446fe1 100644--- a/services/agent_harness_service/src/main.rs+++ b/services/agent_harness_service/src/main.rs@@ -1081,7 +1081,10 @@ async fn run() -> anyhow::Result<()> {AgentSessionServiceImpl::new(session_repo.clone(),FoldedMessageService::new(session_repo.clone()),- ConnectionGatewayAgentSessionRealtime::new(connection_gateway, session_audience),+ ConnectionGatewayAgentSessionRealtime::new(+ connection_gateway.clone(),+ session_audience.clone(),+ ),NoOpAgentSessionNameGenerator,Arc::new(NoOpTurnObserver),lifecycle_publisher,@@ -1215,6 +1218,31 @@ async fn run() -> anyhow::Result<()> {}});+ // A hard abort cannot emit a disconnect. Reconcile expired leases so+ // viewers stop showing a turn as working without issuing another command.+ let recovery_repo = session_repo.clone();+ let recovery_realtime = ConnectionGatewayAgentSessionRealtime::new(+ connection_gateway.clone(),+ session_audience.clone(),+ );+ let recovery = tokio::spawn(async move {+ let mut ticker =+ tokio::time::interval(agent_session::domain::ports::REPLICA_HEARTBEAT_INTERVAL);+ ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip);+ loop {+ ticker.tick().await;+ if let Err(error) = agent_session::domain::recovery::recover_stale_sessions(+ &recovery_repo,+ &recovery_realtime,+ std::num::NonZeroUsize::new(100).expect("nonzero recovery batch"),+ )+ .await+ {+ tracing::warn!(error = ?error, "failed to recover abandoned agent sessions");+ }+ }+ });+// Keep trigger generation in this deployment while retaining Kafka as the// boundary between channel events and harness commands.let mut trigger = tokio::spawn(trigger::supervise(@@ -1434,6 +1462,7 @@ async fn run() -> anyhow::Result<()> {trigger.abort();egress_http.abort();heartbeat.abort();+ recovery.abort();runtime_commands.abort();let stop_failures = container_shutdown.shutdown_all().await;if stop_failures > 0 {