Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(chat): streamed text reaches the chat, and a resync cannot loop
trohitgcd723ddaf3ec
diff --git a/client/src/stores/__tests__/chatRunStore.test.ts b/client/src/stores/__tests__/chatRunStore.test.tsindex 40ff1d04..4cd98e55 100644--- a/client/src/stores/__tests__/chatRunStore.test.ts+++ b/client/src/stores/__tests__/chatRunStore.test.ts@@ -2,7 +2,9 @@* The run store's ordering rules: frames fold in once per animation frame,* by each run's `seq`; a duplicate changes nothing; a gap, an unknown run* already under way or a new server epoch ask for a fresh snapshot and hold- * what arrives meanwhile; a snapshot names the runs that ended unseen.+ * what arrives meanwhile; a snapshot names the runs that ended unseen; a+ * run's end folds in across a gap and wins over any live reading; and a+ * resync never asks again for what its snapshot could not settle.*/import { beforeEach, describe, expect, it } from 'vitest';@@ -93,6 +95,37 @@ describe('chatRunStore', () => {expect(session().runs.r1.state).toBe('running');});+ it('folds the end of a run in across a gap, and nothing after it', () => {+ receive(frame(1, 'started'), frame(7, 'finished', { outcome: { type: 'success' } }));+ expect(session().runs.r1).toMatchObject({ state: 'finished', seq: 7 });+ expect(session().syncing).toBe(false);+ receive(frame(8, 'text.content', { message_id: 's1', delta: 'late' }));+ expect(session().runs.r1.segments).toEqual([]);+ // A run never seen before that ends is recorded as ended.+ receive(frame(4, 'failed', { message: 'Calendar said no', code: 'run_failed' }, 'r2'));+ expect(session().runs.r2).toMatchObject({ state: 'error', seq: 4 });+ });++ it('takes an end the server stored over a live reading, whatever its seq', () => {+ receive(frame(1, 'started'));+ // The hub forgets a run's seq once it ends: its stored end reads 0.+ useChatRunStore.getState().upsertRun(snapshot('r1', 'finished', 0));+ expect(session().runs.r1.state).toBe('finished');+ useChatRunStore.getState().applySubscription('w1', 'e1', [snapshot('r1', 'running', 3)]);+ expect(session().runs.r1.state).toBe('finished');+ });++ it('drops a held frame its snapshot cannot settle instead of asking again', () => {+ receive(frame(1, 'started'), frame(3, 'text.content', { message_id: 's1', delta: 'x' }));+ const asked = session().resync;+ // The run ended meanwhile, so the snapshot no longer lists it: it is read+ // on its own (stale), and the held frame is not asked for again.+ useChatRunStore.getState().applySubscription('w1', 'e1', []);+ expect(session()).toMatchObject({ syncing: false, resync: asked, stale: ['r1'] });+ receive(frame(4, 'text.content', { message_id: 's1', delta: 'y' }, 'r9'));+ expect(session().resync).toBe(asked + 1);+ });+it('asks for a snapshot when the hub dropped frames', () => {receive(frame(1, 'started'));receive({diff --git a/client/src/stores/chatRunStore.ts b/client/src/stores/chatRunStore.tsindex 2bcf28c2..5355f33c 100644--- a/client/src/stores/chatRunStore.ts+++ b/client/src/stores/chatRunStore.ts@@ -12,7 +12,12 @@* server restarted) or the hub's resync frame mark the session `syncing`* and bump `resync`: the subscription takes a fresh snapshot, and the* frames that arrived meanwhile are held and folded in after it, skipping- * what the snapshot already covers.+ * what the snapshot already covers. One that still cannot fold after it is+ * dropped, never asked for again: the snapshot was the server's best;+ * - a run's end (`finished`, `failed`) folds in even across a gap or for a+ * run not seen before, since its saved reply holds what was missed, and an+ * ended run takes nothing more. An ended reading of a run beats a live one+ * whatever their `seq`: the hub forgets a run's `seq` once it ends.** Nothing here talks to the socket.*/@@ -69,14 +74,19 @@ function requestResync(session: SessionRuns, held?: RunEvent): SessionRuns {};}-function foldEvent(session: SessionRuns, event: RunEvent): SessionRuns {+/** Fold one event in. `settling`: a frame held through a resync, folded in+ * after the fresh snapshot; if it still cannot fold it is dropped. */+function foldEvent(session: SessionRuns, event: RunEvent, settling = false): SessionRuns {if (session.syncing) return { ...session, held: [...session.held, event] };- if (session.hubEpoch !== null && session.hubEpoch !== event.hubEpoch) return requestResync(session, event);+ const ask = () => (settling ? session : requestResync(session, event));+ if (session.hubEpoch !== null && session.hubEpoch !== event.hubEpoch) return ask();const known = session.runs[event.runId];- const base = known ?? (event.seq === 1 ? emptyRun(event.runId, event.sessionId) : null);- if (!base) return requestResync(session, event);+ if (known && !isLiveRun(known)) return session;+ const ends = event.type === 'finished' || event.type === 'failed';+ const base = known ?? (event.seq === 1 || ends ? emptyRun(event.runId, event.sessionId) : null);+ if (!base) return ask();if (event.seq <= base.seq) return session;- if (event.seq > base.seq + 1) return requestResync(session, event);+ if (event.seq > base.seq + 1 && !ends) return ask();return {...session,hubEpoch: session.hubEpoch ?? event.hubEpoch,@@ -95,9 +105,12 @@ function foldFrame(sessions: Record<string, SessionRuns>, frame: RunFrame): Recoreturn next === session ? sessions : { ...sessions, [frame.sessionId]: next };}-/** The newer of two readings of one run: a higher `seq` in the same epoch. */+/** The newer of two readings of one run: an ended one over a live one (a+ * run never goes live again), else a higher `seq` in the same epoch. */function newer(local: RunSnapshot | undefined, incoming: RunSnapshot): RunSnapshot {- return local && local.hubEpoch === incoming.hubEpoch && local.seq > incoming.seq ? local : incoming;+ if (!local) return incoming;+ if (isLiveRun(local) !== isLiveRun(incoming)) return isLiveRun(incoming) ? local : incoming;+ return local.hubEpoch === incoming.hubEpoch && local.seq > incoming.seq ? local : incoming;}// Frames wait here for the next animation frame (a hidden page gets a@@ -154,7 +167,7 @@ export const useChatRunStore = create<ChatRunState>((set, get) => ({if (isLiveRun(local)) stale.push(runId);}let session: SessionRuns = { hubEpoch, runs, held: [], resync: previous.resync, syncing: false, subscribed: true, stale };- for (const event of previous.held) session = foldEvent(session, event);+ for (const event of previous.held) session = foldEvent(session, event, true);return { sessions: { ...state.sessions, [sessionId]: session } };});},diff --git a/docs-internal/chat_protocol.md b/docs-internal/chat_protocol.mdindex 74970a9d..eaf92409 100644--- a/docs-internal/chat_protocol.md+++ b/docs-internal/chat_protocol.md@@ -312,7 +312,8 @@ newer version wins in place.- **Start and finish.** MachinaWorkflow, behind the `machina-chat-run-v1` patch, reads the run id only from an eventwith that source and type, claims the run (`chat_run.start`: `pending` or `queued` to `running`, recording theTemporal workflow and run ids) once the firing trigger's output is stored, passes `run_scope {run_id, session_id}`- to every node context, and finishes it at its single exit (`chat_run.finish`). With several chat triggers in one+ to every node context (the chat's session, which the claim answers from the run's row; the workflow's own session+ is its execution's), and finishes it at its single exit (`chat_run.finish`). With several chat triggers in onegraph, the first to claim tracks the run and the others run untracked. Only the claimant may finish.- **The reply.** `chatReply` with a `run_scope` saves the answer through `ledger.post_reply`: the first reply takesthe run's reply id `a_<run id>`, another reply node in the same run `a_<run id>.<n>`, and a retry from the same node@@ -378,7 +379,10 @@ editor's console pane (`ConsoleChat`, compact, scope `live`) are the two hosts.`lib/agui/reduceRun.ts` folds it, the same way `services/chat/reducer.py` does. Frames are folded once peranimation frame. Per run: a duplicate `seq` changes nothing; a gap, an unknown run already past `seq` 1, a new`hub_epoch` or the hub's resync frame mark the session `syncing`, hold what arrives meanwhile, and ask for a fresh- snapshot, after which the held frames fold in and the ones the snapshot covers drop out.+ snapshot, after which the held frames fold in and the ones the snapshot covers drop out; one that still cannot fold+ is dropped, never asked for again. A run's end folds in even across a gap or for a run not seen before (its saved+ reply holds what was missed), an ended run takes nothing more, and an ended reading of a run beats a live one+ whatever their `seq` (the hub forgets a run's `seq` once it ends).- **Subscribing** (`data/runs.ts`): a mounted chat subscribes its session whenever the socket is ready (so againafter every reconnect) and whenever the store asks for a snapshot, retrying a failed subscribe after 1, 3, thenevery 10 seconds; the last chat following a session unsubscribes it. Runs the store held as live that a snapshotdiff --git a/server/services/chat/activities.py b/server/services/chat/activities.pyindex de78bb66..9ad34eef 100644--- a/server/services/chat/activities.py+++ b/server/services/chat/activities.py@@ -20,9 +20,11 @@ async def start_chat_run_activity(payload: Dict[str, Any]) -> Dict[str, Any]:"""Claim the run for the calling workflow.Payload ``{run_id, temporal_workflow_id, temporal_run_id}``. Returns- ``{claimed}``: False when another workflow claimed it first (several- chat triggers in one graph) or it can no longer start (it was stopped,- or the watchdog ended it); that workflow then runs without tracking.+ ``{claimed, session_id?}``: False when another workflow claimed it first+ (several chat triggers in one graph) or it can no longer start (it was+ stopped, or the watchdog ended it); that workflow then runs without+ tracking. ``session_id`` is the chat's session, from the run's row: the+ run's nodes carry it in ``run_scope``."""from core.container import containerfrom services.chat import ledger@@ -33,7 +35,9 @@ async def start_chat_run_activity(payload: Dict[str, Any]) -> Dict[str, Any]:temporal_workflow_id=str(payload.get("temporal_workflow_id") or ""),temporal_run_id=payload.get("temporal_run_id"),)- return {"claimed": run is not None}+ if run is None:+ return {"claimed": False}+ return {"claimed": True, "session_id": run.session_id}@activity.defn(name="chat_run.finish")diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.pyindex 5166d54a..fe4a2e0e 100644--- a/server/services/temporal/workflow.py+++ b/server/services/temporal/workflow.py@@ -356,7 +356,12 @@ class MachinaWorkflow:return Noneif not (isinstance(claimed, dict) and claimed.get("claimed")):return None- return {"run_id": run_id, "session_id": session_id}+ # The chat's session comes from the run's row: this workflow's own+ # session is its execution's ("<workflow>:execution:<n>" on a deployed+ # run). A claim recorded before it carried the session keeps the old+ # value; only activity inputs change, so histories replay as before.+ chat_session = claimed.get("session_id")+ return {"run_id": run_id, "session_id": chat_session if isinstance(chat_session, str) and chat_session else session_id}async def _finish_chat_run(self, run_scope: Dict[str, Any], success: bool, errors: List[Dict]) -> None:"""End the claimed chat run with the run's outcome. Non-fatal: thediff --git a/server/tests/services/chat/test_ledger.py b/server/tests/services/chat/test_ledger.pyindex c5b214e8..bbdaea99 100644--- a/server/tests/services/chat/test_ledger.py+++ b/server/tests/services/chat/test_ledger.py@@ -176,6 +176,21 @@ async def test_the_first_workflow_claims_the_run(database, hub):assert (await started(database, admission, workflow_id="tw-2")) is None+async def test_the_claim_tells_the_workflow_the_chats_session(database, hub, monkeypatch):+ """MachinaWorkflow's own session is its execution's; the run's nodes need+ the chat's, so the claim answers with it (services/chat/activities.py)."""+ from types import SimpleNamespace++ import core.container as container_module+ from services.chat.activities import start_chat_run_activity++ monkeypatch.setattr(container_module, "container", SimpleNamespace(database=lambda: database))+ admission = await admit(database)+ claim = {"run_id": admission.run.run_id, "temporal_workflow_id": "tw-1", "temporal_run_id": "tr-1"}+ assert await start_chat_run_activity(claim) == {"claimed": True, "session_id": "wf"}+ assert await start_chat_run_activity({**claim, "temporal_workflow_id": "tw-2"}) == {"claimed": False}++async def test_a_queued_run_starts_when_the_employee_resumes(database, hub):admission = await admit(database, state="queued")assert admission.run.state == "queued"diff --git a/server/tests/temporal/test_machina_chat_run.py b/server/tests/temporal/test_machina_chat_run.pyindex 1dfffcec..94c36612 100644--- a/server/tests/temporal/test_machina_chat_run.py+++ b/server/tests/temporal/test_machina_chat_run.py@@ -48,7 +48,8 @@ def _graph(*, source: str = SOURCE, event_type: str = TYPE, run_id: Any = "r_1","nodes": [trigger, {"id": "wf-cr:console:1", "type": "console", "data": {"label": "Log"}}],"edges": [{"id": "e1", "source": trigger["id"], "target": "wf-cr:console:1", "targetHandle": "input-main"}],"workflow_id": "wf-cr",- "session_id": "wf-cr",+ # A deployed run's own session is its execution's, never the chat's.+ "session_id": "wf-cr:execution:1","execution_id": "wf-cr:execution:1","generation": 2,"_temporal_routing_v1": ROUTING,@@ -69,6 +70,7 @@ def fakes(monkeypatch):contexts=[],node_success=True,claimed=True,+ claim_session="wf-cr",open_patches={CHAT_RUN_PATCH, "machina-conditional-edges-v1", "machina-run-record-v1"},asked=[],)@@ -91,7 +93,9 @@ def fakes(monkeypatch):async def execute_activity(name, payload=None, **_kwargs):state.activities.append((name, payload))if name == "chat_run.start":- return {"claimed": state.claimed}+ if not state.claimed:+ return {"claimed": False}+ return {"claimed": True, **({"session_id": state.claim_session} if state.claim_session else {})}return Nonemonkeypatch.setattr(temporal_workflow, "start_activity", start_activity)@@ -130,6 +134,12 @@ async def test_a_chat_message_run_is_claimed_scoped_and_finished(fakes):assert names.index("chat_run.finish") < names.index("workflow_runs.record_completion")+async def test_a_claim_recorded_without_the_chats_session_keeps_the_old_scope(fakes):+ fakes.claim_session = None+ await _run(_graph())+ assert [context["run_scope"] for context in fakes.contexts] == [{"run_id": "r_1", "session_id": "wf-cr:execution:1"}]++async def test_a_failed_run_finishes_with_its_error(fakes):fakes.node_success = Falseawait _run(_graph())@@ -207,7 +217,7 @@ async def test_the_activities_go_through_the_ledger(monkeypatch):async def start_run(database, **kwargs):calls.append(("start", kwargs))- return SimpleNamespace(state="running")+ return SimpleNamespace(state="running", session_id="wf-cr")async def finish_run(database, **kwargs):calls.append(("finish", kwargs))@@ -216,7 +226,7 @@ async def test_the_activities_go_through_the_ledger(monkeypatch):monkeypatch.setattr(ledger, "start_run", start_run)monkeypatch.setattr(ledger, "finish_run", finish_run)monkeypatch.setattr(container_module, "container", SimpleNamespace(database=lambda: "db"))- assert await start_chat_run_activity({"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t"}) == {"claimed": True}+ assert await start_chat_run_activity({"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t"}) == {"claimed": True, "session_id": "wf-cr"}assert await finish_chat_run_activity({"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t", "success": True}) == {"state": None}assert calls[0] == ("start", {"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t"})assert calls[1][1]["success"] is True and calls[1][1]["error"] is None@@ -258,7 +268,7 @@ async def _replay_gate() -> None:@activity.defn(name="chat_run.start")async def start(_payload: Dict[str, Any]) -> Dict[str, Any]:- return {"claimed": True}+ return {"claimed": True, "session_id": "wf-cr"}@activity.defn(name="chat_run.finish")async def finish(_payload: Dict[str, Any]) -> Dict[str, Any]: