Skip to content

Self-improving AI Employees for everyone

fix(chat): streamed text reaches the chat, and a resync cannot loop

trohitgcd723ddaf3ec
diff --git a/client/src/stores/__tests__/chatRunStore.test.ts b/client/src/stores/__tests__/chatRunStore.test.ts
index 40ff1d04..4cd98e55 100644
--- a/client/src/stores/__tests__/chatRunStore.test.ts
+++ b/client/src/stores/__tests__/chatRunStore.test.ts
@@ -2,7 +2,9 @@
* The run store's ordering rules: frames fold in once per animation frame,
* by each run's `seq`; a duplicate changes nothing; a gap, an unknown run
* already under way or a new server epoch ask for a fresh snapshot and hold
- * what arrives meanwhile; a snapshot names the runs that ended unseen.
+ * what arrives meanwhile; a snapshot names the runs that ended unseen; a
+ * run's end folds in across a gap and wins over any live reading; and a
+ * resync never asks again for what its snapshot could not settle.
*/
import { beforeEach, describe, expect, it } from 'vitest';
@@ -93,6 +95,37 @@ describe('chatRunStore', () => {
expect(session().runs.r1.state).toBe('running');
});
+ it('folds the end of a run in across a gap, and nothing after it', () => {
+ receive(frame(1, 'started'), frame(7, 'finished', { outcome: { type: 'success' } }));
+ expect(session().runs.r1).toMatchObject({ state: 'finished', seq: 7 });
+ expect(session().syncing).toBe(false);
+ receive(frame(8, 'text.content', { message_id: 's1', delta: 'late' }));
+ expect(session().runs.r1.segments).toEqual([]);
+ // A run never seen before that ends is recorded as ended.
+ receive(frame(4, 'failed', { message: 'Calendar said no', code: 'run_failed' }, 'r2'));
+ expect(session().runs.r2).toMatchObject({ state: 'error', seq: 4 });
+ });
+
+ it('takes an end the server stored over a live reading, whatever its seq', () => {
+ receive(frame(1, 'started'));
+ // The hub forgets a run's seq once it ends: its stored end reads 0.
+ useChatRunStore.getState().upsertRun(snapshot('r1', 'finished', 0));
+ expect(session().runs.r1.state).toBe('finished');
+ useChatRunStore.getState().applySubscription('w1', 'e1', [snapshot('r1', 'running', 3)]);
+ expect(session().runs.r1.state).toBe('finished');
+ });
+
+ it('drops a held frame its snapshot cannot settle instead of asking again', () => {
+ receive(frame(1, 'started'), frame(3, 'text.content', { message_id: 's1', delta: 'x' }));
+ const asked = session().resync;
+ // The run ended meanwhile, so the snapshot no longer lists it: it is read
+ // on its own (stale), and the held frame is not asked for again.
+ useChatRunStore.getState().applySubscription('w1', 'e1', []);
+ expect(session()).toMatchObject({ syncing: false, resync: asked, stale: ['r1'] });
+ receive(frame(4, 'text.content', { message_id: 's1', delta: 'y' }, 'r9'));
+ expect(session().resync).toBe(asked + 1);
+ });
+
it('asks for a snapshot when the hub dropped frames', () => {
receive(frame(1, 'started'));
receive({
diff --git a/client/src/stores/chatRunStore.ts b/client/src/stores/chatRunStore.ts
index 2bcf28c2..5355f33c 100644
--- a/client/src/stores/chatRunStore.ts
+++ b/client/src/stores/chatRunStore.ts
@@ -12,7 +12,12 @@
* server restarted) or the hub's resync frame mark the session `syncing`
* and bump `resync`: the subscription takes a fresh snapshot, and the
* frames that arrived meanwhile are held and folded in after it, skipping
- * what the snapshot already covers.
+ * what the snapshot already covers. One that still cannot fold after it is
+ * dropped, never asked for again: the snapshot was the server's best;
+ * - a run's end (`finished`, `failed`) folds in even across a gap or for a
+ * run not seen before, since its saved reply holds what was missed, and an
+ * ended run takes nothing more. An ended reading of a run beats a live one
+ * whatever their `seq`: the hub forgets a run's `seq` once it ends.
*
* Nothing here talks to the socket.
*/
@@ -69,14 +74,19 @@ function requestResync(session: SessionRuns, held?: RunEvent): SessionRuns {
};
}
-function foldEvent(session: SessionRuns, event: RunEvent): SessionRuns {
+/** Fold one event in. `settling`: a frame held through a resync, folded in
+ * after the fresh snapshot; if it still cannot fold it is dropped. */
+function foldEvent(session: SessionRuns, event: RunEvent, settling = false): SessionRuns {
if (session.syncing) return { ...session, held: [...session.held, event] };
- if (session.hubEpoch !== null && session.hubEpoch !== event.hubEpoch) return requestResync(session, event);
+ const ask = () => (settling ? session : requestResync(session, event));
+ if (session.hubEpoch !== null && session.hubEpoch !== event.hubEpoch) return ask();
const known = session.runs[event.runId];
- const base = known ?? (event.seq === 1 ? emptyRun(event.runId, event.sessionId) : null);
- if (!base) return requestResync(session, event);
+ if (known && !isLiveRun(known)) return session;
+ const ends = event.type === 'finished' || event.type === 'failed';
+ const base = known ?? (event.seq === 1 || ends ? emptyRun(event.runId, event.sessionId) : null);
+ if (!base) return ask();
if (event.seq <= base.seq) return session;
- if (event.seq > base.seq + 1) return requestResync(session, event);
+ if (event.seq > base.seq + 1 && !ends) return ask();
return {
...session,
hubEpoch: session.hubEpoch ?? event.hubEpoch,
@@ -95,9 +105,12 @@ function foldFrame(sessions: Record<string, SessionRuns>, frame: RunFrame): Reco
return next === session ? sessions : { ...sessions, [frame.sessionId]: next };
}
-/** The newer of two readings of one run: a higher `seq` in the same epoch. */
+/** The newer of two readings of one run: an ended one over a live one (a
+ * run never goes live again), else a higher `seq` in the same epoch. */
function newer(local: RunSnapshot | undefined, incoming: RunSnapshot): RunSnapshot {
- return local && local.hubEpoch === incoming.hubEpoch && local.seq > incoming.seq ? local : incoming;
+ if (!local) return incoming;
+ if (isLiveRun(local) !== isLiveRun(incoming)) return isLiveRun(incoming) ? local : incoming;
+ return local.hubEpoch === incoming.hubEpoch && local.seq > incoming.seq ? local : incoming;
}
// Frames wait here for the next animation frame (a hidden page gets a
@@ -154,7 +167,7 @@ export const useChatRunStore = create<ChatRunState>((set, get) => ({
if (isLiveRun(local)) stale.push(runId);
}
let session: SessionRuns = { hubEpoch, runs, held: [], resync: previous.resync, syncing: false, subscribed: true, stale };
- for (const event of previous.held) session = foldEvent(session, event);
+ for (const event of previous.held) session = foldEvent(session, event, true);
return { sessions: { ...state.sessions, [sessionId]: session } };
});
},
diff --git a/docs-internal/chat_protocol.md b/docs-internal/chat_protocol.md
index 74970a9d..eaf92409 100644
--- a/docs-internal/chat_protocol.md
+++ b/docs-internal/chat_protocol.md
@@ -312,7 +312,8 @@ newer version wins in place.
- **Start and finish.** MachinaWorkflow, behind the `machina-chat-run-v1` patch, reads the run id only from an event
with that source and type, claims the run (`chat_run.start`: `pending` or `queued` to `running`, recording the
Temporal workflow and run ids) once the firing trigger's output is stored, passes `run_scope {run_id, session_id}`
- to every node context, and finishes it at its single exit (`chat_run.finish`). With several chat triggers in one
+ to every node context (the chat's session, which the claim answers from the run's row; the workflow's own session
+ is its execution's), and finishes it at its single exit (`chat_run.finish`). With several chat triggers in one
graph, the first to claim tracks the run and the others run untracked. Only the claimant may finish.
- **The reply.** `chatReply` with a `run_scope` saves the answer through `ledger.post_reply`: the first reply takes
the run's reply id `a_<run id>`, another reply node in the same run `a_<run id>.<n>`, and a retry from the same node
@@ -378,7 +379,10 @@ editor's console pane (`ConsoleChat`, compact, scope `live`) are the two hosts.
`lib/agui/reduceRun.ts` folds it, the same way `services/chat/reducer.py` does. Frames are folded once per
animation frame. Per run: a duplicate `seq` changes nothing; a gap, an unknown run already past `seq` 1, a new
`hub_epoch` or the hub's resync frame mark the session `syncing`, hold what arrives meanwhile, and ask for a fresh
- snapshot, after which the held frames fold in and the ones the snapshot covers drop out.
+ snapshot, after which the held frames fold in and the ones the snapshot covers drop out; one that still cannot fold
+ is dropped, never asked for again. A run's end folds in even across a gap or for a run not seen before (its saved
+ reply holds what was missed), an ended run takes nothing more, and an ended reading of a run beats a live one
+ whatever their `seq` (the hub forgets a run's `seq` once it ends).
- **Subscribing** (`data/runs.ts`): a mounted chat subscribes its session whenever the socket is ready (so again
after every reconnect) and whenever the store asks for a snapshot, retrying a failed subscribe after 1, 3, then
every 10 seconds; the last chat following a session unsubscribes it. Runs the store held as live that a snapshot
diff --git a/server/services/chat/activities.py b/server/services/chat/activities.py
index de78bb66..9ad34eef 100644
--- a/server/services/chat/activities.py
+++ b/server/services/chat/activities.py
@@ -20,9 +20,11 @@ async def start_chat_run_activity(payload: Dict[str, Any]) -> Dict[str, Any]:
"""Claim the run for the calling workflow.
Payload ``{run_id, temporal_workflow_id, temporal_run_id}``. Returns
- ``{claimed}``: False when another workflow claimed it first (several
- chat triggers in one graph) or it can no longer start (it was stopped,
- or the watchdog ended it); that workflow then runs without tracking.
+ ``{claimed, session_id?}``: False when another workflow claimed it first
+ (several chat triggers in one graph) or it can no longer start (it was
+ stopped, or the watchdog ended it); that workflow then runs without
+ tracking. ``session_id`` is the chat's session, from the run's row: the
+ run's nodes carry it in ``run_scope``.
"""
from core.container import container
from services.chat import ledger
@@ -33,7 +35,9 @@ async def start_chat_run_activity(payload: Dict[str, Any]) -> Dict[str, Any]:
temporal_workflow_id=str(payload.get("temporal_workflow_id") or ""),
temporal_run_id=payload.get("temporal_run_id"),
)
- return {"claimed": run is not None}
+ if run is None:
+ return {"claimed": False}
+ return {"claimed": True, "session_id": run.session_id}
@activity.defn(name="chat_run.finish")
diff --git a/server/services/temporal/workflow.py b/server/services/temporal/workflow.py
index 5166d54a..fe4a2e0e 100644
--- a/server/services/temporal/workflow.py
+++ b/server/services/temporal/workflow.py
@@ -356,7 +356,12 @@ class MachinaWorkflow:
return None
if not (isinstance(claimed, dict) and claimed.get("claimed")):
return None
- return {"run_id": run_id, "session_id": session_id}
+ # The chat's session comes from the run's row: this workflow's own
+ # session is its execution's ("<workflow>:execution:<n>" on a deployed
+ # run). A claim recorded before it carried the session keeps the old
+ # value; only activity inputs change, so histories replay as before.
+ chat_session = claimed.get("session_id")
+ return {"run_id": run_id, "session_id": chat_session if isinstance(chat_session, str) and chat_session else session_id}
async def _finish_chat_run(self, run_scope: Dict[str, Any], success: bool, errors: List[Dict]) -> None:
"""End the claimed chat run with the run's outcome. Non-fatal: the
diff --git a/server/tests/services/chat/test_ledger.py b/server/tests/services/chat/test_ledger.py
index c5b214e8..bbdaea99 100644
--- a/server/tests/services/chat/test_ledger.py
+++ b/server/tests/services/chat/test_ledger.py
@@ -176,6 +176,21 @@ async def test_the_first_workflow_claims_the_run(database, hub):
assert (await started(database, admission, workflow_id="tw-2")) is None
+async def test_the_claim_tells_the_workflow_the_chats_session(database, hub, monkeypatch):
+ """MachinaWorkflow's own session is its execution's; the run's nodes need
+ the chat's, so the claim answers with it (services/chat/activities.py)."""
+ from types import SimpleNamespace
+
+ import core.container as container_module
+ from services.chat.activities import start_chat_run_activity
+
+ monkeypatch.setattr(container_module, "container", SimpleNamespace(database=lambda: database))
+ admission = await admit(database)
+ claim = {"run_id": admission.run.run_id, "temporal_workflow_id": "tw-1", "temporal_run_id": "tr-1"}
+ assert await start_chat_run_activity(claim) == {"claimed": True, "session_id": "wf"}
+ assert await start_chat_run_activity({**claim, "temporal_workflow_id": "tw-2"}) == {"claimed": False}
+
+
async def test_a_queued_run_starts_when_the_employee_resumes(database, hub):
admission = await admit(database, state="queued")
assert admission.run.state == "queued"
diff --git a/server/tests/temporal/test_machina_chat_run.py b/server/tests/temporal/test_machina_chat_run.py
index 1dfffcec..94c36612 100644
--- a/server/tests/temporal/test_machina_chat_run.py
+++ b/server/tests/temporal/test_machina_chat_run.py
@@ -48,7 +48,8 @@ def _graph(*, source: str = SOURCE, event_type: str = TYPE, run_id: Any = "r_1",
"nodes": [trigger, {"id": "wf-cr:console:1", "type": "console", "data": {"label": "Log"}}],
"edges": [{"id": "e1", "source": trigger["id"], "target": "wf-cr:console:1", "targetHandle": "input-main"}],
"workflow_id": "wf-cr",
- "session_id": "wf-cr",
+ # A deployed run's own session is its execution's, never the chat's.
+ "session_id": "wf-cr:execution:1",
"execution_id": "wf-cr:execution:1",
"generation": 2,
"_temporal_routing_v1": ROUTING,
@@ -69,6 +70,7 @@ def fakes(monkeypatch):
contexts=[],
node_success=True,
claimed=True,
+ claim_session="wf-cr",
open_patches={CHAT_RUN_PATCH, "machina-conditional-edges-v1", "machina-run-record-v1"},
asked=[],
)
@@ -91,7 +93,9 @@ def fakes(monkeypatch):
async def execute_activity(name, payload=None, **_kwargs):
state.activities.append((name, payload))
if name == "chat_run.start":
- return {"claimed": state.claimed}
+ if not state.claimed:
+ return {"claimed": False}
+ return {"claimed": True, **({"session_id": state.claim_session} if state.claim_session else {})}
return None
monkeypatch.setattr(temporal_workflow, "start_activity", start_activity)
@@ -130,6 +134,12 @@ async def test_a_chat_message_run_is_claimed_scoped_and_finished(fakes):
assert names.index("chat_run.finish") < names.index("workflow_runs.record_completion")
+async def test_a_claim_recorded_without_the_chats_session_keeps_the_old_scope(fakes):
+ fakes.claim_session = None
+ await _run(_graph())
+ assert [context["run_scope"] for context in fakes.contexts] == [{"run_id": "r_1", "session_id": "wf-cr:execution:1"}]
+
+
async def test_a_failed_run_finishes_with_its_error(fakes):
fakes.node_success = False
await _run(_graph())
@@ -207,7 +217,7 @@ async def test_the_activities_go_through_the_ledger(monkeypatch):
async def start_run(database, **kwargs):
calls.append(("start", kwargs))
- return SimpleNamespace(state="running")
+ return SimpleNamespace(state="running", session_id="wf-cr")
async def finish_run(database, **kwargs):
calls.append(("finish", kwargs))
@@ -216,7 +226,7 @@ async def test_the_activities_go_through_the_ledger(monkeypatch):
monkeypatch.setattr(ledger, "start_run", start_run)
monkeypatch.setattr(ledger, "finish_run", finish_run)
monkeypatch.setattr(container_module, "container", SimpleNamespace(database=lambda: "db"))
- assert await start_chat_run_activity({"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t"}) == {"claimed": True}
+ assert await start_chat_run_activity({"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t"}) == {"claimed": True, "session_id": "wf-cr"}
assert await finish_chat_run_activity({"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t", "success": True}) == {"state": None}
assert calls[0] == ("start", {"run_id": "r", "temporal_workflow_id": "w", "temporal_run_id": "t"})
assert calls[1][1]["success"] is True and calls[1][1]["error"] is None
@@ -258,7 +268,7 @@ async def _replay_gate() -> None:
@activity.defn(name="chat_run.start")
async def start(_payload: Dict[str, Any]) -> Dict[str, Any]:
- return {"claimed": True}
+ return {"claimed": True, "session_id": "wf-cr"}
@activity.defn(name="chat_run.finish")
async def finish(_payload: Dict[str, Any]) -> Dict[str, Any]: