Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(computer): draw script notify and confirm prompts as Agent messages on Web (#2850)

Maxea69e668a4cb
diff --git a/core/computer_script_interactions.py b/core/computer_script_interactions.py
index 33631014d..a76e7f0d1 100644
--- a/core/computer_script_interactions.py
+++ b/core/computer_script_interactions.py
@@ -13,6 +13,7 @@ from core.computer_script_sessions import (
ComputerScriptSessionManager,
)
from core.delivery_target import routed_delivery_context
+from core.message_output import MessageOutput
from core.services.dispatch import SOURCE_SCHEDULED
from modules.agents.question_ui import PendingQuestion, Question
from modules.im import MessageContext
@@ -25,6 +26,11 @@ MAX_MESSAGE_BYTES = 64 * 1024
# six-byte ``\uXXXX`` escape) plus the connection token and keys, so the transport cap
# leaves room for the worst-case envelope around a message the limit admits.
MAX_INTERACTION_REQUEST_BYTES = 6 * MAX_MESSAGE_BYTES + 4 * 1024
+# ``notify`` row events naming a script's own words to the owner. The message-type
+# catalog lists them under ``agentRenderWhenEvents``, so the Web draws them as the
+# Agent's message instead of a runtime status pill; IM delivery reads no metadata.
+NOTIFY_EVENT = "computer_script_notify"
+CONFIRM_EVENT = "computer_script_confirm"
_YES_ANSWERS = frozenset(
{
"y",
@@ -107,6 +113,7 @@ class ComputerScriptInteractionManager:
message_id = await self._deliver(
context,
message,
+ event=NOTIFY_EVENT,
failure_code="computer_script_notify_failed",
)
return {
@@ -166,6 +173,7 @@ class ComputerScriptInteractionManager:
await self._deliver(
context,
self._translate("computer.cli.confirm.prompt", message=message),
+ event=CONFIRM_EVENT,
failure_code="confirm_prompt_failed",
)
record.armed = True
@@ -287,6 +295,7 @@ class ComputerScriptInteractionManager:
context: MessageContext,
message: str,
*,
+ event: str,
failure_code: str,
) -> str:
"""Send one notify and return its id; a missing id means not delivered."""
@@ -296,6 +305,7 @@ class ComputerScriptInteractionManager:
context,
"notify",
message,
+ output=MessageOutput(metadata={"event": event}),
)
except Exception as exc:
logger.exception("Computer script message delivery failed")
diff --git a/tests/scenarios/message_delivery/catalog.yaml b/tests/scenarios/message_delivery/catalog.yaml
index e58a90f41..53313f244 100644
--- a/tests/scenarios/message_delivery/catalog.yaml
+++ b/tests/scenarios/message_delivery/catalog.yaml
@@ -38,6 +38,14 @@ scenarios:
kind: regression
layer: contract
test: tests/test_codex_prompt_delivery_contract.py::test_excluded_resume_preserves_native_history_and_model_settings
+ - id: MESSAGE-DELIVERY-035
+ name: A Computer Use script's notify and confirm prompt draw as a visible Agent message during a running Web Turn
+ status: covered
+ kind: regression
+ layer: contract
+ test:
+ - tests/test_computer_script_interactions.py::test_script_words_carry_the_event_the_web_draws_as_an_agent_message
+ - "ui/src/components/workbench/ChatPage.hydration.test.tsx::Computer Use script prompt with Activity enabled=true MESSAGE-DELIVERY-035 draws the confirm prompt as a visible Agent message while the Turn runs"
- id: MESSAGE-DELIVERY-001
name: Scheduled result finalizes its delivery anchor
status: covered
diff --git a/tests/test_computer_script_interactions.py b/tests/test_computer_script_interactions.py
index c7998bb76..ac5f1771b 100644
--- a/tests/test_computer_script_interactions.py
+++ b/tests/test_computer_script_interactions.py
@@ -8,14 +8,17 @@ import pytest
from core import internal_server, session_turns
from core.computer_script_interactions import (
+ CONFIRM_EVENT,
MAX_INTERACTION_REQUEST_BYTES,
MAX_MESSAGE_BYTES,
+ NOTIFY_EVENT,
ComputerScriptInteractionError,
ComputerScriptInteractionManager,
)
from core.computer_script_sessions import ComputerScriptSessionManager
from core.services.dispatch import SOURCE_SCHEDULED
from modules.im import MessageContext
+from vibe.message_types import spec_for
def _context(
@@ -48,6 +51,8 @@ class _FakeController:
def __init__(self, language: str = "en") -> None:
self.config = SimpleNamespace(language=language)
self.messages: list[tuple[MessageContext, str, str]] = []
+ # The ``output`` each delivered message carried, aligned with ``messages``.
+ self.outputs: list = []
# ``"none"`` mimics the dispatcher's failed IM send; ``"raise"`` an error.
self.delivery = "ok"
self.release: asyncio.Event | None = None
@@ -57,6 +62,8 @@ class _FakeController:
context: MessageContext,
message_type: str,
text: str,
+ *,
+ output=None,
) -> str | None:
if self.release is not None:
await self.release.wait()
@@ -65,6 +72,7 @@ class _FakeController:
if self.delivery == "none":
return None
self.messages.append((context, message_type, text))
+ self.outputs.append(output)
return f"message-{len(self.messages)}"
@@ -114,6 +122,53 @@ def test_notify_reuses_the_launching_surface(platform: str) -> None:
assert text == "A progress update"
+def test_script_words_carry_the_event_the_web_draws_as_an_agent_message() -> None:
+ """MESSAGE-DELIVERY-035: the owner must notice a script's notify and confirm prompt mid-Turn.
+
+ Both stay ``notify`` rows, so every IM surface delivers them unchanged; the
+ event is what moves them out of the Web's compact status pill and into an
+ Agent bubble, so it must be one the shared catalog declares for that. The
+ answer notice is Avibe's own status line and keeps the pill.
+ """
+
+ async def exercise() -> _FakeController:
+ manager, controller, _turn = _manager(platform="avibe")
+ await manager.notify("script-token", "Calculator shows 42")
+ pending = asyncio.create_task(manager.confirm("script-token", "Press = ?"))
+ for _ in range(3):
+ await asyncio.sleep(0)
+ matched = manager.match_answer(
+ session_id="session", approver_id="owner", message="是"
+ )
+ assert matched is not None
+ manager.resolve_answer(*matched)
+ await pending
+ return controller
+
+ controller = asyncio.run(exercise())
+ agent_render_events = spec_for("notify")["agentRenderWhenEvents"]
+ observed = [
+ (message_type, (output.metadata.get("event") if output is not None else None))
+ for (_context, message_type, _text), output in zip(
+ controller.messages, controller.outputs, strict=True
+ )
+ ]
+ assert observed == [
+ ("notify", NOTIFY_EVENT),
+ ("notify", CONFIRM_EVENT),
+ ("notify", None),
+ ]
+ assert controller.messages[0][2] == "Calculator shows 42"
+ assert "Press = ?" in controller.messages[1][2]
+ for _message_type, event in observed[:2]:
+ assert event in agent_render_events
+ for output in controller.outputs[:2]:
+ # Metadata only: the row must not gain lifecycle authority over the Turn.
+ assert not output.completes_turn
+ assert not output.detached
+ assert output.idempotency_key is None
+
+
@pytest.mark.parametrize(
("operation", "delivery", "code"),
[
diff --git a/tests/test_message_type_catalog.py b/tests/test_message_type_catalog.py
index 83e936af0..3bab3b282 100644
--- a/tests/test_message_type_catalog.py
+++ b/tests/test_message_type_catalog.py
@@ -187,6 +187,7 @@ def test_annotation_catalog_contract_is_explicit() -> None:
"webPush": False,
"webPushWhenEvents": (),
"render": "annotation",
+ "agentRenderWhenEvents": (),
}
diff --git a/ui/src/components/workbench/ChatPage.hydration.test.tsx b/ui/src/components/workbench/ChatPage.hydration.test.tsx
index 8d283d7b5..e45369870 100644
--- a/ui/src/components/workbench/ChatPage.hydration.test.tsx
+++ b/ui/src/components/workbench/ChatPage.hydration.test.tsx
@@ -1032,6 +1032,63 @@ describe('ChatPage transcript hydration', () => {
});
});
+ describe.each([false, true])('Computer Use script prompt with Activity enabled=%s', (activityEnabled) => {
+ it('MESSAGE-DELIVERY-035 draws the confirm prompt as a visible Agent message while the Turn runs', async () => {
+ // The prompt lands mid-Turn over message.new; the owner must notice it and
+ // reply, so it is the Agent's bubble rather than a compact status pill.
+ const prompt = {
+ ...projectedMessage('confirm-prompt', '计算器里已输入 6 × 7,下一步要按「=」。请回复“是”或“否”。'),
+ author: 'agent', source: 'agent', type: 'notify',
+ created_at: '2026-08-15T00:00:30Z',
+ metadata: { event: 'computer_script_confirm', turn_id: 'current-turn', detached: false },
+ };
+ mocks.api.getSession.mockResolvedValue({ id: 'session-new' });
+ mocks.api.getSessionBootstrap.mockResolvedValue({
+ ...bootstrapPayload('session-new'),
+ config: { ui: { show_agent_activity: activityEnabled } },
+ turn_state: { ...idleTurnState, foreground: 'running', in_flight: true },
+ messages: [projectedMessage('owner-start', '开始')],
+ });
+ render(
+ <MemoryRouter initialEntries={['/chat/session-new']}>
+ <Routes>
+ <Route path="/chat/:sessionId" element={<ChatPage />} />
+ </Routes>
+ </MemoryRouter>,
+ );
+ await screen.findByText('chat.thinking');
+ act(() => mocks.events?.onTurnStart({ session_id: 'session-new' }));
+ if (activityEnabled) {
+ act(() => mocks.events?.onMessageNew({
+ ...projectedMessage('script-step', 'vibe computer run'),
+ author: 'agent', source: 'agent', type: 'assistant',
+ created_at: '2026-08-15T00:00:10Z',
+ }));
+ expect(screen.getByText('vibe computer run')).toBeTruthy();
+ }
+ act(() => mocks.events?.onMessageNew(prompt));
+
+ const row = document.querySelector('[data-message-id="confirm-prompt"]');
+ expect(row).not.toBeNull();
+ expect(row!.textContent).toContain(prompt.text);
+ // The Agent's bubble, not the gold notify pill.
+ expect(row!.querySelector('.border-mint\\/25')).not.toBeNull();
+ expect(row!.textContent).not.toContain('chat.notifyLabel');
+ // It does not end the Turn: the running indicator stays and Stop stays live.
+ if (activityEnabled) expect(screen.getByText('vibe computer run')).toBeTruthy();
+ else expect(screen.getByText('chat.thinking')).toBeTruthy();
+ expect(mocks.composer?.busy).toBe(true);
+
+ // Any other notification keeps its compact status pill.
+ act(() => mocks.events?.onMessageNew({
+ ...prompt, id: 'runtime-notice', text: 'runtime status', created_at: '2026-08-15T00:00:40Z', metadata: {},
+ }));
+ const notice = document.querySelector('[data-message-id="runtime-notice"]');
+ expect(notice!.textContent).toContain('chat.notifyLabel');
+ expect(notice!.querySelector('.border-mint\\/25')).toBeNull();
+ });
+ });
+
it('does not render an open activity group as interrupted before the switched chat turn state is known', async () => {
const openGroup = {
id: 'open-turn',
diff --git a/ui/src/components/workbench/ChatPage.tsx b/ui/src/components/workbench/ChatPage.tsx
index 8d63641c0..4a7a366c3 100644
--- a/ui/src/components/workbench/ChatPage.tsx
+++ b/ui/src/components/workbench/ChatPage.tsx
@@ -4747,7 +4747,9 @@ export const MessageRow = memo(function MessageRow({
// mounting the page. Everything below reads the answer; nothing re-derives it.
const row = chatRowKind(message);
const isNotify = row.kind === 'notify';
- const isAgent = row.kind === 'agent';
+ // A notice addressed to the owner draws the Agent's bubble; authorship below
+ // still withholds the Agent-only Markdown affordances from it.
+ const isAgent = row.kind === 'agent' || row.kind === 'agentNotice';
const isBoundary = row.kind === 'boundary';
// ...and, separately, who wrote it. Only the agent's own words may carry the
// agent-authored Markdown affordances, and its reverse annotation is still its
diff --git a/ui/src/lib/chatMessageTypes.ts b/ui/src/lib/chatMessageTypes.ts
index 52d9072b9..e534a9e9e 100644
--- a/ui/src/lib/chatMessageTypes.ts
+++ b/ui/src/lib/chatMessageTypes.ts
@@ -3,6 +3,21 @@ import { specFor } from './messageTypes';
// Status pills rather than authored answers — the catalog's ``status`` render kind.
export const isNotifyMessageType = (type: string): boolean => specFor(type).render === 'status';
+// A status-typed row whose ``metadata.event`` the catalog lists under
+// ``agentRenderWhenEvents``: words addressed to the owner (a Computer Use script's
+// notify or confirm prompt), drawn as the Agent's message so they are noticed
+// mid-Turn. Presentation only — the row keeps its type's lifecycle, authorship and
+// unread semantics, and every IM surface delivers it unchanged.
+export const isAgentRenderedNotice = (message: {
+ type: string;
+ metadata?: Record<string, unknown> | null;
+}): boolean => {
+ const event = message.metadata?.event;
+ return isNotifyMessageType(message.type)
+ && typeof event === 'string'
+ && specFor(message.type).agentRenderWhenEvents.includes(event);
+};
+
// Whether a row belongs in the chat transcript — the catalog's ``transcript``
// property, the same declaration the server filter on
// ``GET /api/sessions/{id}/messages`` reads, so the live ``message.new`` feed
diff --git a/ui/src/lib/chatRowKind.test.ts b/ui/src/lib/chatRowKind.test.ts
index 941c63685..2a2fe3ddd 100644
--- a/ui/src/lib/chatRowKind.test.ts
+++ b/ui/src/lib/chatRowKind.test.ts
@@ -1,6 +1,7 @@
import { describe, expect, it } from 'vitest';
import { chatRowKind, drawsEmptyBodyPlaceholder, isAgentAuthored } from './chatRowKind';
+import { messageTypeNames, specFor } from './messageTypes';
const row = (over: Partial<Parameters<typeof chatRowKind>[0]>) =>
chatRowKind({ type: 'user', author: 'user', source: 'user', ...over });
@@ -35,6 +36,31 @@ describe('chatRowKind', () => {
expect(row({ author: 'agent', type: 'error' })).toEqual({ kind: 'notify' });
});
+ it('draws a notice addressed to the owner as the Agent, and only by its declared event', () => {
+ const declared = messageTypeNames().flatMap((type) =>
+ specFor(type).agentRenderWhenEvents.map((event) => ({ type, event })));
+ // The Computer Use script's notify and confirm prompt are the producers today.
+ expect(declared).toEqual(expect.arrayContaining([
+ { type: 'notify', event: 'computer_script_notify' },
+ { type: 'notify', event: 'computer_script_confirm' },
+ ]));
+ for (const { type, event } of declared) {
+ const notice = { author: 'agent', type, metadata: { event } };
+ expect(row(notice), `${type}/${event}`).toEqual({ kind: 'agentNotice' });
+ // Presentation only: it is still not the Agent's own authored reply.
+ expect(isAgentAuthored(notice), `${type}/${event}`).toBe(false);
+ // The event belongs to that type; elsewhere it changes nothing.
+ for (const other of messageTypeNames()) {
+ if (specFor(other).agentRenderWhenEvents.includes(event)) continue;
+ expect(row({ author: 'agent', type: other, metadata: { event } }), other)
+ .toEqual(row({ author: 'agent', type: other }));
+ }
+ }
+ // Every other status row keeps its pill.
+ expect(row({ author: 'agent', type: 'notify', metadata: { event: 'backend_failure' } })).toEqual({ kind: 'notify' });
+ expect(row({ author: 'agent', type: 'notify', metadata: {} })).toEqual({ kind: 'notify' });
+ });
+
it('keeps legacy detached results in the boundary family', () => {
expect(row({ author: 'agent', type: 'result', metadata: { detached: true } })).toEqual({ kind: 'boundary' });
});
diff --git a/ui/src/lib/chatRowKind.ts b/ui/src/lib/chatRowKind.ts
index b85f520f3..b5a670fe5 100644
--- a/ui/src/lib/chatRowKind.ts
+++ b/ui/src/lib/chatRowKind.ts
@@ -18,7 +18,7 @@
// ``author`` and ``source`` still decide among the ROLE families below, which is
// what they legitimately describe. What they no longer do is decide whether a
// typed row is that type.
-import { isBoundaryMessage, isNotifyMessageType } from './chatMessageTypes';
+import { isAgentRenderedNotice, isBoundaryMessage, isNotifyMessageType } from './chatMessageTypes';
import { readAnnotationView, type AnnotationView } from './annotationView';
import { specFor } from './messageTypes';
@@ -35,6 +35,7 @@ export type ChatRowKind =
| { kind: 'annotation'; annotation: AnnotationView }
| { kind: 'boundary' }
| { kind: 'notify' }
+ | { kind: 'agentNotice' }
| { kind: 'agent' }
| { kind: 'system' }
| { kind: 'harness' }
@@ -58,6 +59,11 @@ export function chatRowKind(message: ChatRowFields): ChatRowKind {
// A phase boundary and an ``interim`` narration share the muted Agent bubble:
// both are the agent's words that do not end the Turn.
if (isBoundaryMessage(message) || specFor(message.type).render === 'muted') return { kind: 'boundary' };
+ // A notification addressed to the owner (a Computer Use script's notify or
+ // confirm prompt) draws as the Agent's message: a compact pill reads as runtime
+ // status, and a confirm nobody notices times out. A family of its own rather than
+ // ``agent``, because it is not a reply — it must not own the Turn's Vault requests.
+ if (isAgentRenderedNotice(message)) return { kind: 'agentNotice' };
// Runtime notifications and legacy error rows are compact status pills, not
// Agent-authored answers.
if (isNotifyMessageType(message.type)) return { kind: 'notify' };
@@ -87,6 +93,9 @@ export function chatRowKind(message: ChatRowFields): ChatRowKind {
export function isAgentAuthored(message: Pick<ChatRowFields, 'type' | 'author'>): boolean {
// Notify/error rows draw a status pill with no Markdown body at all; excluding
// them keeps this identical to the pre-annotation flag rather than widening it.
+ // An ``agentNotice`` stays excluded although it draws the Agent's bubble: a
+ // script's text may quote what it read off the screen, so it must not mint a
+ // secret-input card or a local-file link.
return !isNotifyMessageType(message.type) && message.author === 'agent';
}
diff --git a/vibe/message_types.json b/vibe/message_types.json
index bb1a69c83..818fddf7c 100644
--- a/vibe/message_types.json
+++ b/vibe/message_types.json
@@ -12,7 +12,8 @@
"unread": false,
"webPush": false,
"webPushWhenEvents": [],
- "render": "none"
+ "render": "none",
+ "agentRenderWhenEvents": []
},
"types": {
"user": {
@@ -81,7 +82,11 @@
"webPushWhenEvents": [
"backend_failure"
],
- "render": "status"
+ "render": "status",
+ "agentRenderWhenEvents": [
+ "computer_script_notify",
+ "computer_script_confirm"
+ ]
},
"vault": {
"transcript": true,
diff --git a/vibe/message_types.py b/vibe/message_types.py
index 90de78aa3..f00a22ab3 100644
--- a/vibe/message_types.py
+++ b/vibe/message_types.py
@@ -21,6 +21,9 @@ _LIST_PROPERTIES = {
"inputAuthors",
"terminalWhenEvents",
"webPushWhenEvents",
+ # Events whose row the Web draws as the Agent's own message rather than as
+ # this type's ``render``: words addressed to the owner, not runtime status.
+ "agentRenderWhenEvents",
}
_ENUM_PROPERTIES = {
"activityRole": {"none", "turn_start", "activity", "boundary", "terminal"},