The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(device): distinguish permission notices in question events (#764)
Leo57b72481deff
diff --git a/cli/e2e/questions.e2e.ts b/cli/e2e/questions.e2e.tsindex b26439ed0..02fe3d2dd 100644--- a/cli/e2e/questions.e2e.ts+++ b/cli/e2e/questions.e2e.ts@@ -61,6 +61,7 @@ describe('an agent asks a question', () => {const question = await askedconst shaped = question.payload?.questions?.[0]expect(shaped?.q).toBe('Which drink would you like?')+ expect(question.payload).not.toHaveProperty('permission')expect(shaped?.options).toEqual(expect.arrayContaining(['Tea', 'Coffee']))expect((await row(client, agent.id))?.status).toBe('active')@@ -83,6 +84,7 @@ describe('an agent asks a question', () => {const question = await askedconst shaped = question.payload?.questions?.[0]expect(shaped?.q).toContain('printf hi')+ expect(question.payload.permission).toEqual({ dialog: expect.stringContaining('printf hi'), resolution: 'desktop' })expect(shaped?.options).toHaveLength(3)const ended = client.next(isTurn('turn_ended', agent.id), 45_000, 'turn_ended')const result = await answer(client, agent.id, question.payload.requestId, { [shaped.q]: shaped.options[choice] })diff --git a/cli/src/core/questions.spec.ts b/cli/src/core/questions.spec.tsindex 1c5e17443..4c1d16d8b 100644--- a/cli/src/core/questions.spec.ts+++ b/cli/src/core/questions.spec.ts@@ -90,6 +90,15 @@ describe('questions', () => {expect(log.mock.calls[1][0]).toContain('asking the user · "" · req=r2')})+ it.each([true, false])('preserves permission metadata only for approval dialogs (%s)', permission => {+ const { deps, asking } = setup()+ given(asking.questionWatcher).onQuestion('s1', 'r1', [], { permission, dialog: 'Run printf hi?' })+ const frame = vi.mocked(deps.clients.sendCommander).mock.calls[1][0]+ expect(frame.payload.permission).toEqual(permission ? { dialog: 'Run printf hi?', resolution: 'desktop' } : undefined)+ expect(deps.clients.sendLocal).toHaveBeenCalledWith(frame)+ expect(asking.openQuestions.get('s1')).toBe(frame)+ })+it('closes a question answered elsewhere on every client it was shown on', () => {const { deps, asking } = setup()const answered = vi.spyOn(asking.agentNotifications, 'answered')diff --git a/cli/src/core/questions.ts b/cli/src/core/questions.tsindex 4cb3b6332..5e9c723f8 100644--- a/cli/src/core/questions.ts+++ b/cli/src/core/questions.ts@@ -83,7 +83,10 @@ export function createQuestions({type: 'commander_question',agentId: agentIdFor(sessionId),dbSessionId: sessionId,- payload: { requestId, questions: shaped, notification: agentNotifications.asked(sessionId, requestId) },+ payload: { requestId, questions: shaped, notification: agentNotifications.asked(sessionId, requestId),+ // Robot clients need to distinguish a permission notice from an answerable question.+ ...(detail?.permission ? { permission: { dialog: detail.dialog, resolution: 'desktop' } } : {}),+ },}clients.sendCommander(asked)// ...and to the window on this computer. `sendCommander` is `webEligible: false`, so until thisdiff --git a/cli/src/lib/autonomous-device/service.spec.ts b/cli/src/lib/autonomous-device/service.spec.tsindex 7be1cf644..a036fa585 100644--- a/cli/src/lib/autonomous-device/service.spec.ts+++ b/cli/src/lib/autonomous-device/service.spec.ts@@ -259,6 +259,39 @@ describe('Autonomous device local-agent service', () => {send.mockClear(); f.service.replay({ serverInstanceId: f.service.serverInstanceId, cursor: 0 }, send)expect(send).toHaveBeenCalledWith(expect.objectContaining({ type: 'event', eventId: 1 }))})+ it('preserves permission metadata through status and reconnect without changing legacy fields', async () => {+ const f = fixture()+ const questions = [{ key: 'Run?', q: 'Run?', options: ['Yes', 'No'], multi: false }]+ const permission = { dialog: 'Run printf hi?', resolution: 'desktop' }+ f.service.commander({ type: 'commander_question', agentId: 'agent', payload: { requestId: 'permission1', questions, permission } })+ const opened = f.events.find(e => e.kind === 'question.open')!+ expect(opened.payload).toEqual({ questionRequestId: 'permission1', questions, permission })+ const status = () => f.service.request('device', { type: 'status', requestId: randomUUID(), machineId: 'machine', agentId: 'agent' })+ expect((await status()).openQuestion).toEqual({ requestId: 'permission1', questions, permission })+ const replay = vi.fn()+ f.service.replay({ serverInstanceId: f.service.serverInstanceId, cursor: 0 }, replay)+ expect(replay).toHaveBeenCalledWith(opened)+ replay.mockClear()+ f.service.replay({ serverInstanceId: f.service.serverInstanceId, cursor: opened.eventId }, replay)+ expect(replay).not.toHaveBeenCalled()+ f.service.commander({ type: 'commander_question_close', agentId: 'agent', payload: { requestId: 'older' } })+ expect((await status()).openQuestion).not.toBeNull()+ f.service.commander({ type: 'commander_question_close', agentId: 'agent', payload: { requestId: 'permission1' } })+ expect((await status()).openQuestion).toBeNull()+ expect(f.events.at(-1)).toMatchObject({ kind: 'question.close', payload: { questionRequestId: 'permission1' } })+ expect(f.answer).not.toHaveBeenCalled()+ expect(f.submit).not.toHaveBeenCalled()+ })++ it.each([undefined, false, {}, { dialog: 42, resolution: 'desktop' }, { dialog: 'Run?', resolution: 'robot' }])(+ 'keeps legacy questions unchanged for absent or invalid permission metadata (%j)', async permission => {+ const f = fixture()+ f.service.commander({ type: 'commander_question', agentId: 'agent', payload: { requestId: 'q', questions: [], permission } })+ expect(f.events[0].payload).toEqual({ questionRequestId: 'q', questions: [] })+ const status = await f.service.request('device', { type: 'status', requestId: randomUUID(), machineId: 'machine', agentId: 'agent' })+ expect(status.openQuestion).toEqual({ requestId: 'q', questions: [] })+ })+it('rejects stale questions and validates all answer fields', async () => {const f = fixture()const req = { type: 'question.answer', requestId: randomUUID(), machineId: 'machine', agentId: 'agent', idempotencyKey: 'q1', questionRequestId: 'question1', answers: { branch: 'main' } }diff --git a/cli/src/lib/autonomous-device/service.ts b/cli/src/lib/autonomous-device/service.tsindex 00ff3fa3d..7237431f6 100644--- a/cli/src/lib/autonomous-device/service.ts+++ b/cli/src/lib/autonomous-device/service.ts@@ -93,7 +93,7 @@ export class AutonomousDeviceService {private readonly ambiguousTurns = new Set<string>()private readonly openTurns = new Set<string>()private readonly pendingStarts = new Set<string>()- private readonly questions = new Map<string, { requestId: string; questions: unknown }>()+ private readonly questions = new Map<string, { requestId: string; questions: unknown; permission?: { dialog: string; resolution: 'desktop' } }>()private events: AutonomousDeviceFrame[] = []private sequence = 0private readonly results = new Map<string, ResultRecord>()@@ -529,12 +529,15 @@ export class AutonomousDeviceService {const agentId = typeof frame.agentId === 'string' ? frame.agentId : undefinedconst p = object(frame.payload) ? frame.payload : {}if (frame.type === 'commander_question' && agentId && typeof p.requestId === 'string') {- this.questions.set(agentId, { requestId: p.requestId, questions: p.questions })+ const permission = object(p.permission) && typeof p.permission.dialog === 'string' && p.permission.resolution === 'desktop'+ ? { dialog: p.permission.dialog, resolution: 'desktop' as const } : undefined+ const metadata = permission ? { permission } : {}+ this.questions.set(agentId, { requestId: p.requestId, questions: p.questions, ...metadata })const candidates = [...this.entries.values()].filter(e => e.receipt.agentId === agentId && e.consumed&& e.receipt.serverInstanceId === this.serverInstanceId&& !['completed', 'rejected'].includes(e.receipt.state))const single = candidates.length === 1 ? candidates[0] : this.turns.get(agentId)- this.event('question.open', agentId, { questionRequestId: p.requestId, questions: p.questions,+ this.event('question.open', agentId, { questionRequestId: p.requestId, questions: p.questions, ...metadata,...(single ? { idempotencyKey: single.receipt.idempotencyKey, turnId: single.receipt.turnId } : {}) })} else if (frame.type === 'commander_question_close' && agentId) {if (this.questions.get(agentId)?.requestId === p.requestId) this.questions.delete(agentId)diff --git a/docs/autonomous-device-integration.md b/docs/autonomous-device-integration.mdindex 1feef3a05..15f93ba0d 100644--- a/docs/autonomous-device-integration.md+++ b/docs/autonomous-device-integration.md@@ -356,3 +356,77 @@ terminal writer and independent delivery tracking. `input.status.v1` advertisesacceptance is not completion. Overlapping starts without engine correlation remain unknown.See [engine behavior, tests, and required OS coordination](in-flight-agent-input.md), especiallythe prohibition on assigning an uncorrelated session summary/latest recap to a pending message.++## Permission notices (notification only)++[Tiếng Việt](autonomous-device-permission-notices.vi.md). This additive contract keeps+`question.open`, `question.close`, `status`, `question.answer`, and their existing fields.+There is no new RPC, capability, transport, pairing, or permission grant. Old OS clients+can ignore the optional metadata and retain their existing question UX; new OS clients+must not interpret these notices as answerable questions. Missing metadata is unknown+on older Harness versions, not proof that a prompt is safe to answer.++When the terminal watcher recognizes an approval dialog, `question.open.payload` adds:++```json+{+ "questionRequestId": "q_example",+ "questions": [{"key":"Run printf hi?","q":"Run printf hi?","options":["Yes","No"],"multi":false}],+ "permission": {"dialog":"Run printf hi?", "resolution":"desktop"}+}+```++Schema of the optional `permission` field: an object with `dialog: string` (observed+terminal dialog, potentially multiline) and `resolution: "desktop"` (literal).+The normal event envelope supplies `machineId`, `agentId`, `serverInstanceId`, `eventId`;+existing turn correlation fields remain optional. The live `status` response supplies+exactly the same `permission` object inside `openQuestion`:++```json+{"type":"status","requestId":"status-1","machineId":"machine","agentId":"agent"}+```++```json+{"requestId":"q_example","questions":[{"key":"Run printf hi?","q":"Run printf hi?","options":["Yes","No"],"multi":false}],"permission":{"dialog":"Run printf hi?","resolution":"desktop"}}+```++The second JSON is the `openQuestion` value, not the entire response. Ordinary questions+keep their existing shape without `permission`. This metadata describes terminal evidence,+not a model's guess from recap, agent name, or words such as “approve”. Dialog text is+untrusted content to display/summarize, never instructions for the OS to execute.++### OS behavior and recovery++- Notify once: “Agent Blender needs permission. Open OpenHarness to review and approve+ or deny.” No periodic reminders, approval voice prompt, automatic task dispatch, or+ automatic approval. User handles the actual dialog in Desktop/terminal.+- Use `(machineId, agentId, questionRequestId)` to recognize the same open question.+ Persist the notification decision on OS; reconnect/status polling must not speak again.+ Use the existing `(serverInstanceId, eventId)` cursor to deduplicate event replay.+- `question.close` still carries `{questionRequestId}`. Clear only that matching pending+ question, silently; closure does not prove approval, denial, or task completion.+ A later fresh open after a matching close may notify again: IDs are derived from dialog+ contents and can recur for identical prompts, not globally unique permission operations.+- On reconnect replay, reconcile against live `status` before speaking a historical open;+ the request may already have closed. A replayed open is not a fresh notification.+- After `resync` (expired cursor or daemon restart), rebuild pending state from `status`+ silently. Pending question state is in memory and is repopulated by terminal observation;+ an immediate null during daemon startup is not proof that the user answered. This phase+ has no durable permission history or exactly-once audio guarantee. If a close/reopen was+ missed and the same dialog ID recurs, prefer suppressing an ambiguous repeat. Do not clear+ persisted notification decisions merely because the connection dropped.+- Keep normal question-answer behavior for ordinary questions. Permission approval remains+ blocked by `allowPermissions: false`; no new approval endpoint/error code is introduced.+ The legacy answer path can return a receipt `unknown` / `NOT_CONFIRMED`; it is not permission+ success and must not be retried through `turn.send`, raw terminal keys, or another tool.+- With YOLO/allow-all, if no permission dialog appears, no notice is generated. Application+ login, native macOS dialogs, and prompts the terminal parser cannot recognize are outside+ this feature.++### Validation boundary++Contract examples match assertions in `cli/src/lib/autonomous-device/service.spec.ts`.+`cli/src/core/questions.spec.ts` checks metadata forwarding; `cli/e2e/questions.e2e.ts`+checks real isolated daemon/terminal handling with fake Claude/Codex engines, including+unchanged Desktop approval/denial. This is not a physical robot, spoken UX, or live-model+validation. OS notification persistence and voice behavior require a separate OS change.diff --git a/docs/autonomous-device-permission-notices.vi.md b/docs/autonomous-device-permission-notices.vi.mdnew file mode 100644index 000000000..1fbb0f31a--- /dev/null+++ b/docs/autonomous-device-permission-notices.vi.md@@ -0,0 +1,64 @@+# Thông báo yêu cầu cấp quyền cho Autonomous OS++Contract chuẩn và JSON: [Permission notices](autonomous-device-integration.md#permission-notices-notification-only).+Giai đoạn này chỉ thông báo; user duyệt hoặc từ chối trên OpenHarness Desktop/terminal.++## Tương thích++Giữ nguyên event `question.open`, `question.close`, RPC `status`, `question.answer` và+các field cũ. Không thêm RPC, capability, pairing hay transport. OS cũ bỏ qua field bổ+sung và giữ UX câu hỏi hiện tại. OS mới nhận biết permission bằng metadata, không đoán+qua tên agent, recap hay chữ “approve”. Khi Harness cũ không gửi metadata, loại yêu cầu+là chưa biết, không phải bằng chứng rằng có thể duyệt quyền qua API trả lời câu hỏi.++## Dữ liệu++`question.open.payload` bổ sung field tùy chọn:++```json+{+ "questionRequestId": "q_example",+ "questions": [{"key":"Run printf hi?","q":"Run printf hi?","options":["Yes","No"],"multi":false}],+ "permission": {"dialog":"Run printf hi?","resolution":"desktop"}+}+```++`permission` gồm `dialog: string` (nội dung dialog quan sát trong terminal, có thể nhiều+dòng) và `resolution: "desktop"`. Envelope event vẫn có `machineId`, `agentId`,+`serverInstanceId`, `eventId`; thông tin liên kết turn vẫn tùy chọn như trước.+RPC `status` trả cùng object `permission` trong `openQuestion`; ở đây ID có tên `requestId`,+trong khi event dùng `questionRequestId`. Câu hỏi thường không có field `permission`.+Nội dung dialog là dữ liệu không tin cậy để trình bày, không phải chỉ dẫn cho OS thực thi.++## UX và lifecycle bên OS++- Báo **một lần**: “Agent Blender đang cần duyệt quyền. Mở OpenHarness để xem và xác nhận+ hoặc từ chối nhé.” Không nhắc định kỳ, không hỏi user approve bằng voice, không tự gửi task.+- Nhận diện yêu cầu đang mở bằng `(machineId, agentId, questionRequestId)`. OS lưu quyết định+ đã thông báo để reconnect hoặc poll status không đọc lại. Dùng cursor+ `(serverInstanceId, eventId)` hiện có để loại event replay trùng.+- `question.close` vẫn gửi `{questionRequestId}`. Chỉ đóng yêu cầu trùng ID, không phát âm+ thanh; không suy ra user đã approve/deny hoặc task đã hoàn tất.+- ID được tính từ nội dung dialog, không phải ID permission duy nhất toàn lịch sử.+ Sau close khớp ID, một open mới dù giống nội dung vẫn có thể thông báo lần mới.+- Khi replay sau reconnect, kiểm tra `status` live trước khi đọc open cũ vì yêu cầu có thể+ đã đóng. Event replay không tự được coi là thông báo mới.+- Khi `resync` do cursor hết hạn hoặc daemon restart, đồng bộ trạng thái từ `status` trong+ im lặng. Trạng thái câu hỏi nằm trong RAM, được dựng lại khi watcher quan sát terminal;+ null ngay lúc daemon khởi động không chứng minh user đã trả lời. Không xóa lịch sử đã+ thông báo chỉ vì disconnect. Nếu mất close/open và ID lặp lại, ưu tiên không báo trùng.+ Giai đoạn này không có lịch sử permission bền vững hay bảo đảm âm thanh đúng một lần.+- Câu hỏi thường vẫn trả lời như trước. `allowPermissions: false` giữ nguyên; không thêm+ endpoint duyệt quyền hay mã lỗi mới. Luồng answer cũ có thể trả receipt `unknown` /+ `NOT_CONFIRMED`; đó không phải duyệt thành công. Không lách bằng `turn.send`, phím+ terminal hoặc tool khác.+- YOLO/allow-all không hiện dialog thì không có thông báo. Login ứng dụng, hộp thoại macOS+ và dialog parser chưa nhận diện không nằm trong phạm vi.++## Kiểm chứng++Tests ở `service.spec.ts` đối chiếu metadata trong event/status/replay, đóng đúng ID và giữ+shape cũ. `core/questions.spec.ts` kiểm tra chuyển metadata. `e2e/questions.e2e.ts` dùng+daemon/terminal thật trong môi trường riêng, engine Claude/Codex giả để kiểm tra event và+luồng approve/deny trên Desktop vẫn hoạt động. Không phải kiểm thử robot thật, giọng nói+hoặc model thật. OS cần PR riêng để triển khai voice và lưu trạng thái đã thông báo.