The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Fix device dictation following stale pane focus (#1005)
Dee91fee170bad8
diff --git a/cli/e2e/devicesProcess.e2e.ts b/cli/e2e/devicesProcess.e2e.tsindex e9ba0fdf1..cafeb6f6c 100644--- a/cli/e2e/devicesProcess.e2e.ts+++ b/cli/e2e/devicesProcess.e2e.ts@@ -135,6 +135,28 @@ describe('the devices in their own process', () => {await until('the last window to clear the dial', () => dial.messages.slice(since).some(m => m.t === 'agents.end' && m.tab === '') || null, 5000, 50)})+ it('restores a non-first focused pane on late attachment and after the devices process restarts', async () => {+ const { d, desk } = await fresh()+ const window = await LocalClient.connect(d)+ try {+ const first = await create(d, window, 'devices-first-pane')+ const focused = await create(d, window, 'devices-focused-pane')+ onTab(window, [first, focused])+ window.send('app_focus', { agentId: focused })+ // The socket has processed the selection before there is a device service to hear it.+ await window.request('agents_list', {})+ const dial = await desk.plug('E2E-A', 'e2:e0:00:00:00:0a')+ await heard(dial, 'focus', 0, focused, 20_000)+ const beforeRestart = dial.messages.length+ for (const pid of devicesPids(d)) process.kill(pid, 'SIGKILL')+ await until('the devices to reconnect', () => connections(d) >= 2 || null, 30_000, 200)+ await dial.welcomeAfter(beforeRestart, 30_000)+ await heard(dial, 'focus', beforeRestart, focused, 20_000)+ // No repeat app_focus was needed; the running window kept its original selection.+ expect(d.coresStarted()).toBe(1)+ } finally { window.close() }+ })+it('killed outright: agents and windows go on, ⌘K says so at once, and the dial comes back to its question and its working tile', async () => {const { d, desk } = await fresh()const window = await LocalClient.connect(d)diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.tsindex f6c960872..87b0aff0a 100644--- a/cli/src/cable/cableHost.spec.ts+++ b/cli/src/cable/cableHost.spec.ts@@ -44,6 +44,17 @@ function fleetOf(machines: FleetMachine[]): MachineFleet {const REMOTE: FleetMachine = { machineId: 'other', name: 'office-imac', state: 'ready', authMode: 'remote' }+it('reads the current window selection for each recording instead of caching it with the roster', () => {+ let focused: { machineId: string; agentId: string } | undefined+ const host = new DaemonCableHost(wiring({ appFocus: () => focused }))+ expect(host.appFocus()).toBeUndefined()+ focused = { machineId: 'other', agentId: 'a2' }+ expect(host.appFocus()).toEqual(focused)+ focused = { machineId: 'mine', agentId: 'a1' }+ expect(host.appFocus()).toEqual(focused)+ expect(new DaemonCableHost(wiring()).appFocus()).toBeUndefined()+})+describe('notification read receipts', () => {it('routes the exact occurrence to its machine without focusing or answering', () => {const w = wiring({ notificationRead: vi.fn(), opened: vi.fn(), focused: vi.fn() })diff --git a/cli/src/cable/cableHost.ts b/cli/src/cable/cableHost.tsindex ccb1a3ad4..f753bdcf0 100644--- a/cli/src/cable/cableHost.ts+++ b/cli/src/cable/cableHost.ts@@ -101,6 +101,7 @@ export interface CableHostWiring {scrolled?: (phase: 'down' | 'move' | 'up', dy: number, velocity: number) => void/** The dial came, went, or started taking an update — see CableSession's onDialStatus. */dialStatus?: (status: DialStatus) => void+ appFocus?: CableHost['appFocus']/** Offer a spoken task to the desktop window's palette. Omitted when there is no window plumbing. */routeInWindow?: (text: string, cmd?: string) => Promise<WindowRoute>selectPassage?: (command: SelectionCommand) => Promise<SelectionResult>@@ -316,6 +317,10 @@ export class DaemonCableHost implements CableHost {return locale.startsWith('vi') ? 'vi' : 'en'}+ appFocus(): { machineId: string; agentId: string } | undefined {+ return this.wiring.appFocus?.()+ }+/*** The window's tiles on its active tab, in tile order. This IS the dial's list — see listAgents. A* router this host builds for itself reads it to hold a tile whose machine dropped out; the fleetdiff --git a/cli/src/cable/cableSession.spec.ts b/cli/src/cable/cableSession.spec.tsindex 54b81aa5c..77ad3341b 100644--- a/cli/src/cable/cableSession.spec.ts+++ b/cli/src/cable/cableSession.spec.ts@@ -350,7 +350,8 @@ describe('cable session', () => {? { ok: true, selectionId: 'pick-1', revision: 4, rows: 2, extending: true,excerpt: 'source output', text: ' source line\nsecond line' }: { ok: false, error: 'Closed' })- const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Compare this with your plan.'), route: vi.fn() })+ const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Compare this with your plan.'), route: vi.fn(),+ appFocus: () => ({ machineId: 'mac-local', agentId: 'a1' }) })const { session, port } = await connect(host)try {port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' }); await settle()@@ -497,7 +498,8 @@ describe('cable session', () => {? { ok: true, selectionId: 'selection-1', revision: 4, rows: 2, extending: true,excerpt: 'quoted output', text: ' first line\nsecond line' }: { ok: false, error: 'Selection closed.' })- const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Explain this.') })+ const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Explain this.'),+ appFocus: () => ({ machineId: 'mac-local', agentId: 'a1' }) })const { session, port } = await connect(host)try {port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })@@ -1215,6 +1217,23 @@ describe('cable session', () => {await session.stop()})+ it.each([{ selections: ['a2'] }, { selections: ['a1', 'a2'] }])('delivers the latest focus when $selections supersedes an unsent selection', async ({ selections }) => {+ const { session, port, host } = await connect()+ try {+ port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })+ await settle()+ port.sent.length = 0+ let finish!: () => void+ vi.spyOn(host, 'listAgents').mockImplementationOnce(() => new Promise(resolve => { finish = () => resolve(AGENTS) }))+ const first = session.followApp('mac-local', 'a2')+ await vi.waitFor(() => expect(finish).toBeDefined())+ const pending = selections.map(agentId => session.followApp('mac-local', agentId))+ finish()+ await Promise.all([first, ...pending])+ expect(port.sent.filter(m => m.t === 'focus')).toEqual([{ t: 'focus', agentId: 'a2' }])+ } finally { await session.stop() }+ })+it('forwards a whole stroke, including the reports that carry no travel', async () => {// The ends of a stroke are the point of the message, not padding around it: a `down` with nothing in// it is what stops a fling still running on the far side, and an `up` with nothing in it is a finger@@ -1371,7 +1390,7 @@ describe('cable session', () => {})it('routes a voice turn that names no agent', async () => {- const host = makeHost()+ const host = makeHost({ appFocus: () => ({ machineId: 'mac-local', agentId: 'a2' }) })const { session, port } = await connect(host)port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })await settle()@@ -1498,6 +1517,43 @@ describe('cable session', () => {await session.stop()})+ it('pins ordinary dictation to the pane focused when recording begins, even if the dial is stale', async () => {+ let focused = 'a2'+ let finish!: (text: string) => void+ const transcribe = vi.fn(() => new Promise<string>(resolve => { finish = resolve }))+ const host = makeHost({+ appFocus: () => ({ machineId: 'mac-local', agentId: focused }),+ transcribe,+ route: vi.fn(),+ })+ const { session, port } = await connect(host)+ try {+ port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })+ await settle()+ port.say({ t: 'voice.begin', agentId: 'a1', uploadId: 'focused-pane' })+ port.pcm(Buffer.alloc(3200, 1))+ // Focus can change during capture and during the asynchronous transcription.+ focused = 'a1'+ port.say({ t: 'voice.end', uploadId: 'focused-pane' })+ await vi.waitFor(() => expect(transcribe).toHaveBeenCalledOnce())+ finish('one two three four five')+ await vi.waitFor(() => expect(host.sendTurn).toHaveBeenCalledOnce())+ expect(host.sendTurn).toHaveBeenCalledWith('a2', 'one two three four five')+ expect(host.route).not.toHaveBeenCalled()+ expect(port.sent).toContainEqual(expect.objectContaining({ t: 'voice.transcript', agentId: 'a2' }))+ } finally { await session.stop() }+ })++ it('restores the focused pane when a dial attaches after the window selected it', async () => {+ const host = makeHost({ appFocus: () => ({ machineId: 'mac-local', agentId: 'a2' }) })+ const { session, port } = await connect(host)+ try {+ port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })+ await vi.waitFor(() => expect(port.sent).toContainEqual({ t: 'focus', agentId: 'a2' }))+ expect(port.types().indexOf('agents.end')).toBeLessThan(port.types().indexOf('focus'))+ } finally { await session.stop() }+ })+it('transcribes at the rate the dial states, not a guess', async () => {// 8 kHz described as 16 kHz is not slightly-off speech: the container lies about itself and the// transcriber answers with an empty string.diff --git a/cli/src/cable/cableSession.ts b/cli/src/cable/cableSession.tsindex f6efca0e3..b79cc0d5e 100644--- a/cli/src/cable/cableSession.ts+++ b/cli/src/cable/cableSession.ts@@ -251,6 +251,8 @@ export interface CableHost {selectSwarm(swarmId: string): voidappName(): stringvoiceLang(): string+ /** The window's latest selected pane, including selections made before this dial attached. */+ appFocus?(): { machineId: string; agentId: string } | undefined/** The active tab's agents, in tile order — and nothing else. Empty with no window or an empty tab. */listAgents(): Promise<CableAgent[]>/** Pane rows and workspace identity from the same desktop announcement. */@@ -528,6 +530,8 @@ export class CableSession {* and it is the only thing the daemon ever re-asserts.*/private desiredFocus = ''+ /** A superseded list refresh can record the selection without ever writing its focus frame. */+ private appFocusUnsent = false/*** App-driven machine/focus changes are one transaction. Without this queue, two quick desktop clicks* can interleave their machine lists and let the older click focus last.@@ -1007,6 +1011,8 @@ export class CableSession {// screen. A repeat greeting from the same dial on the same image is a keepalive and is skipped,// which is the whole reason the branch exists.await this.pushAgents()+ const focused = this.host.appFocus?.()+ if (focused) await this.followApp(focused.machineId, focused.agentId)}// Offered on every greeting, but only ONCE per version per session: accepting makes the dial erase// a flash slot before it answers, so a cadence of retries would spend erase cycles on the user's@@ -1100,6 +1106,7 @@ export class CableSession {// Remember where the dial IS, not just that it said so: followApp() compares against this to// avoid echoing the dial's own move back at it.this.desiredFocus = agentId+ this.appFocusUnsent = falsethis.host.focus(agentId)}return@@ -1303,8 +1310,13 @@ export class CableSession {: Promise.resolve({ ok: false, error: 'Choose a passage before searching.' } as const)).catch(() => ({ ok: false, error: 'Could not open output search. Try again.' })) }}+ // The dial can still name the previous pane while a focus frame is in flight, or after the+ // devices process restarted. Ordinary dictation belongs to the window's selection at capture+ // start. Quoted text, carried text, questions and edits already have their own pinned targets.+ const focused = str('agentId') && !formVoice && !searchVoice && !draftVoice && !questionVoice &&+ !carryVoice && !('selectionId' in msg) ? this.host.appFocus?.() : undefinedthis.voice = {- agentId: str('agentId'),+ agentId: focused?.agentId ?? str('agentId'),cmd: str('cmd'),lang: str('lang') ?? this.host.voiceLang(),// The DIAL's rate, never this side's guess. Describing 8 kHz audio as 16 kHz does not make the@@ -2229,7 +2241,8 @@ export class CableSession {async focusAgent(agentId: string): Promise<void> {this.desiredFocus = agentId- await this.send({ t: 'focus', agentId })+ const sent = await this.send({ t: 'focus', agentId })+ if (sent && this.desiredFocus === agentId) this.appFocusUnsent = false}/**@@ -2266,7 +2279,7 @@ export class CableSession {// this older transaction focus after it finishes.// Already where the window is: nothing to command. The record is right either way — this is the// window FOLLOWING the dial, and the dial's own report wrote it.- if (generation !== this.appFocusGeneration || agentId === this.desiredFocus) return+ if (generation !== this.appFocusGeneration || (agentId === this.desiredFocus && !this.appFocusUnsent)) return// Said before the frame goes out, not after: the frame itself succeeds either way.const unknown = this.host.knows?.(agentId) === falsethis.log(`cable: following the app to agent ${agentId}${unknown ? ' — NOT in this daemon\'s list, the dial has no tile for it' : ''}`)@@ -2286,6 +2299,7 @@ export class CableSession {// we are moving to, not the one we are leaving — otherwise the dial visibly steps onto the old// tile on its way. Cheap when nothing changed: syncAgents returns without sending a frame.this.desiredFocus = agentId+ this.appFocusUnsent = trueawait this.syncAgents()if (generation !== this.appFocusGeneration) returnawait this.focusAgent(agentId) // writes the recorddiff --git a/cli/src/services/devices.spec.ts b/cli/src/services/devices.spec.tsindex 5695202ee..f412bc63c 100644--- a/cli/src/services/devices.spec.ts+++ b/cli/src/services/devices.spec.ts@@ -282,7 +282,10 @@ describe('the window bridges, wired to the core', () => {expect(selection.wiring.focus()).toBeUndefined()expect(visit.wiring.focus()).toBeUndefined()expect(form.wiring.focus()).toBeUndefined()+ expect(seen.host.appFocus!()).toBeUndefined()port.windowFocus({ voice: { machineId: 'm1', agentId: 'a1', connId: 'w1' }, form: { machineId: 'm1', connId: 'w1' } })+ expect(seen.host.appFocus!()).toMatchObject({ machineId: 'm1', agentId: 'a1' })+ expect(calls('followApp')).toEqual([['m1', 'a1']])expect(selection.wiring.focus()).toEqual({ machineId: 'm1', agentId: 'a1', connId: 'w1' })expect(visit.wiring.focus()).toEqual({ machineId: 'm1', agentId: 'a1', connId: 'w1' })expect(form.wiring.focus()).toEqual({ machineId: 'm1', connId: 'w1' })@@ -305,6 +308,9 @@ describe('the window bridges, wired to the core', () => {{ type: 'voice_route_request', payload: { voiceId: 'v2', text: 'fix it', cmd: 'review' } },])expect(logs).toContain('[route] voice → the window · 6 bytes · /review')+ port.windowFocus({ voice: null, form: null })+ expect(seen.host.appFocus!()).toBeUndefined()+ expect(calls('followApp')).toEqual([['m1', 'a1']])})it('hand each window\'s answer to its bridge, and tell them a window left', () => {diff --git a/cli/src/services/devices.ts b/cli/src/services/devices.tsindex 4142f91ed..245ddbff0 100644--- a/cli/src/services/devices.ts+++ b/cli/src/services/devices.ts@@ -133,6 +133,7 @@ export function startDevices(core: CoreApi, ports: CorePorts, deps: DevicesDeps)sessions: () => core.agents.advertised(),refresh: deps.refresh,displayName: (session) => core.agents.displayName(session),+ appFocus: () => focus.voice ?? undefined,activityText: (agentId) => core.agents.activityText(agentId),machineName: () => core.machine.name(),machineId: () => core.machine.id(),@@ -279,6 +280,9 @@ export function startDevices(core: CoreApi, ports: CorePorts, deps: DevicesDeps).then((result) => { if (!result.ok) log(`[cable] settings for ${id || 'no device'}: ${result.error}`) })),windowFocus: (next) => {focus = next+ // State replay after a service restart carries windowFocus, without another appFocus event.+ const selected = next.voice+ if (selected) part.call('dial', () => cable.followApp(selected.machineId, selected.agentId))part.call('window', () => windowSelection.focusChanged())},windowReply: (kind, connId, machineId, payload) => part.call('window', () =>