Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Fix device dictation following stale pane focus (#1005)

Dee91fee170bad8
diff --git a/cli/e2e/devicesProcess.e2e.ts b/cli/e2e/devicesProcess.e2e.ts
index e9ba0fdf1..cafeb6f6c 100644
--- a/cli/e2e/devicesProcess.e2e.ts
+++ b/cli/e2e/devicesProcess.e2e.ts
@@ -135,6 +135,28 @@ describe('the devices in their own process', () => {
await until('the last window to clear the dial', () => dial.messages.slice(since).some(m => m.t === 'agents.end' && m.tab === '') || null, 5000, 50)
})
+ it('restores a non-first focused pane on late attachment and after the devices process restarts', async () => {
+ const { d, desk } = await fresh()
+ const window = await LocalClient.connect(d)
+ try {
+ const first = await create(d, window, 'devices-first-pane')
+ const focused = await create(d, window, 'devices-focused-pane')
+ onTab(window, [first, focused])
+ window.send('app_focus', { agentId: focused })
+ // The socket has processed the selection before there is a device service to hear it.
+ await window.request('agents_list', {})
+ const dial = await desk.plug('E2E-A', 'e2:e0:00:00:00:0a')
+ await heard(dial, 'focus', 0, focused, 20_000)
+ const beforeRestart = dial.messages.length
+ for (const pid of devicesPids(d)) process.kill(pid, 'SIGKILL')
+ await until('the devices to reconnect', () => connections(d) >= 2 || null, 30_000, 200)
+ await dial.welcomeAfter(beforeRestart, 30_000)
+ await heard(dial, 'focus', beforeRestart, focused, 20_000)
+ // No repeat app_focus was needed; the running window kept its original selection.
+ expect(d.coresStarted()).toBe(1)
+ } finally { window.close() }
+ })
+
it('killed outright: agents and windows go on, ⌘K says so at once, and the dial comes back to its question and its working tile', async () => {
const { d, desk } = await fresh()
const window = await LocalClient.connect(d)
diff --git a/cli/src/cable/cableHost.spec.ts b/cli/src/cable/cableHost.spec.ts
index f6c960872..87b0aff0a 100644
--- a/cli/src/cable/cableHost.spec.ts
+++ b/cli/src/cable/cableHost.spec.ts
@@ -44,6 +44,17 @@ function fleetOf(machines: FleetMachine[]): MachineFleet {
const REMOTE: FleetMachine = { machineId: 'other', name: 'office-imac', state: 'ready', authMode: 'remote' }
+it('reads the current window selection for each recording instead of caching it with the roster', () => {
+ let focused: { machineId: string; agentId: string } | undefined
+ const host = new DaemonCableHost(wiring({ appFocus: () => focused }))
+ expect(host.appFocus()).toBeUndefined()
+ focused = { machineId: 'other', agentId: 'a2' }
+ expect(host.appFocus()).toEqual(focused)
+ focused = { machineId: 'mine', agentId: 'a1' }
+ expect(host.appFocus()).toEqual(focused)
+ expect(new DaemonCableHost(wiring()).appFocus()).toBeUndefined()
+})
+
describe('notification read receipts', () => {
it('routes the exact occurrence to its machine without focusing or answering', () => {
const w = wiring({ notificationRead: vi.fn(), opened: vi.fn(), focused: vi.fn() })
diff --git a/cli/src/cable/cableHost.ts b/cli/src/cable/cableHost.ts
index ccb1a3ad4..f753bdcf0 100644
--- a/cli/src/cable/cableHost.ts
+++ b/cli/src/cable/cableHost.ts
@@ -101,6 +101,7 @@ export interface CableHostWiring {
scrolled?: (phase: 'down' | 'move' | 'up', dy: number, velocity: number) => void
/** The dial came, went, or started taking an update — see CableSession's onDialStatus. */
dialStatus?: (status: DialStatus) => void
+ appFocus?: CableHost['appFocus']
/** Offer a spoken task to the desktop window's palette. Omitted when there is no window plumbing. */
routeInWindow?: (text: string, cmd?: string) => Promise<WindowRoute>
selectPassage?: (command: SelectionCommand) => Promise<SelectionResult>
@@ -316,6 +317,10 @@ export class DaemonCableHost implements CableHost {
return locale.startsWith('vi') ? 'vi' : 'en'
}
+ appFocus(): { machineId: string; agentId: string } | undefined {
+ return this.wiring.appFocus?.()
+ }
+
/**
* The window's tiles on its active tab, in tile order. This IS the dial's list — see listAgents. A
* router this host builds for itself reads it to hold a tile whose machine dropped out; the fleet
diff --git a/cli/src/cable/cableSession.spec.ts b/cli/src/cable/cableSession.spec.ts
index 54b81aa5c..77ad3341b 100644
--- a/cli/src/cable/cableSession.spec.ts
+++ b/cli/src/cable/cableSession.spec.ts
@@ -350,7 +350,8 @@ describe('cable session', () => {
? { ok: true, selectionId: 'pick-1', revision: 4, rows: 2, extending: true,
excerpt: 'source output', text: ' source line\nsecond line' }
: { ok: false, error: 'Closed' })
- const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Compare this with your plan.'), route: vi.fn() })
+ const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Compare this with your plan.'), route: vi.fn(),
+ appFocus: () => ({ machineId: 'mac-local', agentId: 'a1' }) })
const { session, port } = await connect(host)
try {
port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' }); await settle()
@@ -497,7 +498,8 @@ describe('cable session', () => {
? { ok: true, selectionId: 'selection-1', revision: 4, rows: 2, extending: true,
excerpt: 'quoted output', text: ' first line\nsecond line' }
: { ok: false, error: 'Selection closed.' })
- const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Explain this.') })
+ const host = makeHost({ selectPassage, transcribe: vi.fn(async () => 'Explain this.'),
+ appFocus: () => ({ machineId: 'mac-local', agentId: 'a1' }) })
const { session, port } = await connect(host)
try {
port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })
@@ -1215,6 +1217,23 @@ describe('cable session', () => {
await session.stop()
})
+ it.each([{ selections: ['a2'] }, { selections: ['a1', 'a2'] }])('delivers the latest focus when $selections supersedes an unsent selection', async ({ selections }) => {
+ const { session, port, host } = await connect()
+ try {
+ port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })
+ await settle()
+ port.sent.length = 0
+ let finish!: () => void
+ vi.spyOn(host, 'listAgents').mockImplementationOnce(() => new Promise(resolve => { finish = () => resolve(AGENTS) }))
+ const first = session.followApp('mac-local', 'a2')
+ await vi.waitFor(() => expect(finish).toBeDefined())
+ const pending = selections.map(agentId => session.followApp('mac-local', agentId))
+ finish()
+ await Promise.all([first, ...pending])
+ expect(port.sent.filter(m => m.t === 'focus')).toEqual([{ t: 'focus', agentId: 'a2' }])
+ } finally { await session.stop() }
+ })
+
it('forwards a whole stroke, including the reports that carry no travel', async () => {
// The ends of a stroke are the point of the message, not padding around it: a `down` with nothing in
// it is what stops a fling still running on the far side, and an `up` with nothing in it is a finger
@@ -1371,7 +1390,7 @@ describe('cable session', () => {
})
it('routes a voice turn that names no agent', async () => {
- const host = makeHost()
+ const host = makeHost({ appFocus: () => ({ machineId: 'mac-local', agentId: 'a2' }) })
const { session, port } = await connect(host)
port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })
await settle()
@@ -1498,6 +1517,43 @@ describe('cable session', () => {
await session.stop()
})
+ it('pins ordinary dictation to the pane focused when recording begins, even if the dial is stale', async () => {
+ let focused = 'a2'
+ let finish!: (text: string) => void
+ const transcribe = vi.fn(() => new Promise<string>(resolve => { finish = resolve }))
+ const host = makeHost({
+ appFocus: () => ({ machineId: 'mac-local', agentId: focused }),
+ transcribe,
+ route: vi.fn(),
+ })
+ const { session, port } = await connect(host)
+ try {
+ port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })
+ await settle()
+ port.say({ t: 'voice.begin', agentId: 'a1', uploadId: 'focused-pane' })
+ port.pcm(Buffer.alloc(3200, 1))
+ // Focus can change during capture and during the asynchronous transcription.
+ focused = 'a1'
+ port.say({ t: 'voice.end', uploadId: 'focused-pane' })
+ await vi.waitFor(() => expect(transcribe).toHaveBeenCalledOnce())
+ finish('one two three four five')
+ await vi.waitFor(() => expect(host.sendTurn).toHaveBeenCalledOnce())
+ expect(host.sendTurn).toHaveBeenCalledWith('a2', 'one two three four five')
+ expect(host.route).not.toHaveBeenCalled()
+ expect(port.sent).toContainEqual(expect.objectContaining({ t: 'voice.transcript', agentId: 'a2' }))
+ } finally { await session.stop() }
+ })
+
+ it('restores the focused pane when a dial attaches after the window selected it', async () => {
+ const host = makeHost({ appFocus: () => ({ machineId: 'mac-local', agentId: 'a2' }) })
+ const { session, port } = await connect(host)
+ try {
+ port.say({ t: 'hello', product: 'harness', mac: 'aa:bb' })
+ await vi.waitFor(() => expect(port.sent).toContainEqual({ t: 'focus', agentId: 'a2' }))
+ expect(port.types().indexOf('agents.end')).toBeLessThan(port.types().indexOf('focus'))
+ } finally { await session.stop() }
+ })
+
it('transcribes at the rate the dial states, not a guess', async () => {
// 8 kHz described as 16 kHz is not slightly-off speech: the container lies about itself and the
// transcriber answers with an empty string.
diff --git a/cli/src/cable/cableSession.ts b/cli/src/cable/cableSession.ts
index f6efca0e3..b79cc0d5e 100644
--- a/cli/src/cable/cableSession.ts
+++ b/cli/src/cable/cableSession.ts
@@ -251,6 +251,8 @@ export interface CableHost {
selectSwarm(swarmId: string): void
appName(): string
voiceLang(): string
+ /** The window's latest selected pane, including selections made before this dial attached. */
+ appFocus?(): { machineId: string; agentId: string } | undefined
/** The active tab's agents, in tile order — and nothing else. Empty with no window or an empty tab. */
listAgents(): Promise<CableAgent[]>
/** Pane rows and workspace identity from the same desktop announcement. */
@@ -528,6 +530,8 @@ export class CableSession {
* and it is the only thing the daemon ever re-asserts.
*/
private desiredFocus = ''
+ /** A superseded list refresh can record the selection without ever writing its focus frame. */
+ private appFocusUnsent = false
/**
* App-driven machine/focus changes are one transaction. Without this queue, two quick desktop clicks
* can interleave their machine lists and let the older click focus last.
@@ -1007,6 +1011,8 @@ export class CableSession {
// screen. A repeat greeting from the same dial on the same image is a keepalive and is skipped,
// which is the whole reason the branch exists.
await this.pushAgents()
+ const focused = this.host.appFocus?.()
+ if (focused) await this.followApp(focused.machineId, focused.agentId)
}
// Offered on every greeting, but only ONCE per version per session: accepting makes the dial erase
// a flash slot before it answers, so a cadence of retries would spend erase cycles on the user's
@@ -1100,6 +1106,7 @@ export class CableSession {
// Remember where the dial IS, not just that it said so: followApp() compares against this to
// avoid echoing the dial's own move back at it.
this.desiredFocus = agentId
+ this.appFocusUnsent = false
this.host.focus(agentId)
}
return
@@ -1303,8 +1310,13 @@ export class CableSession {
: Promise.resolve({ ok: false, error: 'Choose a passage before searching.' } as const))
.catch(() => ({ ok: false, error: 'Could not open output search. Try again.' })) }
}
+ // The dial can still name the previous pane while a focus frame is in flight, or after the
+ // devices process restarted. Ordinary dictation belongs to the window's selection at capture
+ // start. Quoted text, carried text, questions and edits already have their own pinned targets.
+ const focused = str('agentId') && !formVoice && !searchVoice && !draftVoice && !questionVoice &&
+ !carryVoice && !('selectionId' in msg) ? this.host.appFocus?.() : undefined
this.voice = {
- agentId: str('agentId'),
+ agentId: focused?.agentId ?? str('agentId'),
cmd: str('cmd'),
lang: str('lang') ?? this.host.voiceLang(),
// The DIAL's rate, never this side's guess. Describing 8 kHz audio as 16 kHz does not make the
@@ -2229,7 +2241,8 @@ export class CableSession {
async focusAgent(agentId: string): Promise<void> {
this.desiredFocus = agentId
- await this.send({ t: 'focus', agentId })
+ const sent = await this.send({ t: 'focus', agentId })
+ if (sent && this.desiredFocus === agentId) this.appFocusUnsent = false
}
/**
@@ -2266,7 +2279,7 @@ export class CableSession {
// this older transaction focus after it finishes.
// Already where the window is: nothing to command. The record is right either way — this is the
// window FOLLOWING the dial, and the dial's own report wrote it.
- if (generation !== this.appFocusGeneration || agentId === this.desiredFocus) return
+ if (generation !== this.appFocusGeneration || (agentId === this.desiredFocus && !this.appFocusUnsent)) return
// Said before the frame goes out, not after: the frame itself succeeds either way.
const unknown = this.host.knows?.(agentId) === false
this.log(`cable: following the app to agent ${agentId}${unknown ? ' — NOT in this daemon\'s list, the dial has no tile for it' : ''}`)
@@ -2286,6 +2299,7 @@ export class CableSession {
// we are moving to, not the one we are leaving — otherwise the dial visibly steps onto the old
// tile on its way. Cheap when nothing changed: syncAgents returns without sending a frame.
this.desiredFocus = agentId
+ this.appFocusUnsent = true
await this.syncAgents()
if (generation !== this.appFocusGeneration) return
await this.focusAgent(agentId) // writes the record
diff --git a/cli/src/services/devices.spec.ts b/cli/src/services/devices.spec.ts
index 5695202ee..f412bc63c 100644
--- a/cli/src/services/devices.spec.ts
+++ b/cli/src/services/devices.spec.ts
@@ -282,7 +282,10 @@ describe('the window bridges, wired to the core', () => {
expect(selection.wiring.focus()).toBeUndefined()
expect(visit.wiring.focus()).toBeUndefined()
expect(form.wiring.focus()).toBeUndefined()
+ expect(seen.host.appFocus!()).toBeUndefined()
port.windowFocus({ voice: { machineId: 'm1', agentId: 'a1', connId: 'w1' }, form: { machineId: 'm1', connId: 'w1' } })
+ expect(seen.host.appFocus!()).toMatchObject({ machineId: 'm1', agentId: 'a1' })
+ expect(calls('followApp')).toEqual([['m1', 'a1']])
expect(selection.wiring.focus()).toEqual({ machineId: 'm1', agentId: 'a1', connId: 'w1' })
expect(visit.wiring.focus()).toEqual({ machineId: 'm1', agentId: 'a1', connId: 'w1' })
expect(form.wiring.focus()).toEqual({ machineId: 'm1', connId: 'w1' })
@@ -305,6 +308,9 @@ describe('the window bridges, wired to the core', () => {
{ type: 'voice_route_request', payload: { voiceId: 'v2', text: 'fix it', cmd: 'review' } },
])
expect(logs).toContain('[route] voice → the window · 6 bytes · /review')
+ port.windowFocus({ voice: null, form: null })
+ expect(seen.host.appFocus!()).toBeUndefined()
+ expect(calls('followApp')).toEqual([['m1', 'a1']])
})
it('hand each window\'s answer to its bridge, and tell them a window left', () => {
diff --git a/cli/src/services/devices.ts b/cli/src/services/devices.ts
index 4142f91ed..245ddbff0 100644
--- a/cli/src/services/devices.ts
+++ b/cli/src/services/devices.ts
@@ -133,6 +133,7 @@ export function startDevices(core: CoreApi, ports: CorePorts, deps: DevicesDeps)
sessions: () => core.agents.advertised(),
refresh: deps.refresh,
displayName: (session) => core.agents.displayName(session),
+ appFocus: () => focus.voice ?? undefined,
activityText: (agentId) => core.agents.activityText(agentId),
machineName: () => core.machine.name(),
machineId: () => core.machine.id(),
@@ -279,6 +280,9 @@ export function startDevices(core: CoreApi, ports: CorePorts, deps: DevicesDeps)
.then((result) => { if (!result.ok) log(`[cable] settings for ${id || 'no device'}: ${result.error}`) })),
windowFocus: (next) => {
focus = next
+ // State replay after a service restart carries windowFocus, without another appFocus event.
+ const selected = next.voice
+ if (selected) part.call('dial', () => cable.followApp(selected.machineId, selected.agentId))
part.call('window', () => windowSelection.focusChanged())
},
windowReply: (kind, connId, machineId, payload) => part.call('window', () =>