The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
feat(mobile): enhance voice input functionality and update related tests
Dev jacob081a9df44432
diff --git a/mobile/lib/phone/terminal_input_dock.dart b/mobile/lib/phone/terminal_input_dock.dartindex 858334111..f61651600 100644--- a/mobile/lib/phone/terminal_input_dock.dart+++ b/mobile/lib/phone/terminal_input_dock.dart@@ -1,6 +1,7 @@import 'dart:async';import 'package:flutter/material.dart';+import 'package:xterm/xterm.dart';import 'package:harness_mobile/terminal/terminal_session.dart';@@ -39,6 +40,26 @@ class TerminalInputDock extends StatelessWidget {final VoidCallback? onPickImage;final VoidCallback? onTakePhoto;+ /// Send: what was said, or — with nothing said — what is already in the+ /// prompt.+ ///+ /// Speech goes the composer's own path, not as typed bytes: the machine+ /// injects it as one turn and owns the submit Enter — see+ /// `TerminalSession.sendComposerText`.+ ///+ /// ⚠️ With no speech, Send is Enter. The prompt can hold text this panel+ /// never saw — typed, pasted, or dictated by the keyboard's own mic before+ /// switching back — and a Send that stayed dead over it left no way to send+ /// that text short of raising the keyboard again for its return key.+ void _send() {+ if (voice.hasSpeech) {+ unawaited(voice.submit(session.sendComposerText));+ return;+ }+ session.terminal.keyInput(TerminalKey.enter);+ voice.clear();+ }+@overrideWidget build(BuildContext context) => ValueListenableBuilder<bool>(valueListenable: voice.openState,@@ -63,10 +84,7 @@ class TerminalInputDock extends StatelessWidget {VoiceInputPanel(voice: voice,canSend: session.acceptsInput,- // The composer's own path, not typed bytes: the machine injects- // it as one turn and owns the submit Enter — see- // `TerminalSession.sendComposerText`.- onSend: () => unawaited(voice.submit(session.sendComposerText)),+ onSend: _send,onUseKeyboard: onUseKeyboard,onDismiss: onDismiss,),diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dartindex e465a5c03..a69fead6d 100644--- a/mobile/lib/phone/terminal_page.dart+++ b/mobile/lib/phone/terminal_page.dart@@ -244,10 +244,9 @@ class _TerminalPageState extends State<TerminalPage>widget.isActive && (_keyboardIsUp || _keyboardRequested);/// A tap on the terminal while no keyboard is up or coming: voice input,- /// already recording. Null while the keyboard is, so the tap is xterm's and- /// the keyboard stays.- VoidCallback? get _onInputTap =>- _shouldFocus ? null : () => unawaited(widget.voice.open());+ /// waiting on its mic button. Null while the keyboard is, so the tap is+ /// xterm's and the keyboard stays.+ VoidCallback? get _onInputTap => _shouldFocus ? null : widget.voice.open;/// The Keyboard button in voice input. What was heard is typed into the/// prompt rather than dropped, so the keyboard picks up where the voice leftdiff --git a/mobile/lib/phone/voice_input_controller.dart b/mobile/lib/phone/voice_input_controller.dartindex 3f5efb4bc..282570b6a 100644--- a/mobile/lib/phone/voice_input_controller.dart+++ b/mobile/lib/phone/voice_input_controller.dart@@ -102,11 +102,19 @@ class VoiceInputController extends ChangeNotifier {/// A code from [voiceLanguages].String get language => _language;- /// Opens the panel already recording — what a tap on the terminal does.- Future<void> open() {- _open.value = true;- return startListening();- }+ /// Whether Send has speech to send: words already heard, or a take still+ /// being recorded that Send will finish first.+ bool get hasSpeech =>+ _heard.isNotEmpty || _status == VoiceInputStatus.listening;++ /// Opens the panel — what a tap on the terminal does. It does NOT record.+ ///+ /// ⚠️ The microphone is the mic button's alone. Opening straight into a+ /// recording turned every tap on the terminal — to put the caret back, to+ /// send what was already typed — into a take of room noise that came back+ /// as "didn't catch that", and a microphone that switches itself on is not+ /// one anybody asked for.+ void open() => _open.value = true;/// Closes the panel, discarding anything not sent.void close() {diff --git a/mobile/lib/phone/voice_input_panel.dart b/mobile/lib/phone/voice_input_panel.dartindex 6b0e6c682..b63f64b77 100644--- a/mobile/lib/phone/voice_input_panel.dart+++ b/mobile/lib/phone/voice_input_panel.dart@@ -15,6 +15,8 @@ import 'voice_panel_parts.dart';/// the backend's `/api/voice/stt`, which transcribes a finished recording. Send/// works mid-take for that reason — it ends the take and sends what it heard.///+/// Opening it records nothing: the mic button starts a take.+////// It sits where the keyboard would, alone: [TerminalKeyBar] belongs to the/// keyboard, so this panel carries its own `⌄`.///@@ -52,8 +54,6 @@ class VoiceInputPanel extends StatelessWidget {listenable: voice,builder: (context, _) {final status = voice.status;- final hasWords = voice.transcript.isNotEmpty;- final canFinish = hasWords || status == VoiceInputStatus.listening;return ExcludeFocus(// Same reason as the key bar's: nothing here may take focus from the// terminal the keyboard button is about to hand it to.@@ -90,10 +90,14 @@ class VoiceInputPanel extends StatelessWidget {icon: LucideIcons.arrowUp300,label: 'Send',emphasized: true,+ // Live with nothing said too: Send is then Enter+ // for what is already in the prompt — see+ // `TerminalInputDock`. Dead only while a take is+ // opening or its words are still on the way.onTap:canSend &&- canFinish &&!voice.isSending &&+ status != VoiceInputStatus.starting &&status != VoiceInputStatus.transcribing? onSend: null,diff --git a/mobile/test/terminal_input_dock_test.dart b/mobile/test/terminal_input_dock_test.dartindex 7c4984594..cab2385d8 100644--- a/mobile/test/terminal_input_dock_test.dart+++ b/mobile/test/terminal_input_dock_test.dart@@ -4,6 +4,7 @@ import 'package:harness_mobile/phone/terminal_input_dock.dart';import 'package:harness_mobile/phone/terminal_key_bar.dart';import 'package:harness_mobile/phone/voice_input_controller.dart';import 'package:harness_mobile/phone/voice_input_panel.dart';+import 'package:harness_mobile/terminal/terminal_binary.dart';import 'package:harness_mobile/terminal/terminal_session.dart';import 'voice_fakes.dart';@@ -14,6 +15,7 @@ void main() {late VoiceInputController voice;late TerminalSession session;late List<(String, Map<String, dynamic>)> frames;+ late List<TerminalBinaryFrame> keystrokes;late int dismissals;late int keyboardRequests;@@ -27,6 +29,7 @@ void main() {preferredLocales: const [],);frames = [];+ keystrokes = [];dismissals = 0;keyboardRequests = 0;session =@@ -39,7 +42,10 @@ void main() {frames.add((type, payload));return true;},- sendBinary: (_) async => true,+ sendBinary: (frame) async {+ keystrokes.add(frame);+ return true;+ },)..status = TerminalSessionStatus.controlling..streamId = 's';@@ -86,16 +92,21 @@ void main() {tester,) async {await pumpDock(tester);- await voice.open();+ voice.open();await tester.pump();expect(find.byType(TerminalKeyBar), findsNothing);expect(find.byType(VoiceInputPanel), findsOneWidget);- expect(find.text('Listening…'), findsOneWidget);+ // Open is not recording: the mic waits to be pressed.+ expect(find.text('Tap the mic to talk'), findsOneWidget);+ expect(recorder.starts, 0);backend.replies.add('summarise the diff');await tester.tap(find.byKey(const ValueKey('voice-mic')));await tester.pump();+ expect(find.text('Listening…'), findsOneWidget);+ await tester.tap(find.byKey(const ValueKey('voice-mic')));+ await tester.pump();expect(find.text('summarise the diff'), findsOneWidget);await tester.tap(find.byKey(const ValueKey('voice-send')));@@ -112,7 +123,8 @@ void main() {session.status = TerminalSessionStatus.takenOver;await pumpDock(tester);backend.replies.add('hello');- await voice.open();+ voice.open();+ await voice.startListening();await voice.stopListening();await tester.pump();@@ -125,7 +137,7 @@ void main() {testWidgets('Keyboard and ⌄ hand back to the page', (tester) async {await pumpDock(tester);- await voice.open();+ voice.open();await tester.pump();await tester.tap(find.byKey(const ValueKey('voice-keyboard')));@@ -133,8 +145,28 @@ void main() {expect(keyboardRequests, 1);expect(dismissals, 1);- // The page is what closes it; this dock only reports the taps.- voice.close();+ });++ testWidgets('with nothing said, Send is Enter for what the prompt holds', (+ tester,+ ) async {+ await pumpDock(tester);+ voice.open();+ await tester.pump();++ await tester.tap(find.byKey(const ValueKey('voice-send')));+ await tester.pump();+ // The input path queues its send on a future chain the fake clock does not+ // drain; a real turn of the event loop does.+ await tester.runAsync(() => Future<void>.delayed(Duration.zero));++ expect(frames, isEmpty, reason: 'no message: nothing was said');+ expect(+ keystrokes+ .where((frame) => frame.kind == TerminalBinaryKind.input)+ .expand((frame) => frame.bytes),+ [13],+ );});testWidgets('Send mid-sentence finishes the take and sends it', (@@ -142,7 +174,8 @@ void main() {) async {backend.replies.add('ship it');await pumpDock(tester);- await voice.open();+ voice.open();+ await voice.startListening();await tester.pump();await tester.tap(find.byKey(const ValueKey('voice-send')));@@ -156,7 +189,7 @@ void main() {tester,) async {await pumpDock(tester);- await voice.open();+ voice.open();await tester.pump();expect(find.text('EN'), findsOneWidget);@@ -167,13 +200,12 @@ void main() {expect(find.text('VI'), findsOneWidget);expect(voice.language, 'vi');- voice.close();- await tester.pumpAndSettle();});testWidgets('the mic stops a take and starts the next', (tester) async {await pumpDock(tester);- await voice.open();+ voice.open();+ await voice.startListening();await tester.pump();await tester.tap(find.byKey(const ValueKey('voice-mic')));diff --git a/mobile/test/voice_input_controller_test.dart b/mobile/test/voice_input_controller_test.dartindex 8fdb9a720..d3f18380d 100644--- a/mobile/test/voice_input_controller_test.dart+++ b/mobile/test/voice_input_controller_test.dart@@ -31,17 +31,39 @@ void main() {tearDown(() => voice.dispose());- test('opening is recording: the panel is up and the mic is on', () async {- await voice.open();+ test('opening puts the panel up and leaves the microphone off', () {+ voice.open();expect(voice.isOpen, isTrue);+ expect(voice.status, VoiceInputStatus.idle);+ expect(recorder.starts, 0);+ });++ test('the mic button is what records', () async {+ voice.open();++ await voice.toggleListening();+expect(voice.status, VoiceInputStatus.listening);expect(recorder.recording, isTrue);});+ test(+ 'Send has speech only once something was said or is being said',+ () async {+ expect(voice.hasSpeech, isFalse);++ await voice.startListening();+ expect(voice.hasSpeech, isTrue);++ await voice.stopListening();+ expect(voice.hasSpeech, isFalse, reason: 'the take came back empty');+ },+ );+test('stopping sends the take to the backend and keeps its words', () async {backend.replies.add('fix the failing test');- await voice.open();+ await voice.startListening();await voice.stopListening();@@ -54,7 +76,7 @@ void main() {'the words wait for the backend: nothing shows mid-transcription',() async {backend.pending = Completer<String>();- await voice.open();+ await voice.startListening();final stopping = voice.stopListening();await Future<void>.delayed(Duration.zero);@@ -69,7 +91,7 @@ void main() {test('a second take adds to what the first one heard', () async {backend.replies.addAll(['run the tests', 'then commit']);- await voice.open();+ await voice.startListening();await voice.toggleListening();await voice.toggleListening();@@ -80,20 +102,20 @@ void main() {test('a refused microphone says so, and the next open asks again', () async {recorder.permitted = false;- await voice.open();+ await voice.startListening();expect(voice.status, VoiceInputStatus.unavailable);expect(voice.notice, VoiceNotice.unavailable);recorder.permitted = true;voice.close();- await voice.open();+ await voice.startListening();expect(voice.status, VoiceInputStatus.listening);});test('closing while the permission prompt is up records nothing', () async {recorder.pendingPermission = Completer<bool>();- final opening = voice.open();+ final opening = voice.startListening();expect(voice.status, VoiceInputStatus.starting);voice.close();@@ -106,7 +128,7 @@ void main() {test('a failed transcription says so and adds nothing', () async {backend.fails = true;- await voice.open();+ await voice.startListening();await voice.stopListening();@@ -115,7 +137,7 @@ void main() {});test('an empty transcript comes back as a notice, not a message', () async {- await voice.open();+ await voice.startListening();await voice.stopListening();@@ -129,7 +151,7 @@ void main() {length: const Duration(seconds: 3),peak: 0,);- await voice.open();+ await voice.startListening();await voice.stopListening();@@ -139,7 +161,7 @@ void main() {test('a take with no audio at all is said so too', () async {recorder.captured = null;- await voice.open();+ await voice.startListening();await voice.stopListening();@@ -149,7 +171,7 @@ void main() {test('send hands over everything heard and empties the panel', () async {backend.replies.add('open a PR');- await voice.open();+ await voice.startListening();await voice.stopListening();final delivered = <String>[];@@ -164,7 +186,7 @@ void main() {test('send mid-sentence ends the take and sends it with the rest', () async {backend.replies.addAll(['open a PR', 'for this branch']);- await voice.open();+ await voice.startListening();await voice.stopListening();await voice.startListening();@@ -180,7 +202,7 @@ void main() {test('a take that fails mid-send sends none of the message', () async {backend.replies.add('delete the');- await voice.open();+ await voice.startListening();await voice.stopListening();await voice.startListening();backend.fails = true;@@ -195,7 +217,7 @@ void main() {test('a send that fails keeps the words for another try', () async {backend.replies.add('deploy');- await voice.open();+ await voice.startListening();await voice.stopListening();await voice.submit((_) async => false);@@ -207,7 +229,7 @@ void main() {test('closing mid-transcription drops the words when they arrive', () async {backend.pending = Completer<String>();- await voice.open();+ await voice.startListening();final stopping = voice.stopListening();await Future<void>.delayed(Duration.zero);@@ -220,7 +242,7 @@ void main() {});test('closing while recording throws the recording away', () async {- await voice.open();+ await voice.startListening();voice.close();@@ -232,7 +254,7 @@ void main() {'handing over to the keyboard transcribes the take in progress',() async {backend.replies.addAll(['rename the', 'module']);- await voice.open();+ await voice.startListening();await voice.stopListening();await voice.startListening();@@ -256,7 +278,7 @@ void main() {() async {await voice.selectLanguage('vi');backend.replies.add('xin chào');- await voice.open();+ await voice.startListening();await voice.stopListening();expect(backend.calls.single.lang, 'vi');