Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat(mobile): enhance voice input functionality and update related tests

Dev jacob081a9df44432
diff --git a/mobile/lib/phone/terminal_input_dock.dart b/mobile/lib/phone/terminal_input_dock.dart
index 858334111..f61651600 100644
--- a/mobile/lib/phone/terminal_input_dock.dart
+++ b/mobile/lib/phone/terminal_input_dock.dart
@@ -1,6 +1,7 @@
import 'dart:async';
import 'package:flutter/material.dart';
+import 'package:xterm/xterm.dart';
import 'package:harness_mobile/terminal/terminal_session.dart';
@@ -39,6 +40,26 @@ class TerminalInputDock extends StatelessWidget {
final VoidCallback? onPickImage;
final VoidCallback? onTakePhoto;
+ /// Send: what was said, or — with nothing said — what is already in the
+ /// prompt.
+ ///
+ /// Speech goes the composer's own path, not as typed bytes: the machine
+ /// injects it as one turn and owns the submit Enter — see
+ /// `TerminalSession.sendComposerText`.
+ ///
+ /// ⚠️ With no speech, Send is Enter. The prompt can hold text this panel
+ /// never saw — typed, pasted, or dictated by the keyboard's own mic before
+ /// switching back — and a Send that stayed dead over it left no way to send
+ /// that text short of raising the keyboard again for its return key.
+ void _send() {
+ if (voice.hasSpeech) {
+ unawaited(voice.submit(session.sendComposerText));
+ return;
+ }
+ session.terminal.keyInput(TerminalKey.enter);
+ voice.clear();
+ }
+
@override
Widget build(BuildContext context) => ValueListenableBuilder<bool>(
valueListenable: voice.openState,
@@ -63,10 +84,7 @@ class TerminalInputDock extends StatelessWidget {
VoiceInputPanel(
voice: voice,
canSend: session.acceptsInput,
- // The composer's own path, not typed bytes: the machine injects
- // it as one turn and owns the submit Enter — see
- // `TerminalSession.sendComposerText`.
- onSend: () => unawaited(voice.submit(session.sendComposerText)),
+ onSend: _send,
onUseKeyboard: onUseKeyboard,
onDismiss: onDismiss,
),
diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dart
index e465a5c03..a69fead6d 100644
--- a/mobile/lib/phone/terminal_page.dart
+++ b/mobile/lib/phone/terminal_page.dart
@@ -244,10 +244,9 @@ class _TerminalPageState extends State<TerminalPage>
widget.isActive && (_keyboardIsUp || _keyboardRequested);
/// A tap on the terminal while no keyboard is up or coming: voice input,
- /// already recording. Null while the keyboard is, so the tap is xterm's and
- /// the keyboard stays.
- VoidCallback? get _onInputTap =>
- _shouldFocus ? null : () => unawaited(widget.voice.open());
+ /// waiting on its mic button. Null while the keyboard is, so the tap is
+ /// xterm's and the keyboard stays.
+ VoidCallback? get _onInputTap => _shouldFocus ? null : widget.voice.open;
/// The Keyboard button in voice input. What was heard is typed into the
/// prompt rather than dropped, so the keyboard picks up where the voice left
diff --git a/mobile/lib/phone/voice_input_controller.dart b/mobile/lib/phone/voice_input_controller.dart
index 3f5efb4bc..282570b6a 100644
--- a/mobile/lib/phone/voice_input_controller.dart
+++ b/mobile/lib/phone/voice_input_controller.dart
@@ -102,11 +102,19 @@ class VoiceInputController extends ChangeNotifier {
/// A code from [voiceLanguages].
String get language => _language;
- /// Opens the panel already recording — what a tap on the terminal does.
- Future<void> open() {
- _open.value = true;
- return startListening();
- }
+ /// Whether Send has speech to send: words already heard, or a take still
+ /// being recorded that Send will finish first.
+ bool get hasSpeech =>
+ _heard.isNotEmpty || _status == VoiceInputStatus.listening;
+
+ /// Opens the panel — what a tap on the terminal does. It does NOT record.
+ ///
+ /// ⚠️ The microphone is the mic button's alone. Opening straight into a
+ /// recording turned every tap on the terminal — to put the caret back, to
+ /// send what was already typed — into a take of room noise that came back
+ /// as "didn't catch that", and a microphone that switches itself on is not
+ /// one anybody asked for.
+ void open() => _open.value = true;
/// Closes the panel, discarding anything not sent.
void close() {
diff --git a/mobile/lib/phone/voice_input_panel.dart b/mobile/lib/phone/voice_input_panel.dart
index 6b0e6c682..b63f64b77 100644
--- a/mobile/lib/phone/voice_input_panel.dart
+++ b/mobile/lib/phone/voice_input_panel.dart
@@ -15,6 +15,8 @@ import 'voice_panel_parts.dart';
/// the backend's `/api/voice/stt`, which transcribes a finished recording. Send
/// works mid-take for that reason — it ends the take and sends what it heard.
///
+/// Opening it records nothing: the mic button starts a take.
+///
/// It sits where the keyboard would, alone: [TerminalKeyBar] belongs to the
/// keyboard, so this panel carries its own `⌄`.
///
@@ -52,8 +54,6 @@ class VoiceInputPanel extends StatelessWidget {
listenable: voice,
builder: (context, _) {
final status = voice.status;
- final hasWords = voice.transcript.isNotEmpty;
- final canFinish = hasWords || status == VoiceInputStatus.listening;
return ExcludeFocus(
// Same reason as the key bar's: nothing here may take focus from the
// terminal the keyboard button is about to hand it to.
@@ -90,10 +90,14 @@ class VoiceInputPanel extends StatelessWidget {
icon: LucideIcons.arrowUp300,
label: 'Send',
emphasized: true,
+ // Live with nothing said too: Send is then Enter
+ // for what is already in the prompt — see
+ // `TerminalInputDock`. Dead only while a take is
+ // opening or its words are still on the way.
onTap:
canSend &&
- canFinish &&
!voice.isSending &&
+ status != VoiceInputStatus.starting &&
status != VoiceInputStatus.transcribing
? onSend
: null,
diff --git a/mobile/test/terminal_input_dock_test.dart b/mobile/test/terminal_input_dock_test.dart
index 7c4984594..cab2385d8 100644
--- a/mobile/test/terminal_input_dock_test.dart
+++ b/mobile/test/terminal_input_dock_test.dart
@@ -4,6 +4,7 @@ import 'package:harness_mobile/phone/terminal_input_dock.dart';
import 'package:harness_mobile/phone/terminal_key_bar.dart';
import 'package:harness_mobile/phone/voice_input_controller.dart';
import 'package:harness_mobile/phone/voice_input_panel.dart';
+import 'package:harness_mobile/terminal/terminal_binary.dart';
import 'package:harness_mobile/terminal/terminal_session.dart';
import 'voice_fakes.dart';
@@ -14,6 +15,7 @@ void main() {
late VoiceInputController voice;
late TerminalSession session;
late List<(String, Map<String, dynamic>)> frames;
+ late List<TerminalBinaryFrame> keystrokes;
late int dismissals;
late int keyboardRequests;
@@ -27,6 +29,7 @@ void main() {
preferredLocales: const [],
);
frames = [];
+ keystrokes = [];
dismissals = 0;
keyboardRequests = 0;
session =
@@ -39,7 +42,10 @@ void main() {
frames.add((type, payload));
return true;
},
- sendBinary: (_) async => true,
+ sendBinary: (frame) async {
+ keystrokes.add(frame);
+ return true;
+ },
)
..status = TerminalSessionStatus.controlling
..streamId = 's';
@@ -86,16 +92,21 @@ void main() {
tester,
) async {
await pumpDock(tester);
- await voice.open();
+ voice.open();
await tester.pump();
expect(find.byType(TerminalKeyBar), findsNothing);
expect(find.byType(VoiceInputPanel), findsOneWidget);
- expect(find.text('Listening…'), findsOneWidget);
+ // Open is not recording: the mic waits to be pressed.
+ expect(find.text('Tap the mic to talk'), findsOneWidget);
+ expect(recorder.starts, 0);
backend.replies.add('summarise the diff');
await tester.tap(find.byKey(const ValueKey('voice-mic')));
await tester.pump();
+ expect(find.text('Listening…'), findsOneWidget);
+ await tester.tap(find.byKey(const ValueKey('voice-mic')));
+ await tester.pump();
expect(find.text('summarise the diff'), findsOneWidget);
await tester.tap(find.byKey(const ValueKey('voice-send')));
@@ -112,7 +123,8 @@ void main() {
session.status = TerminalSessionStatus.takenOver;
await pumpDock(tester);
backend.replies.add('hello');
- await voice.open();
+ voice.open();
+ await voice.startListening();
await voice.stopListening();
await tester.pump();
@@ -125,7 +137,7 @@ void main() {
testWidgets('Keyboard and ⌄ hand back to the page', (tester) async {
await pumpDock(tester);
- await voice.open();
+ voice.open();
await tester.pump();
await tester.tap(find.byKey(const ValueKey('voice-keyboard')));
@@ -133,8 +145,28 @@ void main() {
expect(keyboardRequests, 1);
expect(dismissals, 1);
- // The page is what closes it; this dock only reports the taps.
- voice.close();
+ });
+
+ testWidgets('with nothing said, Send is Enter for what the prompt holds', (
+ tester,
+ ) async {
+ await pumpDock(tester);
+ voice.open();
+ await tester.pump();
+
+ await tester.tap(find.byKey(const ValueKey('voice-send')));
+ await tester.pump();
+ // The input path queues its send on a future chain the fake clock does not
+ // drain; a real turn of the event loop does.
+ await tester.runAsync(() => Future<void>.delayed(Duration.zero));
+
+ expect(frames, isEmpty, reason: 'no message: nothing was said');
+ expect(
+ keystrokes
+ .where((frame) => frame.kind == TerminalBinaryKind.input)
+ .expand((frame) => frame.bytes),
+ [13],
+ );
});
testWidgets('Send mid-sentence finishes the take and sends it', (
@@ -142,7 +174,8 @@ void main() {
) async {
backend.replies.add('ship it');
await pumpDock(tester);
- await voice.open();
+ voice.open();
+ await voice.startListening();
await tester.pump();
await tester.tap(find.byKey(const ValueKey('voice-send')));
@@ -156,7 +189,7 @@ void main() {
tester,
) async {
await pumpDock(tester);
- await voice.open();
+ voice.open();
await tester.pump();
expect(find.text('EN'), findsOneWidget);
@@ -167,13 +200,12 @@ void main() {
expect(find.text('VI'), findsOneWidget);
expect(voice.language, 'vi');
- voice.close();
- await tester.pumpAndSettle();
});
testWidgets('the mic stops a take and starts the next', (tester) async {
await pumpDock(tester);
- await voice.open();
+ voice.open();
+ await voice.startListening();
await tester.pump();
await tester.tap(find.byKey(const ValueKey('voice-mic')));
diff --git a/mobile/test/voice_input_controller_test.dart b/mobile/test/voice_input_controller_test.dart
index 8fdb9a720..d3f18380d 100644
--- a/mobile/test/voice_input_controller_test.dart
+++ b/mobile/test/voice_input_controller_test.dart
@@ -31,17 +31,39 @@ void main() {
tearDown(() => voice.dispose());
- test('opening is recording: the panel is up and the mic is on', () async {
- await voice.open();
+ test('opening puts the panel up and leaves the microphone off', () {
+ voice.open();
expect(voice.isOpen, isTrue);
+ expect(voice.status, VoiceInputStatus.idle);
+ expect(recorder.starts, 0);
+ });
+
+ test('the mic button is what records', () async {
+ voice.open();
+
+ await voice.toggleListening();
+
expect(voice.status, VoiceInputStatus.listening);
expect(recorder.recording, isTrue);
});
+ test(
+ 'Send has speech only once something was said or is being said',
+ () async {
+ expect(voice.hasSpeech, isFalse);
+
+ await voice.startListening();
+ expect(voice.hasSpeech, isTrue);
+
+ await voice.stopListening();
+ expect(voice.hasSpeech, isFalse, reason: 'the take came back empty');
+ },
+ );
+
test('stopping sends the take to the backend and keeps its words', () async {
backend.replies.add('fix the failing test');
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
@@ -54,7 +76,7 @@ void main() {
'the words wait for the backend: nothing shows mid-transcription',
() async {
backend.pending = Completer<String>();
- await voice.open();
+ await voice.startListening();
final stopping = voice.stopListening();
await Future<void>.delayed(Duration.zero);
@@ -69,7 +91,7 @@ void main() {
test('a second take adds to what the first one heard', () async {
backend.replies.addAll(['run the tests', 'then commit']);
- await voice.open();
+ await voice.startListening();
await voice.toggleListening();
await voice.toggleListening();
@@ -80,20 +102,20 @@ void main() {
test('a refused microphone says so, and the next open asks again', () async {
recorder.permitted = false;
- await voice.open();
+ await voice.startListening();
expect(voice.status, VoiceInputStatus.unavailable);
expect(voice.notice, VoiceNotice.unavailable);
recorder.permitted = true;
voice.close();
- await voice.open();
+ await voice.startListening();
expect(voice.status, VoiceInputStatus.listening);
});
test('closing while the permission prompt is up records nothing', () async {
recorder.pendingPermission = Completer<bool>();
- final opening = voice.open();
+ final opening = voice.startListening();
expect(voice.status, VoiceInputStatus.starting);
voice.close();
@@ -106,7 +128,7 @@ void main() {
test('a failed transcription says so and adds nothing', () async {
backend.fails = true;
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
@@ -115,7 +137,7 @@ void main() {
});
test('an empty transcript comes back as a notice, not a message', () async {
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
@@ -129,7 +151,7 @@ void main() {
length: const Duration(seconds: 3),
peak: 0,
);
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
@@ -139,7 +161,7 @@ void main() {
test('a take with no audio at all is said so too', () async {
recorder.captured = null;
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
@@ -149,7 +171,7 @@ void main() {
test('send hands over everything heard and empties the panel', () async {
backend.replies.add('open a PR');
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
final delivered = <String>[];
@@ -164,7 +186,7 @@ void main() {
test('send mid-sentence ends the take and sends it with the rest', () async {
backend.replies.addAll(['open a PR', 'for this branch']);
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
await voice.startListening();
@@ -180,7 +202,7 @@ void main() {
test('a take that fails mid-send sends none of the message', () async {
backend.replies.add('delete the');
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
await voice.startListening();
backend.fails = true;
@@ -195,7 +217,7 @@ void main() {
test('a send that fails keeps the words for another try', () async {
backend.replies.add('deploy');
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
await voice.submit((_) async => false);
@@ -207,7 +229,7 @@ void main() {
test('closing mid-transcription drops the words when they arrive', () async {
backend.pending = Completer<String>();
- await voice.open();
+ await voice.startListening();
final stopping = voice.stopListening();
await Future<void>.delayed(Duration.zero);
@@ -220,7 +242,7 @@ void main() {
});
test('closing while recording throws the recording away', () async {
- await voice.open();
+ await voice.startListening();
voice.close();
@@ -232,7 +254,7 @@ void main() {
'handing over to the keyboard transcribes the take in progress',
() async {
backend.replies.addAll(['rename the', 'module']);
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
await voice.startListening();
@@ -256,7 +278,7 @@ void main() {
() async {
await voice.selectLanguage('vi');
backend.replies.add('xin chào');
- await voice.open();
+ await voice.startListening();
await voice.stopListening();
expect(backend.calls.single.lang, 'vi');