Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

feat: implement voice input panel and related components

Dev jacobd630f981cfd6
diff --git a/mobile/android/app/src/main/AndroidManifest.xml b/mobile/android/app/src/main/AndroidManifest.xml
index a9f0ae0b3..197f61d8b 100644
--- a/mobile/android/app/src/main/AndroidManifest.xml
+++ b/mobile/android/app/src/main/AndroidManifest.xml
@@ -18,6 +18,12 @@
cannot run without: a tablet or emulator with no camera must still
install and show every other screen. -->
<uses-feature android:name="android.hardware.camera" android:required="false" />
+ <!-- Voice input, the terminal's default way to type (speech_to_text). No
+ BLUETOOTH_CONNECT beside it: the recognizer is initialised with
+ `androidNoBluetooth`, so a headset is never asked for and nobody is
+ shown a "nearby devices" prompt for talking into the phone. -->
+ <uses-permission android:name="android.permission.RECORD_AUDIO" />
+ <uses-feature android:name="android.hardware.microphone" android:required="false" />
<application
android:label="harness"
android:name="${applicationName}"
@@ -60,5 +66,10 @@
<action android:name="android.intent.action.PROCESS_TEXT"/>
<data android:mimeType="text/plain"/>
</intent>
+ <!-- Android 11+ hides the speech recognizer service from an app that
+ does not declare it, and voice input then reads as unavailable. -->
+ <intent>
+ <action android:name="android.speech.RecognitionService"/>
+ </intent>
</queries>
</manifest>
diff --git a/mobile/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/mobile/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
new file mode 100644
index 000000000..454fee525
--- /dev/null
+++ b/mobile/ios/Runner.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -0,0 +1,14 @@
+{
+ "pins" : [
+ {
+ "identity" : "cwlcatchexception",
+ "kind" : "remoteSourceControl",
+ "location" : "https://github.com/mattgallagher/CwlCatchException.git",
+ "state" : {
+ "revision" : "07b2ba21d361c223e25e3c1e924288742923f08c",
+ "version" : "2.2.1"
+ }
+ }
+ ],
+ "version" : 2
+}
diff --git a/mobile/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved b/mobile/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved
new file mode 100644
index 000000000..454fee525
--- /dev/null
+++ b/mobile/ios/Runner.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -0,0 +1,14 @@
+{
+ "pins" : [
+ {
+ "identity" : "cwlcatchexception",
+ "kind" : "remoteSourceControl",
+ "location" : "https://github.com/mattgallagher/CwlCatchException.git",
+ "state" : {
+ "revision" : "07b2ba21d361c223e25e3c1e924288742923f08c",
+ "version" : "2.2.1"
+ }
+ }
+ ],
+ "version" : 2
+}
diff --git a/mobile/ios/Runner/Info.plist b/mobile/ios/Runner/Info.plist
index 60b32dda6..fddef7cce 100644
--- a/mobile/ios/Runner/Info.plist
+++ b/mobile/ios/Runner/Info.plist
@@ -43,6 +43,13 @@
be shown and is deliberately absent. -->
<key>NSCameraUsageDescription</key>
<string>Take a photo to send to an agent in a terminal.</string>
+ <!-- Voice input, what tapping a terminal opens (speech_to_text). Both are
+ asked for on the first tap, not at launch: the microphone to hear it,
+ Speech Recognition to turn it into the text sent to the agent. -->
+ <key>NSMicrophoneUsageDescription</key>
+ <string>Talk to an agent in a terminal instead of typing.</string>
+ <key>NSSpeechRecognitionUsageDescription</key>
+ <string>Turn what you say into the message sent to an agent.</string>
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
diff --git a/mobile/lib/phone/agent_swipe.dart b/mobile/lib/phone/agent_swipe.dart
index e950cd2a6..2fde11171 100644
--- a/mobile/lib/phone/agent_swipe.dart
+++ b/mobile/lib/phone/agent_swipe.dart
@@ -5,6 +5,7 @@ import 'package:flutter/material.dart';
import 'package:harness_mobile/state/app_state.dart';
import 'agent_index.dart';
import 'terminal_page.dart';
+import 'voice_input_controller.dart';
/// The agents a terminal page can swipe between, in the order the list drew them.
///
@@ -122,6 +123,11 @@ class _AgentSwipeHostState extends State<AgentSwipeHost> {
/// and the panes to close are the ones actually opened, not the ones a fresh list would name.
final Set<({String machineId, String agentId})> _attached = {};
+ /// Voice input for every page of this pager: open or closed, and what has been heard so far,
+ /// survive a swipe the way a keyboard that is up does. Disposed with the pager, which is what
+ /// turns the microphone off on the way out.
+ final VoiceInputController _voice = VoiceInputController();
+
@override
void initState() {
super.initState();
@@ -149,6 +155,7 @@ class _AgentSwipeHostState extends State<AgentSwipeHost> {
@override
void dispose() {
+ _voice.dispose();
_controller?.dispose();
_detachAll();
super.dispose();
@@ -183,6 +190,7 @@ class _AgentSwipeHostState extends State<AgentSwipeHost> {
notifier: widget.notifier,
machineId: widget.machineId,
agentId: widget.agentId,
+ voice: _voice,
isActive: true,
);
}
@@ -213,6 +221,7 @@ class _AgentSwipeHostState extends State<AgentSwipeHost> {
notifier: widget.notifier,
machineId: entry.machineId,
agentId: entry.agent.id,
+ voice: _voice,
// Exactly one mounted page, by page number — see [_page].
isActive: i == _page,
);
diff --git a/mobile/lib/phone/phone_sheet.dart b/mobile/lib/phone/phone_sheet.dart
index be605148d..d3f79bbf5 100644
--- a/mobile/lib/phone/phone_sheet.dart
+++ b/mobile/lib/phone/phone_sheet.dart
@@ -48,16 +48,26 @@ Future<void> showPhoneSheet(
style: TextStyle(color: AppPalette.textSecondary, fontSize: 13),
),
),
- for (final action in actions)
- _SheetRow(
- action: action,
- // Close first, then act: an action that opens a dialog or pushes a page must not do it
- // underneath a sheet that is still animating out.
- onTap: () {
- Navigator.of(sheetContext).pop();
- action.onTap();
- },
+ // Scrolls once the rows outgrow the sheet — a list of languages does — and is laid out
+ // exactly like a plain column until then.
+ Flexible(
+ child: ListView(
+ shrinkWrap: true,
+ padding: EdgeInsets.zero,
+ children: [
+ for (final action in actions)
+ _SheetRow(
+ action: action,
+ // Close first, then act: an action that opens a dialog or pushes a page must not
+ // do it underneath a sheet that is still animating out.
+ onTap: () {
+ Navigator.of(sheetContext).pop();
+ action.onTap();
+ },
+ ),
+ ],
),
+ ),
const SizedBox(height: 8),
],
),
diff --git a/mobile/lib/phone/terminal_input_dock.dart b/mobile/lib/phone/terminal_input_dock.dart
new file mode 100644
index 000000000..858334111
--- /dev/null
+++ b/mobile/lib/phone/terminal_input_dock.dart
@@ -0,0 +1,78 @@
+import 'dart:async';
+
+import 'package:flutter/material.dart';
+
+import 'package:harness_mobile/terminal/terminal_session.dart';
+
+import 'terminal_key_bar.dart';
+import 'voice_input_controller.dart';
+import 'voice_input_panel.dart';
+
+/// The bottom of a terminal page: [TerminalKeyBar] over the keyboard, or
+/// [VoiceInputPanel] in its place.
+///
+/// Voice is what a tap on the terminal opens; the keyboard is what its
+/// Keyboard button hands over to. One at a time, and the key bar goes with the
+/// keyboard: `esc`, the arrows and the digits are keys to TYPE with, and while
+/// someone is talking they are two rows of terminal the panel does not need.
+class TerminalInputDock extends StatelessWidget {
+ const TerminalInputDock({
+ super.key,
+ required this.session,
+ required this.voice,
+ required this.keyboardUp,
+ required this.onDismiss,
+ required this.onUseKeyboard,
+ this.onPickImage,
+ this.onTakePhoto,
+ });
+
+ final TerminalSession session;
+ final VoiceInputController voice;
+
+ /// The software keyboard is up, or has been asked for and is on its way.
+ final bool keyboardUp;
+
+ /// `⌄`, on the key bar or on the panel: puts away whichever input is up.
+ final VoidCallback onDismiss;
+ final VoidCallback onUseKeyboard;
+ final VoidCallback? onPickImage;
+ final VoidCallback? onTakePhoto;
+
+ @override
+ Widget build(BuildContext context) => ValueListenableBuilder<bool>(
+ valueListenable: voice.openState,
+ builder: (context, voiceOpen, _) {
+ if (!keyboardUp && !voiceOpen) return const SizedBox.shrink();
+ return ListenableBuilder(
+ listenable: session,
+ builder: (context, _) => Column(
+ mainAxisSize: MainAxisSize.min,
+ children: [
+ if (keyboardUp)
+ TerminalKeyBar(
+ terminal: session.terminal,
+ enabled: session.acceptsInput,
+ controlArmed: session.controlArmed,
+ onControlToggle: session.armControl,
+ onDismissKeyboard: onDismiss,
+ onPickImage: onPickImage,
+ onTakePhoto: onTakePhoto,
+ )
+ else
+ VoiceInputPanel(
+ voice: voice,
+ canSend: session.acceptsInput,
+ // The composer's own path, not typed bytes: the machine injects
+ // it as one turn and owns the submit Enter — see
+ // `TerminalSession.sendComposerText`.
+ onSend: () => unawaited(voice.submit(session.sendComposerText)),
+ onUseKeyboard: onUseKeyboard,
+ onDismiss: onDismiss,
+ ),
+ ],
+ ),
+ );
+ },
+ );
+}
diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dart
index 3531a52f9..066ed73ca 100644
--- a/mobile/lib/phone/terminal_page.dart
+++ b/mobile/lib/phone/terminal_page.dart
@@ -21,7 +21,8 @@ import 'phone_header.dart';
import 'phone_sheet.dart';
import 'phone_status.dart';
import 'status_pill.dart';
-import 'terminal_key_bar.dart';
+import 'terminal_input_dock.dart';
+import 'voice_input_controller.dart';
/// One agent's terminal, filling the phone. The header says whose it is and whether it is live;
/// everything below it is the same [TerminalPanel] a desktop tile draws, minus that tile's own
@@ -35,6 +36,7 @@ class TerminalPage extends StatefulWidget {
required this.notifier,
required this.machineId,
required this.agentId,
+ required this.voice,
this.isActive = true,
});
@@ -42,6 +44,10 @@ class TerminalPage extends StatefulWidget {
final String machineId;
final String agentId;
+ /// Voice input, shared by every page of the pager this page is in — see
+ /// [VoiceInputController] for why it is not this page's own.
+ final VoiceInputController voice;
+
/// Whether this is the page being LOOKED AT, rather than one parked beside it in the pager.
///
/// ⚠️ **Load-bearing for correctness, not just for tidiness.** [TerminalPanel] claims the keyboard
@@ -61,42 +67,24 @@ class TerminalPage extends StatefulWidget {
/// Whether the software keyboard is up, as one fact rather than as a flag each
/// page keeps for itself.
///
-/// ⚠️ Per-page state cannot answer the question the pager asks. A page keeps
-/// [_TerminalPageState._claimSpent] to stop itself re-summoning a keyboard the
-/// person dismissed — but that flag is only ever armed by a keyboard this page
-/// WATCHED rise. Swiping A → B → C and hiding the keyboard at C leaves B
-/// having mounted while it was already down: its claim was never spent, so on
-/// the way back it looked exactly like a page arriving fresh and pulled the
-/// keyboard up again. Whether the keyboard is up is a property of the SCREEN,
-/// not of any one page, so it is kept once here.
+/// ⚠️ Per-page state cannot answer the question the pager asks. Swiping to the
+/// next agent while typing must HOLD the keyboard — the outgoing page releases
+/// focus, and unless the incoming one takes it in the same frame the platform
+/// closes the keyboard — and a page built while the keyboard was down, then
+/// swiped to after it rose, never watched it rise. Whether the keyboard is up is
+/// a property of the SCREEN, not of any one page, so it is kept once here.
///
/// Written by every mounted page's `didChangeMetrics` — they all see the same
/// inset, so they all write the same value.
bool _keyboardIsUp = false;
-/// Whether the keyboard has been PUT AWAY while the pager was open, as opposed
-/// to never having been raised.
-///
-/// ⚠️ The pager's second half, and per-page state cannot hold it either. A page
-/// raises the keyboard on arrival because that is what opening an agent should
-/// do — but once somebody has dismissed it, every page reached by swiping after
-/// that must leave it down, including pages that have never been on screen and
-/// so have nothing of their own to remember. It is the PERSON's choice, made
-/// once, and it outlives any one page.
-///
-/// Set when the inset falls to zero with the pager open; cleared when it rises
-/// again, so tapping a pane to type re-arms the ordinary behaviour.
-bool _keyboardDismissed = false;
-
/// Forgets what the keyboard did during the last run of terminal pages.
///
-/// Called when a pager opens. The dismissal above is deliberately sticky ACROSS
-/// pages so a swipe cannot undo it, which makes it sticky across whole visits
-/// too unless something clears it — and "I put the keyboard away on that agent
-/// an hour ago" should not decide what opening a different agent does now.
+/// Called when a pager opens. A pager popped with the keyboard up is disposed
+/// before the inset falls, so no page is left to see it fall — and the next
+/// pager would otherwise open believing the keyboard is up, and summon it.
void resetKeyboardSession() {
_keyboardIsUp = false;
- _keyboardDismissed = false;
}
class _TerminalPageState extends State<TerminalPage>
@@ -113,44 +101,18 @@ class _TerminalPageState extends State<TerminalPage>
/// TerminalPage still sitting in the Agents tab's stack.
bool _hadPane = false;
- /// Whether this page has already used its one chance to summon the keyboard.
- ///
- /// It starts `false`, so the terminal is focused on arrival and the keyboard
- /// rises by itself. It flips the moment the keyboard IS up, and NEVER goes
- /// back: from then on `TerminalPanel` is passed `focused: false` and stops
- /// claiming, for the life of the page. Bringing the keyboard back is the
- /// terminal's own job — xterm's tap handler calls `requestKeyboard()` without
- /// consulting this flag, so nothing here needs to re-arm.
+ /// Whether the Keyboard button in voice input has asked for the keyboard,
+ /// and it has not risen yet.
///
- /// ⚠️ Without this, Back could not put the keyboard away at all:
- /// `TerminalPanel._claimFocus` calls `TerminalView.requestKeyboard()`, which
- /// RE-TAKES focus when it finds none, so the keyboard returned a frame after
- /// the system dismissed it. Measured on a Pixel 8 Pro — `onRequestShow`
- /// ELEVEN times against a single `onRequestHide`.
- ///
- /// ⚠️ It flips on the keyboard APPEARING, not on it going away, and that is
- /// the difference between Back working on the first press and on the second.
- /// `_claimFocus` runs from a post-frame callback and BEATS `didChangeMetrics`
- /// to the news that the keyboard is gone:
- ///
- /// onDispatched keyboard hidden
- /// CLAIM focused=true claim ran first — re-took focus
- /// onRequestShow keyboard on its way back
- /// KB raw=0.0 our metrics callback, one beat too late
- ///
- /// Arming on the way up removes the race: by the time any Back arrives,
- /// claiming has been off for as long as the keyboard has been visible.
- bool _claimSpent = false;
-
- /// Whether this page has already had a turn on screen and been swiped away.
+ /// What makes [TerminalPanel] claim focus at all: arriving on a page raises
+ /// nothing, and a tap on the terminal opens voice input, so this is the one
+ /// way the keyboard is SUMMONED. Spent the moment the keyboard is up — from
+ /// then on [_keyboardIsUp] holds it — so Back or `⌄` can put it away without
+ /// a claim fetching it straight back.
///
- /// ⚠️ What separates "arriving" from "coming back", which [_claimSpent]
- /// cannot. A page the pager built while the keyboard was down never spent its
- /// claim, so on its second turn it looked exactly like one opened fresh — and
- /// summoned the keyboard again after the person had dismissed it two agents
- /// away. A page raises the keyboard on its FIRST turn only; after that it
- /// holds whatever is there and summons nothing.
- bool _hasHadATurn = false;
+ /// It stays set when no inset ever arrives, which is what a hardware keyboard
+ /// looks like: the terminal keeps its focus, and the key bar stays for `esc`.
+ bool _keyboardRequested = false;
/// Whether the software keyboard is up, and with it [TerminalKeyBar].
///
@@ -198,10 +160,9 @@ class _TerminalPageState extends State<TerminalPage>
void didUpdateWidget(TerminalPage oldWidget) {
super.didUpdateWidget(oldWidget);
if (oldWidget.isActive && !widget.isActive) {
- // Marked on the way OUT, not on the way in: the flag means "has had its
- // turn", and a page still on its first one must keep the summon it has
- // not used yet.
- _hasHadATurn = true;
+ // A keyboard asked for and still on its way belongs to the page that
+ // asked; a parked page must not claim it when it comes.
+ _keyboardRequested = false;
_cancelSettle();
if (_keyboardSettling) setState(() => _keyboardSettling = false);
}
@@ -229,19 +190,16 @@ class _TerminalPageState extends State<TerminalPage>
super.didChangeMetrics();
if (!mounted) return;
final inset = View.of(context).viewInsets.bottom;
- // The FIRST frame of the keyboard rising is enough — it need not finish.
- // Spending the claim this early is the point: it is off long before any
- // Back press can arrive.
final up = inset > 0;
// Every mounted page writes it, and they all see the same inset — so a page
// that was parked while the keyboard came and went still reads the truth.
- if (up != _keyboardIsUp) {
- _keyboardIsUp = up;
- // Falling to zero is a dismissal; rising again is somebody asking for it
- // back, which restores what a freshly opened agent does.
- _keyboardDismissed = !up;
- }
- final claim = _claimSpent || up;
+ _keyboardIsUp = up;
+ // The FIRST frame of the keyboard rising spends the request — it need not
+ // finish. Spending it this early is the point: it is off long before any
+ // Back press can arrive.
+ final requested = _keyboardRequested && !up;
+ // One input at a time. Whatever raised the keyboard, voice input yields.
+ if (up && widget.voice.isOpen) widget.voice.close();
// Every tick that MOVES the inset is the animation still running; the run
// ends when one window passes without another move. Gated on a real change
@@ -264,44 +222,50 @@ class _TerminalPageState extends State<TerminalPage>
final settling = _settleTimer != null;
if (up == _keyboardUp &&
- claim == _claimSpent &&
+ requested == _keyboardRequested &&
settling == _keyboardSettling) {
return;
}
setState(() {
_keyboardUp = up;
- _claimSpent = claim;
+ _keyboardRequested = requested;
_keyboardSettling = settling;
});
}
- /// Whether [TerminalPanel] should hold the input connection.
- ///
- /// Two different questions, and [_claimSpent] alone can only answer one:
+ /// Whether [TerminalPanel] should hold the input connection: while the
+ /// keyboard is up, and while one is on its way — see [_keyboardRequested].
///
- /// - **Raise a keyboard that is down.** One chance per page, spent the
- /// moment the keyboard appears, so Back can put it away without the claim
- /// fetching it straight back — see [_claimSpent].
- /// - **Hold a keyboard that is already up.** What swiping between agents
- /// needs: the outgoing page releases focus, and unless the incoming one
- /// takes it in the same frame the platform closes the keyboard.
- ///
- /// ⚠️ The second case reads [_keyboardIsUp], the one screen-wide fact, rather
- /// than anything this page remembers. Per-page flags get the pager wrong in
- /// both directions: [_claimSpent] is armed by a parked neighbour merely
- /// WATCHING the keyboard rise, and it is never armed at all on a page built
- /// while the keyboard was down — so a page coming back for its second turn
- /// was indistinguishable from one opened fresh, and summoned a keyboard the
- /// person had dismissed two agents away. [_hasHadATurn] closes that: the
- /// summon belongs to a page's first turn only.
+ /// ⚠️ Holding reads [_keyboardIsUp], the one screen-wide fact, rather than
+ /// anything this page remembers — see that flag for why swiping needs it.
bool get _shouldFocus =>
- widget.isActive &&
- (_keyboardIsUp || (!_claimSpent && !_hasHadATurn && !_keyboardDismissed));
+ widget.isActive && (_keyboardIsUp || _keyboardRequested);
+
+ /// A tap on the terminal while no keyboard is up or coming: voice input,
+ /// already listening. Null while the keyboard is, so the tap is xterm's and
+ /// the keyboard stays.
+ VoidCallback? get _onInputTap =>
+ _shouldFocus ? null : () => unawaited(widget.voice.open());
+
+ /// The Keyboard button in voice input. What was heard is typed into the
+ /// prompt rather than dropped, so the keyboard picks up where the voice left
+ /// off — to correct a word, or to finish the sentence.
+ void _useKeyboard(TerminalSession session) {
+ final heard = widget.voice.transcript;
+ widget.voice.close();
+ if (heard.isNotEmpty && session.acceptsInput) {
+ session.terminal.textInput(heard);
+ }
+ setState(() => _keyboardRequested = true);
+ }
- /// Puts the keyboard away without leaving the page — the `⌄` key on
- /// [TerminalKeyBar]. Dropping focus is what closes the input connection;
- /// xterm reopens it on the next tap in the pane.
- void _dismissKeyboard() => FocusManager.instance.primaryFocus?.unfocus();
+ /// Puts away whichever input is up without leaving the page — the `⌄` key on
+ /// [TerminalKeyBar]. Dropping focus is what closes the input connection.
+ void _dismissInput() {
+ widget.voice.close();
+ FocusManager.instance.primaryFocus?.unfocus();
+ if (_keyboardRequested) setState(() => _keyboardRequested = false);
+ }
/// Guards against a second picker while one is already up.
///
@@ -463,17 +427,6 @@ class _TerminalPageState extends State<TerminalPage>
Expanded(
child: pane == null || session == null
? const _Attaching()
- // ⚠️ Nothing re-arms [_claimSpent] on tap, and that
- // is deliberate. A `Listener` doing so was written
- // and removed: xterm's own `_onTapDown` already
- // calls `requestKeyboard()`, so the tap opened the
- // keyboard and THEN the re-armed claim asked for it
- // a second time — Android answers a show arriving
- // mid-animation by cancelling and restarting it.
- // Measured: two `onRequestShow` and two
- // `onCancelled at PHASE_CLIENT_APPLY_ANIMATION` per
- // tap. The claim exists only to raise the keyboard
- // on arrival; after that the terminal handles it.
: TerminalPanel(
key: ValueKey(pane.id),
notifier: widget.notifier,
@@ -493,47 +446,51 @@ class _TerminalPageState extends State<TerminalPage>
// must NOT release focus — the animation being
// waited on is the one that focus started.
settling: _keyboardSettling,
+ // ⚠️ The tap is taken in the panel, not by a
+ // `Listener` over it. xterm's own `_onTapDown`
+ // calls `requestKeyboard()`, so anything that
+ // merely ALSO reacted to the tap would get the
+ // keyboard rising under voice input — and a
+ // re-armed claim on top of it was measured
+ // asking Android twice per tap, which answers a
+ // show mid-animation by cancelling and
+ // restarting it.
+ onInputTap: _onInputTap,
showHeader: false,
// No composer, and so no grip above it: the
// page hands the pane its full height and the
// software keyboard drives the terminal
- // directly — `TerminalPanel` autofocuses the
- // view precisely when no box is covering it.
- // What goes with the box is the batched send,
- // and the Esc/Tab/Ctrl an on-screen keyboard
- // never had anyway.
+ // directly. Voice input's Send is what kept
+ // the composer's batched turn.
),
),
// The bottom of this page IS just above the keyboard:
// `PhoneShell`'s Scaffold has already resized for it —
// the same resize that empties this page's MediaQuery
// insets (see [didChangeMetrics]).
- if (_keyboardUp && session != null)
- ListenableBuilder(
- listenable: session,
- builder: (context, _) => TerminalKeyBar(
- terminal: session.terminal,
- enabled: session.acceptsInput,
- controlArmed: session.controlArmed,
- onControlToggle: session.armControl,
- onDismissKeyboard: _dismissKeyboard,
- // Only where the far side can actually take one: an
- // older CLI never advertises the binary kind, so the
- // upload would go nowhere silently. Null leaves the
- // buttons undrawn rather than drawn dead.
- onPickImage:
- machine?.terminalImagePasteAvailable == true
- ? () => unawaited(
- _sendImage(session, ImageSource.gallery),
- )
- : null,
- onTakePhoto:
- machine?.terminalImagePasteAvailable == true
- ? () => unawaited(
- _sendImage(session, ImageSource.camera),
- )
- : null,
- ),
+ if (session != null)
+ TerminalInputDock(
+ session: session,
+ voice: widget.voice,
+ keyboardUp: _keyboardUp || _keyboardRequested,
+ onDismiss: _dismissInput,
+ onUseKeyboard: () => _useKeyboard(session),
+ // Only where the far side can actually take one: an
+ // older CLI never advertises the binary kind, so the
+ // upload would go nowhere silently. Null leaves the
+ // buttons undrawn rather than drawn dead.
+ onPickImage:
+ machine?.terminalImagePasteAvailable == true
+ ? () => unawaited(
+ _sendImage(session, ImageSource.gallery),
+ )
+ : null,
+ onTakePhoto:
+ machine?.terminalImagePasteAvailable == true
+ ? () => unawaited(
+ _sendImage(session, ImageSource.camera),
+ )
+ : null,
),
],
),
diff --git a/mobile/lib/phone/voice_input_controller.dart b/mobile/lib/phone/voice_input_controller.dart
new file mode 100644
index 000000000..b16e70c4b
--- /dev/null
+++ b/mobile/lib/phone/voice_input_controller.dart
@@ -0,0 +1,250 @@
+import 'dart:async';
+
+import 'package:flutter/foundation.dart';
+
+import 'package:harness_mobile/core/harness_file_store.dart';
+import 'package:harness_mobile/core/local_key_value_store.dart';
+
+import 'voice_notice.dart';
+import 'voice_speech_engine.dart';
+
+enum VoiceInputStatus {
+ /// Not hearing anything. What was heard so far stays.
+ idle,
+
+ /// Waiting on a permission prompt, or on the recognizer to begin.
+ starting,
+ listening,
+
+ /// Permission refused, or no recognizer on this phone: the keyboard is the
+ /// only way in until that changes.
+ unavailable,
+}
+
+/// Voice input for one pager of terminal pages: whether its panel is open, what
+/// has been heard, and the language it listens in.
+///
+/// Owned by `AgentSwipeHost`, not by a page, for the reason the keyboard is one
+/// screen-wide fact in `terminal_page.dart`: swiping to the next agent must not
+/// close what is open, nor drop what was said. What is SENT goes to whichever
+/// page is active when Send is pressed.
+class VoiceInputController extends ChangeNotifier {
+ VoiceInputController({SpeechEngine? engine, LocalKeyValueStore? storage})
+ : _engine = engine ?? PlatformSpeechEngine(),
+ _storage = storage ?? HarnessFileStore.shared;
+
+ static const _localeKey = 'voice_input_locale';
+
+ final SpeechEngine _engine;
+ final LocalKeyValueStore _storage;
+
+ final ValueNotifier<bool> _open = ValueNotifier(false);
+ bool _isSending = false;
+ bool _prepared = false;
+ bool _disposed = false;
+ VoiceInputStatus _status = VoiceInputStatus.idle;
+ String? _notice;
+ VoiceLocales _locales = (all: const [], systemId: null);
+ String? _chosenLocaleId;
+
+ /// Words from finished listens, and the running guess of the current one.
+ String _heard = '';
+ String _hearing = '';
+
+ /// Bumped by everything that abandons a listen, so a result or error still
+ /// in flight from it lands nowhere — above all after Send, where a late final
+ /// result would put the words just sent back in the panel.
+ int _listen = 0;
+
+ /// Whether the panel is up, as its own listenable: the terminal page watches
+ /// THIS, not the controller, so words arriving several times a second repaint
+ /// the panel and never rebuild the terminal above it.
+ ValueListenable<bool> get openState => _open;
+ bool get isOpen => _open.value;
+ bool get isSending => _isSending;
+ VoiceInputStatus get status => _status;
+ bool get isListening =>
+ _status == VoiceInputStatus.starting ||
+ _status == VoiceInputStatus.listening;
+ String get transcript => _joinWords(_heard, _hearing);
+ String? get notice => _notice;
+
+ /// Empty until the first listen has been allowed.
+ List<VoiceLocale> get locales => _locales.all;
+ String? get localeId => _chosenLocaleId ?? _locales.systemId;
+
+ /// Opens the panel already listening — what a tap on the terminal does.
+ Future<void> open() {
+ _open.value = true;
+ return startListening();
+ }
+
+ /// Closes the panel, discarding anything not sent.
+ void close() {
+ _open.value = false;
+ clear();
+ }
+
+ void clear() {
+ _abandonListen();
+ _heard = '';
+ _hearing = '';
+ _setStatus(VoiceInputStatus.idle);
+ }
+
+ Future<void> toggleListening() =>
+ isListening ? stopListening() : startListening();
+
+ Future<void> startListening() async {
+ if (isListening || _isSending || _disposed) return;
+ _keepHearing();
+ final listen = ++_listen;
+ _setStatus(VoiceInputStatus.starting);
+ if (!await _prepare()) {
+ if (listen != _listen) return;
+ _setStatus(VoiceInputStatus.unavailable, notice: VoiceNotice.unavailable);
+ return;
+ }
+ // Closed, cleared or stopped while the permission prompt was up.
+ if (listen != _listen) return;
+ try {
+ await _engine.listen(
+ localeId: _chosenLocaleId,
+ onWords: (words, isFinal) => _onWords(listen, words, isFinal),
+ onStopped: () => _onStopped(listen),
+ onError: (code) => _onError(listen, code),
+ );
+ } on Exception {
+ if (listen != _listen) return;
+ _setStatus(VoiceInputStatus.idle, notice: VoiceNotice.couldNotStart);
+ return;
+ }
+ if (listen == _listen && _status == VoiceInputStatus.starting) {
+ _setStatus(VoiceInputStatus.listening);
+ }
+ }
+
+ /// Ends the listen but keeps its words: the final result is still to come.
+ Future<void> stopListening() async {
+ if (_status == VoiceInputStatus.starting) {
+ _abandonListen();
+ } else if (_status == VoiceInputStatus.listening) {
+ unawaited(_engine.stop());
+ }
+ _setStatus(VoiceInputStatus.idle);
+ }
+
+ /// Hands the transcript to [deliver], and empties the panel once it is sent.
+ /// Kept when delivery fails, so nothing said has to be said twice.
+ Future<void> submit(Future<bool> Function(String text) deliver) async {
+ final text = transcript.trim();
+ if (text.isEmpty || _isSending) return;
+ _abandonListen();
+ _keepHearing();
+ _isSending = true;
+ _setStatus(VoiceInputStatus.idle);
+ var sent = false;
+ try {
+ sent = await deliver(text);
+ } on Exception {
+ // Reported below exactly like a refusal: the words stay, Send stays live.
+ }
+ _isSending = false;
+ if (sent) _heard = '';
+ _setStatus(
+ VoiceInputStatus.idle,
+ notice: sent ? null : VoiceNotice.notSent,
+ );
+ }
+
+ /// Listens in [id] from now on, on this phone and in later pagers too. A
+ /// listen already running restarts in it; the words it heard stay.
+ Future<void> selectLocale(String id) async {
+ final wasListening = isListening;
+ _abandonListen();
+ _chosenLocaleId = id;
+ _setStatus(VoiceInputStatus.idle);
+ if (wasListening) unawaited(startListening());
+ try {
+ await _storage.write(_localeKey, id);
+ } on Exception {
+ // Still used for this pager; only remembering it failed.
+ }
+ }
+
+ @override
+ void dispose() {
+ _abandonListen();
+ _disposed = true;
+ _open.dispose();
+ super.dispose();
+ }
+
+ Future<bool> _prepare() async {
+ if (_prepared) return true;
+ if (!await _engine.prepare()) return false;
+ _prepared = true;
+ try {
+ _locales = await _engine.locales();
+ final saved = await _storage.read(_localeKey);
+ if (_locales.all.any((locale) => locale.id == saved)) {
+ _chosenLocaleId = saved;
+ }
+ } on Exception {
+ // The system language still works; only the picker goes missing.
+ }
+ return true;
+ }
+
+ void _onWords(int listen, String words, bool isFinal) {
+ if (listen != _listen) return;
+ if (isFinal) {
+ _heard = _joinWords(_heard, words);
+ _hearing = '';
+ } else {
+ _hearing = words;
+ }
+ _notify();
+ }
+
+ void _onStopped(int listen) {
+ if (listen != _listen || !isListening) return;
+ _setStatus(VoiceInputStatus.idle);
+ }
+
+ void _onError(int listen, String code) {
+ if (listen != _listen) return;
+ _setStatus(
+ VoiceNotice.isPermanent(code)
+ ? VoiceInputStatus.unavailable
+ : VoiceInputStatus.idle,
+ notice: VoiceNotice.forError(code),
+ );
+ }
+
+ /// Folds an unfinished guess into the transcript. A listen that ended with
+ /// no final result — an error, a cancel — would otherwise lose it to the
+ /// next listen's first guess.
+ void _keepHearing() {
+ _heard = transcript;
+ _hearing = '';
+ }
+
+ void _abandonListen() {
+ _listen++;
+ if (isListening) unawaited(_engine.cancel());
+ }
+
+ void _setStatus(VoiceInputStatus status, {String? notice}) {
+ _status = status;
+ _notice = notice;
+ _notify();
+ }
+
+ void _notify() {
+ if (!_disposed) notifyListeners();
+ }
+}
+
+String _joinWords(String first, String second) =>
+ [first.trim(), second.trim()].where((part) => part.isNotEmpty).join(' ');
diff --git a/mobile/lib/phone/voice_input_panel.dart b/mobile/lib/phone/voice_input_panel.dart
new file mode 100644
index 000000000..a93809900
--- /dev/null
+++ b/mobile/lib/phone/voice_input_panel.dart
@@ -0,0 +1,200 @@
+import 'package:flutter/material.dart';
+import 'package:lucide_icons_flutter/lucide_icons.dart';
+
+import 'package:harness_mobile/shared/theme/app_theme.dart';
+
+import 'voice_input_controller.dart';
+import 'voice_locale_picker.dart';
+import 'voice_mic_button.dart';
+import 'voice_panel_parts.dart';
+
+/// What a tap on the terminal opens: talk to the agent, read back what was
+/// heard, send it — or hand over to the keyboard.
+///
+/// It sits where the keyboard would, alone: [TerminalKeyBar] belongs to the
+/// keyboard, so this panel carries its own `⌄`.
+///
+/// ⚠️ **One fixed height, whatever it says.** The transcript scrolls inside a
+/// box of its own rather than growing the panel: this panel's top edge is the
+/// terminal's bottom edge, so a panel that grew by a line per sentence would
+/// resize the remote shell once per sentence.
+class VoiceInputPanel extends StatelessWidget {
+ const VoiceInputPanel({
+ super.key,
+ required this.voice,
+ required this.canSend,
+ required this.onSend,
+ required this.onUseKeyboard,
+ required this.onDismiss,
+ });
+
+ final VoiceInputController voice;
+
+ /// False while the stream is not accepting input. Listening still works —
+ /// what was said waits for Send.
+ final bool canSend;
+ final VoidCallback onSend;
+
+ /// Closes this panel for the keyboard, carrying what was heard with it.
+ final VoidCallback onUseKeyboard;
+
+ /// Puts the panel away, and what was not sent with it.
+ final VoidCallback onDismiss;
+
+ @override
+ Widget build(BuildContext context) {
+ AppTheme.watch(context);
+ return ListenableBuilder(
+ listenable: voice,
+ builder: (context, _) {
+ final hasWords = voice.transcript.isNotEmpty;
+ return ExcludeFocus(
+ // Same reason as the key bar's: nothing here may take focus from the
+ // terminal the keyboard button is about to hand it to.
+ child: DecoratedBox(
+ decoration: BoxDecoration(
+ color: AppPalette.panelBg,
+ border: Border(top: BorderSide(color: AppGlass.hair)),
+ ),
+ child: Padding(
+ padding: const EdgeInsets.fromLTRB(12, 8, 12, 10),
+ child: MediaQuery.withNoTextScaling(
+ child: Column(
+ mainAxisSize: MainAxisSize.min,
+ children: [
+ _StatusRow(voice: voice, onDismiss: onDismiss),
+ _Transcript(voice: voice),
+ const SizedBox(height: 6),
+ Row(
+ children: [
+ VoicePanelAction(
+ key: const ValueKey('voice-keyboard'),
+ icon: LucideIcons.keyboard300,
+ label: 'Keyboard',
+ onTap: onUseKeyboard,
+ ),
+ const Spacer(),
+ VoiceMicButton(
+ status: voice.status,
+ onPressed: voice.toggleListening,
+ ),
+ const Spacer(),
+ VoicePanelAction(
+ key: const ValueKey('voice-send'),
+ icon: LucideIcons.arrowUp300,
+ label: 'Send',
+ emphasized: true,
+ onTap: canSend && hasWords && !voice.isSending
+ ? onSend
+ : null,
+ ),
+ ],
+ ),
+ ],
+ ),
+ ),
+ ),
+ ),
+ );
+ },
+ );
+ }
+}
+
+/// Language, what the recognizer is doing, the way to start over and the way
+/// out.
+class _StatusRow extends StatelessWidget {
+ const _StatusRow({required this.voice, required this.onDismiss});
+
+ final VoiceInputController voice;
+ final VoidCallback onDismiss;
+
+ String get _label => switch (voice.status) {
+ VoiceInputStatus.starting => 'Starting…',
+ VoiceInputStatus.listening => 'Listening…',
+ VoiceInputStatus.unavailable => 'Voice input is off',
+ VoiceInputStatus.idle when voice.isSending => 'Sending…',
+ VoiceInputStatus.idle => 'Tap the mic to talk',
+ };
+
+ @override
+ Widget build(BuildContext context) {
+ final language = voiceLocaleLabel(voice.localeId);
+ // With words on screen the transcript box has no room for a notice, so a
+ // failed Send says so here instead.
+ final notice = voice.transcript.isNotEmpty ? voice.notice : null;
+ return SizedBox(
+ height: 28,
+ child: Row(
+ children: [
+ // Only once there is a choice: the list arrives with the first listen.
+ if (voice.locales.length > 1 && language.isNotEmpty) ...[
+ VoiceLocaleChip(
+ label: language,
+ onTap: () => showVoiceLocalePicker(context, voice),
+ ),
+ const SizedBox(width: 8),
+ ],
+ Expanded(
+ child: Text(
+ notice ?? _label,
+ maxLines: 1,
+ overflow: TextOverflow.ellipsis,
+ style: TextStyle(
+ fontSize: 12,
+ color: notice != null
+ ? AppPalette.warn
+ : AppPalette.textSecondary,
+ ),
+ ),
+ ),
+ if (voice.transcript.isNotEmpty)
+ VoicePanelIconButton(
+ key: const ValueKey('voice-clear'),
+ icon: LucideIcons.x300,
+ semanticLabel: 'Clear',
+ onTap: voice.clear,
+ ),
+ VoicePanelIconButton(
+ key: const ValueKey('voice-hide'),
+ icon: LucideIcons.chevronDown300,
+ semanticLabel: 'Hide voice input',
+ onTap: onDismiss,
+ ),
+ ],
+ ),
+ );
+ }
+}
+
+/// What was heard, newest words kept in view; the notice when there are none.
+class _Transcript extends StatelessWidget {
+ const _Transcript({required this.voice});
+
+ final VoiceInputController voice;
+
+ static const double _height = 58;
+
+ @override
+ Widget build(BuildContext context) {
+ final words = voice.transcript;
+ final notice = voice.notice;
+ final showsWords = words.isNotEmpty;
+ return SizedBox(
+ height: _height,
+ width: double.infinity,
+ child: SingleChildScrollView(
+ // Reversed so the latest words are the ones on screen as they arrive.
+ reverse: true,
+ child: Text(
+ showsWords ? words : notice ?? '',
+ style: TextStyle(
+ fontSize: showsWords ? 16 : 13,
+ height: 1.35,
+ color: showsWords ? AppPalette.textPrimary : AppPalette.warn,
+ ),
+ ),
+ ),
+ );
+ }
+}
diff --git a/mobile/lib/phone/voice_locale_picker.dart b/mobile/lib/phone/voice_locale_picker.dart
new file mode 100644
index 000000000..62e8c2160
--- /dev/null
+++ b/mobile/lib/phone/voice_locale_picker.dart
@@ -0,0 +1,65 @@
+import 'dart:async';
+import 'dart:ui' show Locale, PlatformDispatcher;
+
+import 'package:flutter/widgets.dart';
+import 'package:lucide_icons_flutter/lucide_icons.dart';
+
+import 'phone_sheet.dart';
+import 'voice_input_controller.dart';
+import 'voice_speech_engine.dart';
+
+/// The chip's short name for a recognizer locale: `vi-VN` and `vi_VN` are both
+/// `VI`.
+String voiceLocaleLabel(String? id) {
+ if (id == null || id.isEmpty) return '';
+ return id.split(RegExp('[-_]')).first.toUpperCase();
+}
+
+/// The recognizer's locales with the phone's own languages first.
+///
+/// A recognizer offers sixty-odd, alphabetically; the two a person actually
+/// talks in — the ones in the phone's language settings — would otherwise sit
+/// somewhere in the middle. The rest keep the recognizer's order.
+List<VoiceLocale> orderVoiceLocales(
+ List<VoiceLocale> all,
+ List<Locale> preferred,
+) {
+ final languages = [for (final locale in preferred) locale.languageCode];
+ int rank(VoiceLocale locale) {
+ final index = languages.indexOf(voiceLocaleLabel(locale.id).toLowerCase());
+ return index < 0 ? languages.length : index;
+ }
+
+ final indexed = all.indexed.toList()
+ ..sort((a, b) {
+ final byLanguage = rank(a.$2).compareTo(rank(b.$2));
+ return byLanguage != 0 ? byLanguage : a.$1.compareTo(b.$1);
+ });
+ return [for (final (_, locale) in indexed) locale];
+}
+
+/// Picks the language voice input listens in.
+Future<void> showVoiceLocalePicker(
+ BuildContext context,
+ VoiceInputController voice,
+) {
+ final current = voice.localeId;
+ final locales = orderVoiceLocales(
+ voice.locales,
+ PlatformDispatcher.instance.locales,
+ );
+ return showPhoneSheet(
+ context,
+ title: 'Voice input language',
+ actions: [
+ for (final locale in locales)
+ PhoneSheetAction(
+ icon: locale.id == current
+ ? LucideIcons.check300
+ : LucideIcons.languages300,
+ label: locale.name,
+ onTap: () => unawaited(voice.selectLocale(locale.id)),
+ ),
+ ],
+ );
+}
diff --git a/mobile/lib/phone/voice_mic_button.dart b/mobile/lib/phone/voice_mic_button.dart
new file mode 100644
index 000000000..66af73d05
--- /dev/null
+++ b/mobile/lib/phone/voice_mic_button.dart
@@ -0,0 +1,171 @@
+import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
+import 'package:lucide_icons_flutter/lucide_icons.dart';
+
+import 'package:harness_mobile/shared/theme/app_theme.dart';
+
+import 'voice_input_controller.dart';
+
+/// The round button voice input is started and stopped with.
+///
+/// While listening it breathes — a glow and a ring that swell and fade — so it
+/// reads as hearing from across the screen, which is where the eyes are: on the
+/// terminal, not on this button.
+///
+/// ⚠️ The swell is painted, never laid out. The button sits in a fixed square
+/// with room for the ring already in it: a panel whose height moved with the
+/// animation would resize the terminal above it, and every resize is a
+/// SIGWINCH on the far machine.
+class VoiceMicButton extends StatefulWidget {
+ const VoiceMicButton({
+ super.key,
+ required this.status,
+ required this.onPressed,
+ });
+
+ final VoiceInputStatus status;
+ final VoidCallback onPressed;
+
+ static const double _extent = 72;
+ static const double _core = 56;
+
+ @override
+ State<VoiceMicButton> createState() => _VoiceMicButtonState();
+}
+
+class _VoiceMicButtonState extends State<VoiceMicButton>
+ with SingleTickerProviderStateMixin {
+ late final AnimationController _breath = AnimationController(
+ vsync: this,
+ duration: const Duration(milliseconds: 1400),
+ );
+
+ bool get _listening => widget.status == VoiceInputStatus.listening;
+
+ @override
+ void initState() {
+ super.initState();
+ _syncBreath();
+ }
+
+ @override
+ void didUpdateWidget(VoiceMicButton oldWidget) {
+ super.didUpdateWidget(oldWidget);
+ if (oldWidget.status != widget.status) _syncBreath();
+ }
+
+ @override
+ void dispose() {
+ _breath.dispose();
+ super.dispose();
+ }
+
+ void _syncBreath() {
+ if (_listening) {
+ _breath.repeat();
+ return;
+ }
+ _breath
+ ..stop()
+ ..value = 0;
+ }
+
+ @override
+ Widget build(BuildContext context) {
+ AppTheme.watch(context);
+ final status = widget.status;
+ final lit =
+ status == VoiceInputStatus.listening ||
+ status == VoiceInputStatus.starting;
+ final off = status == VoiceInputStatus.unavailable;
+ return Semantics(
+ button: true,
+ label: lit ? 'Stop listening' : 'Start listening',
+ child: GestureDetector(
+ key: const ValueKey('voice-mic'),
+ behavior: HitTestBehavior.opaque,
+ onTap: () {
+ HapticFeedback.lightImpact();
+ widget.onPressed();
+ },
+ child: SizedBox.square(
+ dimension: VoiceMicButton._extent,
+ child: Stack(
+ alignment: Alignment.center,
+ children: [
+ if (_listening) _Ring(breath: _breath),
+ AnimatedContainer(
+ duration: const Duration(milliseconds: 220),
+ curve: Curves.easeOutCubic,
+ width: VoiceMicButton._core,
+ height: VoiceMicButton._core,
+ decoration: BoxDecoration(
+ shape: BoxShape.circle,
+ gradient: lit
+ ? LinearGradient(
+ begin: Alignment.topLeft,
+ end: Alignment.bottomRight,
+ colors: [
+ Color.lerp(AppPalette.accent, Colors.white, 0.18)!,
+ AppPalette.accent,
+ ],
+ )
+ : null,
+ color: lit ? null : AppGlass.surfaceFill,
+ border: Border.all(
+ color: lit
+ ? Colors.white.withValues(alpha: 0.28)
+ : AppGlass.lift,
+ ),
+ boxShadow: lit
+ ? [
+ BoxShadow(
+ color: AppPalette.accent.withValues(alpha: 0.45),
+ blurRadius: 18,
+ ),
+ ]
+ : null,
+ ),
+ child: Icon(
+ off ? LucideIcons.micOff300 : LucideIcons.mic300,
+ size: 24,
+ color: lit
+ ? Colors.white
+ : off
+ ? AppPalette.textFaint
+ : AppPalette.textPrimary,
+ ),
+ ),
+ ],
+ ),
+ ),
+ ),
+ );
+ }
+}
+
+/// The ring that leaves the button while it listens.
+class _Ring extends StatelessWidget {
+ const _Ring({required this.breath});
+
+ final Animation<double> breath;
+
+ @override
+ Widget build(BuildContext context) => AnimatedBuilder(
+ animation: breath,
+ builder: (context, _) {
+ final t = Curves.easeOut.transform(breath.value);
+ return Transform.scale(
+ scale: 1 + 0.28 * t,
+ child: Container(
+ width: VoiceMicButton._core,
+ height: VoiceMicButton._core,
+ decoration: BoxDecoration(
+ shape: BoxShape.circle,
+ color: AppPalette.accent.withValues(alpha: 0.35 * (1 - t)),
+ ),
+ ),
+ );
+ },
+ );
+}
diff --git a/mobile/lib/phone/voice_notice.dart b/mobile/lib/phone/voice_notice.dart
new file mode 100644
index 000000000..811f99ef6
--- /dev/null
+++ b/mobile/lib/phone/voice_notice.dart
@@ -0,0 +1,33 @@
+/// The one line voice input shows when it cannot do what was asked.
+///
+/// Worded for the person, keyed on the plugin's `error_*` codes — which are
+/// not meant for display and differ between iOS and Android.
+abstract final class VoiceNotice {
+ static const unavailable =
+ 'Voice input is off. Allow Harness the microphone and speech recognition in Settings, or use the keyboard.';
+
+ static const notSent =
+ "Not sent — this terminal isn't taking input right now.";
+
+ static const couldNotStart =
+ "Voice input couldn't start. Tap the mic to try again.";
+
+ /// Whether [code] means permission is missing, rather than one listen going
+ /// wrong. Trying again will not help; Settings will.
+ static bool isPermanent(String code) =>
+ code.contains('permission') || code.contains('not_authorized');
+
+ static String forError(String code) {
+ if (isPermanent(code)) return unavailable;
+ if (code.contains('no_match') || code.contains('speech_timeout')) {
+ return "Didn't catch that. Tap the mic to try again.";
+ }
+ if (code.contains('language') || code.contains('assets_not_installed')) {
+ return "This language isn't available for voice input on this phone.";
+ }
+ if (code.contains('network') || code.contains('server')) {
+ return 'Voice input needs a connection. Tap the mic to try again.';
+ }
+ return 'Voice input stopped. Tap the mic to try again.';
+ }
+}
diff --git a/mobile/lib/phone/voice_panel_parts.dart b/mobile/lib/phone/voice_panel_parts.dart
new file mode 100644
index 000000000..b65fb48e2
--- /dev/null
+++ b/mobile/lib/phone/voice_panel_parts.dart
@@ -0,0 +1,131 @@
+import 'package:flutter/material.dart';
+
+import 'package:harness_mobile/shared/theme/app_theme.dart';
+
+/// The language voice input listens in, as a two-letter chip.
+class VoiceLocaleChip extends StatelessWidget {
+ const VoiceLocaleChip({super.key, required this.label, required this.onTap});
+
+ final String label;
+ final VoidCallback onTap;
+
+ @override
+ Widget build(BuildContext context) => Semantics(
+ button: true,
+ label: 'Voice input language, $label',
+ child: GestureDetector(
+ key: const ValueKey('voice-locale'),
+ onTap: onTap,
+ child: Container(
+ padding: const EdgeInsets.symmetric(horizontal: 8, vertical: 3),
+ decoration: BoxDecoration(
+ color: AppGlass.surfaceFill,
+ borderRadius: BorderRadius.circular(6),
+ border: Border.all(color: AppGlass.lift),
+ ),
+ child: Text(
+ label,
+ style: TextStyle(
+ fontSize: 11,
+ fontWeight: FontWeight.w600,
+ color: AppPalette.textPrimary,
+ ),
+ ),
+ ),
+ ),
+ );
+}
+
+/// A pill beside the mic. Null [onTap] draws it dimmed and dead.
+class VoicePanelAction extends StatelessWidget {
+ const VoicePanelAction({
+ super.key,
+ required this.icon,
+ required this.label,
+ required this.onTap,
+ this.emphasized = false,
+ });
+
+ final IconData icon;
+ final String label;
+ final VoidCallback? onTap;
+ final bool emphasized;
+
+ @override
+ Widget build(BuildContext context) {
+ final live = onTap != null;
+ final filled = emphasized && live;
+ final foreground = filled
+ ? Colors.white
+ : live
+ ? AppPalette.textPrimary
+ : AppPalette.textFaint;
+ return Semantics(
+ button: true,
+ enabled: live,
+ label: label,
+ child: GestureDetector(
+ behavior: HitTestBehavior.opaque,
+ onTap: onTap,
+ child: AnimatedContainer(
+ duration: const Duration(milliseconds: 160),
+ curve: Curves.easeOut,
+ height: 40,
+ padding: const EdgeInsets.symmetric(horizontal: 14),
+ decoration: BoxDecoration(
+ color: filled ? AppPalette.accent : AppGlass.surfaceFill,
+ borderRadius: BorderRadius.circular(20),
+ border: Border.all(
+ color: filled
+ ? Colors.white.withValues(alpha: 0.2)
+ : AppGlass.lift,
+ ),
+ ),
+ child: Row(
+ mainAxisSize: MainAxisSize.min,
+ children: [
+ Icon(icon, size: 16, color: foreground),
+ const SizedBox(width: 6),
+ Text(
+ label,
+ style: TextStyle(
+ fontSize: 14,
+ fontWeight: FontWeight.w600,
+ color: foreground,
+ ),
+ ),
+ ],
+ ),
+ ),
+ ),
+ );
+ }
+}
+
+/// A bare icon in the panel's status row: Clear, and `⌄`.
+class VoicePanelIconButton extends StatelessWidget {
+ const VoicePanelIconButton({
+ super.key,
+ required this.icon,
+ required this.semanticLabel,
+ required this.onTap,
+ });
+
+ final IconData icon;
+ final String semanticLabel;
+ final VoidCallback onTap;
+
+ @override
+ Widget build(BuildContext context) => Semantics(
+ button: true,
+ label: semanticLabel,
+ child: GestureDetector(
+ behavior: HitTestBehavior.opaque,
+ onTap: onTap,
+ child: Padding(
+ padding: const EdgeInsets.symmetric(horizontal: 6, vertical: 4),
+ child: Icon(icon, size: 18, color: AppPalette.textSecondary),
+ ),
+ ),
+ );
+}
diff --git a/mobile/lib/phone/voice_speech_engine.dart b/mobile/lib/phone/voice_speech_engine.dart
new file mode 100644
index 000000000..f72558b4f
--- /dev/null
+++ b/mobile/lib/phone/voice_speech_engine.dart
@@ -0,0 +1,109 @@
+import 'package:flutter/foundation.dart';
+import 'package:speech_to_text/speech_to_text.dart';
+
+/// One language the recognizer can listen in, as the OS names it.
+typedef VoiceLocale = ({String id, String name});
+
+/// The recognizer's languages, and the one it uses when none is named.
+typedef VoiceLocales = ({List<VoiceLocale> all, String? systemId});
+
+/// The speech recognizer behind [VoiceInputController], as the four things the
+/// phone asks of it.
+///
+/// A seam rather than `SpeechToText` itself: the plugin is a platform-channel
+/// singleton, so a widget test that reached it would hang on a method call no
+/// engine answers.
+abstract interface class SpeechEngine {
+ /// Asks for the microphone and speech permissions the first time it runs.
+ /// False when either is refused, or the device has no recognizer at all.
+ Future<bool> prepare();
+
+ /// Starts one listen. [onWords] gets the running guess and, last, the final
+ /// one; [onStopped] fires once the recognizer has stopped hearing, and
+ /// [onError] carries the plugin's `error_*` code. Throws when the listen
+ /// cannot start.
+ Future<void> listen({
+ required String? localeId,
+ required void Function(String words, bool isFinal) onWords,
+ required VoidCallback onStopped,
+ required ValueChanged<String> onError,
+ });
+
+ /// Ends the listen and lets the final result arrive.
+ Future<void> stop();
+
+ /// Ends the listen with no final result.
+ Future<void> cancel();
+
+ /// Only meaningful after [prepare] has returned true.
+ Future<VoiceLocales> locales();
+}
+
+/// [SpeechEngine] over the OS recognizer: Speech on iOS, SpeechRecognizer on
+/// Android.
+class PlatformSpeechEngine implements SpeechEngine {
+ final SpeechToText _speech = SpeechToText();
+
+ /// A silence this long ends the listen. Long enough to think mid-sentence;
+ /// Android may still cut it shorter, which the plugin cannot override.
+ static const _pauseFor = Duration(seconds: 4);
+
+ /// Both OSes stop a recognition near a minute on their own.
+ static const _listenFor = Duration(minutes: 1);
+
+ @override
+ Future<bool> prepare() async {
+ try {
+ return await _speech.initialize(
+ options: [SpeechToText.androidNoBluetooth],
+ );
+ } on Exception {
+ return false;
+ }
+ }
+
+ @override
+ Future<void> listen({
+ required String? localeId,
+ required void Function(String words, bool isFinal) onWords,
+ required VoidCallback onStopped,
+ required ValueChanged<String> onError,
+ }) async {
+ // ⚠️ Assigned before EVERY listen, not handed to `initialize`. The plugin
+ // is one instance for the whole app and `initialize` returns early once it
+ // has worked, so listeners given to it would stay those of the first
+ // controller ever made — a pager long since disposed.
+ _speech
+ ..statusListener = (status) {
+ if (status != SpeechToText.listeningStatus) onStopped();
+ }
+ ..errorListener = (error) => onError(error.errorMsg);
+ await _speech.listen(
+ onResult: (result) => onWords(result.recognizedWords, result.finalResult),
+ listenOptions: SpeechListenOptions(
+ listenMode: ListenMode.dictation,
+ autoPunctuation: true,
+ cancelOnError: true,
+ localeId: localeId,
+ pauseFor: _pauseFor,
+ listenFor: _listenFor,
+ ),
+ );
+ }
+
+ @override
+ Future<void> stop() => _speech.stop();
+
+ @override
+ Future<void> cancel() => _speech.cancel();
+
+ @override
+ Future<VoiceLocales> locales() async {
+ final all = await _speech.locales();
+ final system = await _speech.systemLocale();
+ return (
+ all: [for (final locale in all) (id: locale.localeId, name: locale.name)],
+ systemId: system?.localeId,
+ );
+ }
+}
diff --git a/mobile/lib/widgets/terminal_panel.dart b/mobile/lib/widgets/terminal_panel.dart
index 19928c0d5..4da04fb95 100644
--- a/mobile/lib/widgets/terminal_panel.dart
+++ b/mobile/lib/widgets/terminal_panel.dart
@@ -89,6 +89,16 @@ class TerminalPanel extends StatefulWidget {
final bool showHeader;
final int focusRequest;
+ /// Takes over the tap that would raise the software keyboard. Null leaves it
+ /// to xterm, which is what every desktop tile does.
+ ///
+ /// Set on the phone, where that tap opens voice input instead
+ /// (`phone/terminal_page.dart`). Claimed on tap DOWN — xterm then neither
+ /// raises the keyboard nor reports the click to a mouse-tracking program —
+ /// but run on tap UP, so a scroll that began as a press opens nothing. A tap
+ /// that clears a selection, or opens a link, is still exactly that.
+ final VoidCallback? onInputTap;
+
/// Whether this tile's composer textbox is showing. Only consulted for a remote machine.
final bool composerVisible;
final bool readOnly;
@@ -118,6 +128,7 @@ class TerminalPanel extends StatefulWidget {
this.compactHeader = false,
this.showHeader = true,
this.focusRequest = 0,
+ this.onInputTap,
this.composerVisible = false,
this.readOnly = false,
this.notice,
@@ -193,6 +204,9 @@ class _TerminalPanelState extends State<TerminalPanel>
null,
);
String? _pressedLink;
+
+ /// Whether the tap in progress was claimed for [TerminalPanel.onInputTap].
+ bool _inputTapClaimed = false;
bool _openingLink = false;
bool _linkRefreshPending = false;
bool _followTail = true;
@@ -1153,6 +1167,24 @@ class _TerminalPanelState extends State<TerminalPanel>
_hoveredLink.value = target;
}
+ bool _onTerminalTapDown(TapDownDetails details, CellOffset cell) {
+ _inputTapClaimed = false;
+ if (_onLinkTapDown(details, cell)) return true;
+ if (widget.onInputTap == null || _controller.selection != null) {
+ return false;
+ }
+ return _inputTapClaimed = true;
+ }
+
+ void _onTerminalTapUp(TapUpDetails details, CellOffset cell) {
+ if (!_inputTapClaimed) {
+ _onLinkTapUp(details, cell);
+ return;
+ }
+ _inputTapClaimed = false;
+ widget.onInputTap?.call();
+ }
+
bool _onLinkTapDown(TapDownDetails details, CellOffset cell) {
_pressedLink = _linkModifierPressed
? _linkAtPointer(details.globalPosition)
@@ -1374,8 +1406,8 @@ class _TerminalPanelState extends State<TerminalPanel>
allowedMimeTypes: const ['image/png'],
onContentInserted: _onContentInserted,
onKeyEvent: _onTerminalKey,
- onTapDown: _onLinkTapDown,
- onTapUp: _onLinkTapUp,
+ onTapDown: _onTerminalTapDown,
+ onTapUp: _onTerminalTapUp,
// Constant on purpose. The click cursor is applied by
// [_LinkTooltip]'s own MouseRegion, which repaints
// without rebuilding this view.
diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock
index 04143c677..591037df3 100644
--- a/mobile/pubspec.lock
+++ b/mobile/pubspec.lock
@@ -344,6 +344,14 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.0.0"
+ json_annotation:
+ dependency: transitive
+ description:
+ name: json_annotation
+ sha256: "2a743920d81b7910627f68ee2c9ac1fc0bfee32b9fc3403587d7c6791ca12f80"
+ url: "https://pub.dev"
+ source: hosted
+ version: "4.12.0"
leak_tracker:
dependency: transitive
description:
@@ -581,6 +589,30 @@ packages:
url: "https://pub.dev"
source: hosted
version: "1.10.2"
+ speech_to_text:
+ dependency: "direct main"
+ description:
+ name: speech_to_text
+ sha256: f2fabd017b5cb07f4e06d496b2d4b957e39534bc848f92b1e6acac80a10f4967
+ url: "https://pub.dev"
+ source: hosted
+ version: "7.5.0"
+ speech_to_text_platform_interface:
+ dependency: transitive
+ description:
+ name: speech_to_text_platform_interface
+ sha256: a7e16e02853853ed7534ac2bde9a1c4f39c8879970a7974ac6ff832d4bdaa4b0
+ url: "https://pub.dev"
+ source: hosted
+ version: "2.4.0"
+ speech_to_text_windows:
+ dependency: transitive
+ description:
+ name: speech_to_text_windows
+ sha256: "2d1d10565b23262386b453b33656299608dc7a66784453735d6c1318f13f44d7"
+ url: "https://pub.dev"
+ source: hosted
+ version: "1.0.1"
stack_trace:
dependency: transitive
description:
diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml
index 99f7c77d6..9ecacf13f 100644
--- a/mobile/pubspec.yaml
+++ b/mobile/pubspec.yaml
@@ -59,6 +59,12 @@ dependencies:
# for why the whole pipe past this point accepts nothing else.
image_picker: ^1.2.3
+ # Talking to an agent: tapping the terminal opens voice input before the
+ # keyboard (`phone/voice_input_panel.dart`). The OS's own recognizer — Speech
+ # on iOS, SpeechRecognizer on Android — so no audio reaches a server of ours,
+ # and it is the one thing in this app that opens the microphone.
+ speech_to_text: ^7.5.0
+
dev_dependencies:
flutter_test:
sdk: flutter
diff --git a/mobile/test/terminal_input_dock_test.dart b/mobile/test/terminal_input_dock_test.dart
new file mode 100644
index 000000000..65affc8ac
--- /dev/null
+++ b/mobile/test/terminal_input_dock_test.dart
@@ -0,0 +1,149 @@
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness_mobile/phone/terminal_input_dock.dart';
+import 'package:harness_mobile/phone/terminal_key_bar.dart';
+import 'package:harness_mobile/phone/voice_input_controller.dart';
+import 'package:harness_mobile/phone/voice_input_panel.dart';
+import 'package:harness_mobile/terminal/terminal_session.dart';
+
+import 'voice_fakes.dart';
+
+void main() {
+ late FakeSpeechEngine engine;
+ late VoiceInputController voice;
+ late TerminalSession session;
+ late List<(String, Map<String, dynamic>)> frames;
+ late int dismissals;
+ late int keyboardRequests;
+
+ setUp(() {
+ engine = FakeSpeechEngine();
+ voice = VoiceInputController(
+ engine: engine,
+ storage: MemoryKeyValueStore(),
+ );
+ frames = [];
+ dismissals = 0;
+ keyboardRequests = 0;
+ session =
+ TerminalSession(
+ machineId: 'm',
+ agentId: 'a',
+ agentName: 'Agent',
+ engineId: 'claude',
+ send: (type, payload) async {
+ frames.add((type, payload));
+ return true;
+ },
+ sendBinary: (_) async => true,
+ )
+ ..status = TerminalSessionStatus.controlling
+ ..streamId = 's';
+ });
+
+ tearDown(() {
+ voice.dispose();
+ session.dispose();
+ });
+
+ Future<void> pumpDock(WidgetTester tester, {bool keyboardUp = false}) =>
+ tester.pumpWidget(
+ MaterialApp(
+ home: Scaffold(
+ body: Align(
+ alignment: Alignment.bottomCenter,
+ child: TerminalInputDock(
+ session: session,
+ voice: voice,
+ keyboardUp: keyboardUp,
+ onDismiss: () => dismissals++,
+ onUseKeyboard: () => keyboardRequests++,
+ ),
+ ),
+ ),
+ ),
+ );
+
+ testWidgets('nothing is drawn while no input is up', (tester) async {
+ await pumpDock(tester);
+
+ expect(find.byType(TerminalKeyBar), findsNothing);
+ expect(find.byType(VoiceInputPanel), findsNothing);
+ });
+
+ testWidgets('the keyboard gets the key bar alone', (tester) async {
+ await pumpDock(tester, keyboardUp: true);
+
+ expect(find.byType(TerminalKeyBar), findsOneWidget);
+ expect(find.byType(VoiceInputPanel), findsNothing);
+ });
+
+ testWidgets('voice input takes the key bar\'s place, and Send makes a turn', (
+ tester,
+ ) async {
+ await pumpDock(tester);
+ await voice.open();
+ await tester.pump();
+
+ expect(find.byType(TerminalKeyBar), findsNothing);
+ expect(find.byType(VoiceInputPanel), findsOneWidget);
+ expect(find.text('Listening…'), findsOneWidget);
+
+ engine.finish('summarise the diff');
+ await tester.pump();
+ expect(find.text('summarise the diff'), findsOneWidget);
+
+ await tester.tap(find.byKey(const ValueKey('voice-send')));
+ await tester.pumpAndSettle();
+
+ // The composer's frame, not typed bytes — the machine owns the Enter.
+ expect(frames, hasLength(1));
+ expect(frames.single.$1, 'message');
+ expect(frames.single.$2['content'], 'summarise the diff');
+ expect(find.text('summarise the diff'), findsNothing);
+ });
+
+ testWidgets('Send is dead while the terminal takes no input', (tester) async {
+ session.status = TerminalSessionStatus.takenOver;
+ await pumpDock(tester);
+ await voice.open();
+ engine.finish('hello');
+ await tester.pump();
+
+ await tester.tap(find.byKey(const ValueKey('voice-send')));
+ await tester.pump();
+
+ expect(frames, isEmpty);
+ expect(voice.transcript, 'hello');
+ });
+
+ testWidgets('Keyboard and ⌄ hand back to the page', (tester) async {
+ await pumpDock(tester);
+ await voice.open();
+ await tester.pump();
+
+ await tester.tap(find.byKey(const ValueKey('voice-keyboard')));
+ await tester.tap(find.byKey(const ValueKey('voice-hide')));
+
+ expect(keyboardRequests, 1);
+ expect(dismissals, 1);
+ });
+
+ testWidgets('the mic stops a listen and starts the next', (tester) async {
+ await pumpDock(tester);
+ await voice.open();
+ await tester.pump();
+
+ await tester.tap(find.byKey(const ValueKey('voice-mic')));
+ await tester.pump();
+ expect(voice.status, VoiceInputStatus.idle);
+ expect(engine.stops, 1);
+
+ await tester.tap(find.byKey(const ValueKey('voice-mic')));
+ await tester.pump();
+ expect(voice.status, VoiceInputStatus.listening);
+ // Stopped here, not left breathing: the ring repeats forever.
+ voice.close();
+ await tester.pumpAndSettle();
+ });
+}
diff --git a/mobile/test/terminal_panel_input_tap_test.dart b/mobile/test/terminal_panel_input_tap_test.dart
new file mode 100644
index 000000000..9e228a63c
--- /dev/null
+++ b/mobile/test/terminal_panel_input_tap_test.dart
@@ -0,0 +1,109 @@
+import 'package:flutter/gestures.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness_mobile/auth/auth_session.dart';
+import 'package:harness_mobile/core/config.dart';
+import 'package:harness_mobile/state/app_state.dart';
+import 'package:harness_mobile/terminal/terminal_session.dart';
+import 'package:harness_mobile/widgets/terminal_panel.dart';
+import 'package:xterm/xterm.dart';
+
+/// On the phone a tap on the terminal opens voice input, and the keyboard is
+/// only ever what voice input hands over to. The panel is where that tap is
+/// taken: xterm would otherwise raise the keyboard itself before anyone else
+/// could say otherwise.
+void main() {
+ late AppNotifier notifier;
+ late TerminalSession session;
+ late int inputTaps;
+
+ setUp(() {
+ notifier = AppNotifier(
+ config: AppConfig.dev,
+ authSession: AuthSession(),
+ configStore: null,
+ );
+ session = TerminalSession(
+ machineId: 'm',
+ agentId: 'a',
+ agentName: 'Agent',
+ engineId: 'claude',
+ send: (_, _) async => true,
+ sendBinary: (_) async => true,
+ );
+ session.status = TerminalSessionStatus.controlling;
+ session.streamId = 's';
+ inputTaps = 0;
+ });
+
+ tearDown(() {
+ session.dispose();
+ notifier.dispose();
+ });
+
+ Future<void> pumpPanel(
+ WidgetTester tester, {
+ VoidCallback? onInputTap,
+ }) async {
+ await tester.pumpWidget(
+ MaterialApp(
+ home: Scaffold(
+ body: SizedBox(
+ width: 400,
+ height: 320,
+ child: TerminalPanel(
+ notifier: notifier,
+ session: session,
+ focused: false,
+ showHeader: false,
+ onInputTap: onInputTap,
+ ),
+ ),
+ ),
+ ),
+ );
+ await tester.pump();
+ }
+
+ testWidgets('a taken tap raises no keyboard, and runs the host instead', (
+ tester,
+ ) async {
+ await pumpPanel(tester, onInputTap: () => inputTaps++);
+
+ await tester.tap(find.byType(TerminalView));
+ // Past the double-tap window, which is what holds a single tap back.
+ await tester.pump(kDoubleTapTimeout);
+
+ expect(inputTaps, 1);
+ expect(tester.testTextInput.hasAnyClients, isFalse);
+ });
+
+ testWidgets('without a host the tap is still xterm\'s, keyboard and all', (
+ tester,
+ ) async {
+ await pumpPanel(tester);
+
+ await tester.tap(find.byType(TerminalView));
+ // Past the double-tap window, which is what holds a single tap back.
+ await tester.pump(kDoubleTapTimeout);
+
+ expect(tester.testTextInput.hasAnyClients, isTrue);
+ });
+
+ testWidgets('a scroll that began as a press opens nothing', (tester) async {
+ await pumpPanel(tester, onInputTap: () => inputTaps++);
+
+ final gesture = await tester.startGesture(
+ tester.getCenter(find.byType(TerminalView)),
+ );
+ // Held past the press timeout, so the tap recognizer has already reported
+ // tap DOWN before the finger moves.
+ await tester.pump(const Duration(milliseconds: 200));
+ await gesture.moveBy(const Offset(0, -120));
+ await gesture.up();
+ await tester.pump();
+
+ expect(inputTaps, 0);
+ expect(tester.testTextInput.hasAnyClients, isFalse);
+ });
+}
diff --git a/mobile/test/voice_fakes.dart b/mobile/test/voice_fakes.dart
new file mode 100644
index 000000000..967129751
--- /dev/null
+++ b/mobile/test/voice_fakes.dart
@@ -0,0 +1,82 @@
+import 'dart:async';
+
+import 'package:flutter/foundation.dart';
+import 'package:harness_mobile/core/local_key_value_store.dart';
+import 'package:harness_mobile/phone/voice_speech_engine.dart';
+
+/// A recognizer the test speaks for: [say] and [finish] stand in for a person
+/// talking, [fail] for the OS giving up.
+class FakeSpeechEngine implements SpeechEngine {
+ FakeSpeechEngine({this.allowed = true});
+
+ bool allowed;
+ VoiceLocales available = (
+ all: const [
+ (id: 'en-US', name: 'English'),
+ (id: 'vi-VN', name: 'Tiếng Việt'),
+ ],
+ systemId: 'en-US',
+ );
+
+ /// Held open to keep a listen in `starting`, like a permission prompt does.
+ Completer<bool>? pendingPrepare;
+
+ int prepares = 0;
+ int stops = 0;
+ int cancels = 0;
+ final List<String?> listenedIn = [];
+
+ void Function(String words, bool isFinal)? _onWords;
+ VoidCallback? _onStopped;
+ ValueChanged<String>? _onError;
+
+ @override
+ Future<bool> prepare() {
+ prepares++;
+ return pendingPrepare?.future ?? Future.value(allowed);
+ }
+
+ @override
+ Future<void> listen({
+ required String? localeId,
+ required void Function(String words, bool isFinal) onWords,
+ required VoidCallback onStopped,
+ required ValueChanged<String> onError,
+ }) async {
+ listenedIn.add(localeId);
+ _onWords = onWords;
+ _onStopped = onStopped;
+ _onError = onError;
+ }
+
+ void say(String words) => _onWords?.call(words, false);
+
+ void finish(String words) {
+ _onWords?.call(words, true);
+ _onStopped?.call();
+ }
+
+ void fail(String code) => _onError?.call(code);
+
+ @override
+ Future<void> stop() async => stops++;
+
+ @override
+ Future<void> cancel() async => cancels++;
+
+ @override
+ Future<VoiceLocales> locales() async => available;
+}
+
+class MemoryKeyValueStore implements LocalKeyValueStore {
+ final Map<String, String> values = {};
+
+ @override
+ Future<String?> read(String key) async => values[key];
+
+ @override
+ Future<void> write(String key, String value) async => values[key] = value;
+
+ @override
+ Future<void> delete(String key) async => values.remove(key);
+}
diff --git a/mobile/test/voice_input_controller_test.dart b/mobile/test/voice_input_controller_test.dart
new file mode 100644
index 000000000..2bed504e7
--- /dev/null
+++ b/mobile/test/voice_input_controller_test.dart
@@ -0,0 +1,166 @@
+import 'dart:async';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness_mobile/phone/voice_input_controller.dart';
+import 'package:harness_mobile/phone/voice_notice.dart';
+
+import 'voice_fakes.dart';
+
+void main() {
+ late FakeSpeechEngine engine;
+ late MemoryKeyValueStore storage;
+ late VoiceInputController voice;
+
+ setUp(() {
+ engine = FakeSpeechEngine();
+ storage = MemoryKeyValueStore();
+ voice = VoiceInputController(engine: engine, storage: storage);
+ });
+
+ tearDown(() => voice.dispose());
+
+ test(
+ 'opening is listening: the panel is up and the words arrive live',
+ () async {
+ await voice.open();
+
+ expect(voice.isOpen, isTrue);
+ expect(voice.status, VoiceInputStatus.listening);
+
+ engine.say('fix the');
+ expect(voice.transcript, 'fix the');
+
+ engine.finish('fix the failing test');
+ expect(voice.transcript, 'fix the failing test');
+ expect(voice.status, VoiceInputStatus.idle);
+ },
+ );
+
+ test('a second listen adds to what the first one heard', () async {
+ await voice.open();
+ engine.finish('run the tests');
+
+ await voice.toggleListening();
+ engine.say('then commit');
+
+ expect(voice.transcript, 'run the tests then commit');
+ });
+
+ test('a guess a failed listen never finalised is kept, not lost', () async {
+ await voice.open();
+ engine.say('rename the');
+ engine.fail('error_no_match');
+
+ expect(voice.status, VoiceInputStatus.idle);
+ expect(voice.notice, VoiceNotice.forError('error_no_match'));
+
+ await voice.startListening();
+ engine.say('module');
+ expect(voice.transcript, 'rename the module');
+ });
+
+ test('a refused permission says so, and the next open asks again', () async {
+ engine.allowed = false;
+ await voice.open();
+
+ expect(voice.status, VoiceInputStatus.unavailable);
+ expect(voice.notice, VoiceNotice.unavailable);
+
+ engine.allowed = true;
+ voice.close();
+ await voice.open();
+ expect(voice.status, VoiceInputStatus.listening);
+ expect(engine.prepares, 2);
+ });
+
+ test('closing while the permission prompt is up starts no listen', () async {
+ engine.pendingPrepare = Completer<bool>();
+ final opening = voice.open();
+ expect(voice.status, VoiceInputStatus.starting);
+
+ voice.close();
+ engine.pendingPrepare!.complete(true);
+ await opening;
+
+ expect(engine.listenedIn, isEmpty);
+ expect(voice.status, VoiceInputStatus.idle);
+ });
+
+ test('send hands over everything heard and empties the panel', () async {
+ await voice.open();
+ engine.finish('open a PR');
+ await voice.startListening();
+ engine.say('for this branch');
+
+ final delivered = <String>[];
+ await voice.submit((text) async {
+ delivered.add(text);
+ return true;
+ });
+
+ expect(delivered, ['open a PR for this branch']);
+ expect(voice.transcript, isEmpty);
+ expect(engine.cancels, 1);
+
+ // The listen it cut short answers late: those words were already sent.
+ engine.finish('for this branch');
+ expect(voice.transcript, isEmpty);
+ });
+
+ test('a send that fails keeps the words for another try', () async {
+ await voice.open();
+ engine.finish('deploy');
+
+ await voice.submit((_) async => false);
+
+ expect(voice.transcript, 'deploy');
+ expect(voice.notice, VoiceNotice.notSent);
+ expect(voice.isSending, isFalse);
+ });
+
+ test('closing stops the microphone and forgets what was not sent', () async {
+ await voice.open();
+ engine.say('never mind');
+
+ voice.close();
+
+ expect(voice.isOpen, isFalse);
+ expect(voice.transcript, isEmpty);
+ expect(engine.cancels, 1);
+ });
+
+ test(
+ 'stopping keeps the listen\'s words: its final result still counts',
+ () async {
+ await voice.open();
+ engine.say('almost');
+
+ await voice.stopListening();
+ expect(engine.stops, 1);
+ engine.finish('almost done');
+
+ expect(voice.transcript, 'almost done');
+ },
+ );
+
+ test(
+ 'a chosen language is used, remembered, and restored next time',
+ () async {
+ await voice.open();
+ expect(voice.localeId, 'en-US');
+
+ await voice.selectLocale('vi-VN');
+
+ expect(storage.values['voice_input_locale'], 'vi-VN');
+ expect(engine.listenedIn.last, 'vi-VN');
+
+ final later = VoiceInputController(
+ engine: FakeSpeechEngine(),
+ storage: storage,
+ );
+ addTearDown(later.dispose);
+ await later.open();
+ expect(later.localeId, 'vi-VN');
+ },
+ );
+}
diff --git a/mobile/test/voice_locale_picker_test.dart b/mobile/test/voice_locale_picker_test.dart
new file mode 100644
index 000000000..eccb14776
--- /dev/null
+++ b/mobile/test/voice_locale_picker_test.dart
@@ -0,0 +1,32 @@
+import 'dart:ui';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:harness_mobile/phone/voice_locale_picker.dart';
+
+void main() {
+ test('the chip names the language, whichever separator the OS uses', () {
+ expect(voiceLocaleLabel('vi-VN'), 'VI');
+ expect(voiceLocaleLabel('en_US'), 'EN');
+ expect(voiceLocaleLabel(null), isEmpty);
+ });
+
+ test("the phone's own languages come first, in its order", () {
+ const all = [
+ (id: 'de-DE', name: 'Deutsch'),
+ (id: 'en-GB', name: 'English (UK)'),
+ (id: 'en-US', name: 'English (US)'),
+ (id: 'fr-FR', name: 'Français'),
+ (id: 'vi-VN', name: 'Tiếng Việt'),
+ ];
+
+ final ordered = orderVoiceLocales(all, const [
+ Locale('vi', 'VN'),
+ Locale('en', 'US'),
+ ]);
+
+ expect(
+ [for (final locale in ordered) locale.id],
+ ['vi-VN', 'en-GB', 'en-US', 'de-DE', 'fr-FR'],
+ );
+ });
+}