The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Phone: the voice button in a thread's reply box (Dee) (#2472)
0xken15f4f853d93ab3
diff --git a/mobile/lib/swarm/swarm_thread_page.dart b/mobile/lib/swarm/swarm_thread_page.dartindex 1922a0443..ac75dee0c 100644--- a/mobile/lib/swarm/swarm_thread_page.dart+++ b/mobile/lib/swarm/swarm_thread_page.dart@@ -20,6 +20,7 @@ import 'swarm_outbox.dart';import 'swarm_paste.dart';import 'swarm_session.dart';import 'swarm_thread.dart';+import 'swarm_voice.dart';/// Draws one of a thread's messages as its chat draws them; a tap on it/// opens or closes its actions ([acting]).@@ -114,15 +115,32 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {SwarmSession? _session;bool _watching = false;+ /// Not covered by a screen pushed over this one.+ bool _visible = true;++ /// The reply box's mic, while the shell can hear one ([SwarmScope.voiceOf]):+ /// the main chat's, a take sent as a reply.+ SwarmVoice? _voice;+@overridevoid initState() {super.initState();_text.addListener(_typed);_thread.composer.addListener(_drafted);_thread.addListener(_seen);+ _thread.addListener(_guardVoice);unawaited(_thread.load());}+ /// A thread the person can no longer reply in hears nothing: a take under+ /// way goes, nothing of it written or sent.+ void _guardVoice() {+ if (!_mayReply) _voice?.hide();+ }++ /// The person may still reply here.+ bool get _mayReply => _thread.state != SwarmChatState.lost && _thread.canPost;+/// While on screen, what the thread shows is read ([SwarmThread.markRead]).void _seen() {if (_watching) _thread.markRead();@@ -133,7 +151,25 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {super.didChangeDependencies();final session = SwarmScope.of(context);// Off while a screen pushed over this one covers it.- final watch = session != null && TickerMode.valuesOf(context).enabled;+ _visible = TickerMode.valuesOf(context).enabled;+ final watch = session != null && _visible;+ if (_voice == null) {+ if (SwarmScope.voiceOf(context) case final create?) {+ _voice = SwarmVoice(+ create: create,+ dictation: SwarmScope.dictationOf(context),+ draft: () => _thread.draft,+ write: (text) {+ if (_mayReply) _thread.draft = text;+ },+ send: _spoken,+ );+ }+ }+ // A covered thread hears nothing, nor one signed out of: its take goes+ // with it. Another account loses the thread's read ([_guardVoice]).+ if (!watch) _voice?.hide();+ _guardVoice();if (session != _session || watch != _watching) {if (_watching) _session?.unwatch(widget.swarmId);_session = session;@@ -148,6 +184,8 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {if (_watching) _session?.unwatch(widget.swarmId);_thread.composer.removeListener(_drafted);_thread.removeListener(_seen);+ _thread.removeListener(_guardVoice);+ _voice?.dispose();_thread.dispose();_attachNote.dispose();_farBack.dispose();@@ -173,6 +211,17 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {);}+ /// The second tap on the mic ended a take: what is in the box goes as a+ /// reply, as Send would send it, to the chat too with Also send. Nothing+ /// when the box is empty, the thread is covered or no longer the person's+ /// to reply in, or a reply or files are already on their way.+ void _spoken() {+ final t = _thread;+ if (!mounted || !_visible || !_mayReply) return;+ if (t.draft.trim().isEmpty || !t.canSend) return;+ unawaited(_send());+ }+/// The box's + ([attachSwarmFiles]), its files into the thread's.Future<void> _pickFiles() async {if (_picking) return;@@ -287,7 +336,7 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {),),ListenableBuilder(- listenable: Listenable.merge([_thread, _thread.composer]),+ listenable: Listenable.merge([_thread, _thread.composer, _voice]),builder: (context, _) => _foot(tty),),],@@ -500,8 +549,8 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {}/// The reply box: the files going with the reply, a field that grows with- /// what is typed, then Also send, the + and a round Send (the desktop's- /// thread composer).+ /// what is typed, then Also send, the +, the mic and a round Send (the+ /// desktop's thread composer, with the main chat's mic).Widget _box(SwarmThread t, Tty tty) {// A reply on its way never holds the box: the next is typed meanwhile.final canSend = t.canSend;@@ -539,6 +588,35 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {),),),+ if (_voice?.notice case final notice?)+ Padding(+ padding: const EdgeInsets.fromLTRB(+ AppSpace.xs,+ 0,+ AppSpace.xs,+ AppSpace.s,+ ),+ child: Text(+ notice,+ key: const ValueKey('swarm-thread-voice-note'),+ style: AppType.caption(color: tty.yellow),+ ),+ ),+ // Quiet: nothing went wrong, the words stay in the box.+ if (_voice?.stopped case final stopped?)+ Padding(+ padding: const EdgeInsets.fromLTRB(+ AppSpace.xs,+ 0,+ AppSpace.xs,+ AppSpace.s,+ ),+ child: Text(+ stopped,+ key: const ValueKey('swarm-thread-voice-stopped'),+ style: AppType.caption(color: AppPalette.textSecondary),+ ),+ ),SwarmOutgoingChips(box: t),SwarmMessageField(controller: _text,@@ -566,6 +644,8 @@ class _SwarmThreadPageState extends State<SwarmThreadPage> {key: const ValueKey('swarm-thread-attach'),onTap: canAttach ? () => unawaited(_pickFiles()) : null,),+ // The main chat's mic, beside Send as there.+ if (_voice case final voice?) SwarmVoiceMic(voice: voice),SwarmRoundButton(key: const ValueKey('swarm-thread-send'),label: 'Send reply',diff --git a/mobile/test/render/swarm_thread_render_test.dart b/mobile/test/render/swarm_thread_render_test.dartindex 36b7726d0..90cd72735 100644--- a/mobile/test/render/swarm_thread_render_test.dart+++ b/mobile/test/render/swarm_thread_render_test.dart@@ -7,6 +7,7 @@ import 'package:flutter/rendering.dart';import 'package:flutter/services.dart';import 'package:flutter_test/flutter_test.dart';import 'package:harness_mobile/notify/system_notices.dart';+import 'package:harness_mobile/phone/voice_input_controller.dart';import 'package:harness_mobile/settings/experimental_features.dart';import 'package:harness_mobile/shared/theme/app_theme.dart' as grid;import 'package:harness_mobile/shared/theme/color_palette.dart';@@ -303,6 +304,12 @@ void main() {upload: stuck,notices: SilentSystemNotices(),features: features,+ // The app's mic, in the chat's box and the thread's.+ voice: () => VoiceInputController(+ transcriber: FakeTranscriber().call,+ recorder: FakeVoiceRecorder(),+ language: ValueNotifier('en'),+ ),)..foreground = false;addTearDown(host.dispose);final key = GlobalKey();@@ -346,6 +353,19 @@ void main() {await shoot(tester, key, 'swarm-thread-open-$look');});+ testWidgets('the thread, its mic listening ($look)', skip: skip, (+ tester,+ ) async {+ final key = await chat(tester);+ await tester.tap(find.byKey(const ValueKey('swarm-thread-line-m1')));+ await tester.pumpAndSettle();+ await tester.tap(find.byKey(const ValueKey('swarm-mic')));+ for (var i = 0; i < 5; i++) {+ await tester.pump();+ }+ await shoot(tester, key, 'swarm-thread-mic-listening-$look');+ });+Future<void> pick(WidgetTester tester, String row) async {await tester.tap(find.byKey(const ValueKey('swarm-thread-attach')));await tester.pumpAndSettle();diff --git a/mobile/test/swarm/swarm_thread_voice_test.dart b/mobile/test/swarm/swarm_thread_voice_test.dartnew file mode 100644index 000000000..a4458447b--- /dev/null+++ b/mobile/test/swarm/swarm_thread_voice_test.dart@@ -0,0 +1,480 @@+import 'dart:async';++import 'package:flutter/material.dart';+import 'package:flutter_test/flutter_test.dart';+import 'package:harness_mobile/notify/system_notices.dart';+import 'package:harness_mobile/phone/voice_input_controller.dart';+import 'package:harness_mobile/phone/voice_mic_face.dart';+import 'package:harness_mobile/settings/experimental_features.dart';+import 'package:harness_mobile/shared/theme/app_theme.dart' as grid;+import 'package:harness_mobile/swarm/swarm_conversation_page.dart';+import 'package:harness_mobile/swarm/swarm_session.dart';+import 'package:harness_mobile/swarm/swarm_thread_page.dart';++import '../voice_fakes.dart';+import 'swarm_chat_dictation_test.dart' show FakeDictation;+import 'swarm_fixture.dart';++/// Talking instead of typing in a thread (Dee, 11-01: "how do I voice in a+/// thread?"): the main chat's mic, in the thread's reply box beside Send. Tap+/// to talk, tap again and the box goes as a reply — to the chat too with Also+/// send. A take goes, nothing of it written or sent, once the person can no+/// longer reply there. The mic itself: `swarm_chat_voice_test` and+/// `swarm_chat_dictation_test`.+void main() {+ const messages = '/api/super-tabs/$tab/conversations/g1/messages';+ const commands = '/api/super-tabs/$tab/commands';+ const parts = '/api/super-tabs/$tab/parts';++ late FakeSwarm fake;+ late FakeVoiceRecorder recorder;+ late FakeTranscriber stt;+ late ValueNotifier<String?> account;++ Map<String, dynamic> message(String id, int sequence, String text) => {+ 'id': id,+ 'conversationId': 'g1',+ 'text': text,+ 'sequence': sequence,+ 'thread': 'm1',+ if (id != 'm1') 'replyTo': 'm1',+ 'author': {'kind': 'agent', 'id': 'agent-builder', 'name': 'Nova'},+ 'attachments': <Object>[],+ 'reactions': <Object>[],+ 'createdAt': DateTime.utc(2026, 11, 1, 1, sequence).toIso8601String(),+ };++ /// The Swarm's read: chat g1 while [listed], the person in it as [role].+ Map<String, dynamic> read({bool listed = true, String role = 'owner'}) => {+ ...firstLoad(+ groups: [+ if (listed)+ groupData(+ 'g1',+ title: 'Fix login',+ agentIds: ['agent-lead', 'agent-builder'],+ lastMessageSequence: 2,+ ),+ ],+ ),+ 'members': [part(me, 1, person(me, role: role))],+ };++ setUp(() {+ recorder = FakeVoiceRecorder();+ stt = FakeTranscriber();+ fake = FakeSwarm();+ fake.routes['/api/super-tabs/list'] = (_) => {+ 'tabs': [+ {'id': tab, 'name': 'Phone Swarm', 'ownerId': me},+ ],+ 'next': null,+ };+ fake.routes[parts] = (_) => read();+ fake.routes['/api/super-tabs/$tab/changes'] = (_) => changes(cursor: 'c2');+ fake.routes['/api/super-tabs/$tab/decisions?state=open'] = (_) => {+ 'decisions': <Object>[],+ };+ fake.routes['/api/super-tabs/$tab/mentions?state=waiting'] = (_) => {+ 'mentions': <Object>[],+ };+ final rows = [+ message('m1', 1, 'Can you check the drawer mockup?'),+ message('m2', 2, 'Looks good. One question.'),+ ];+ final line = {+ 'id': 'm1',+ 'replies': 1,+ 'lastSequence': 2,+ 'lastAt': DateTime.now().toUtc().toIso8601String(),+ 'authors': [+ {'kind': 'agent', 'id': 'agent-builder', 'name': 'Nova'},+ ],+ 'people': <String>[],+ };+ fake.routes[messages] = (_) => {+ 'messages': rows.reversed.toList(),+ 'next': null,+ 'threads': [line],+ };+ fake.routes['$messages?thread=m1'] = (_) => {+ 'messages': rows.reversed.toList(),+ 'next': null,+ 'threads': [line],+ };+ fake.routes[commands] = (_) => {'result': <String, dynamic>{}};+ });++ /// Chat g1's thread m1 open, its box's mic heard through [recorder] and+ /// [stt] — or the phone's own [dictation], or no voice input at all.+ Future<SwarmHost> pump(+ WidgetTester tester, {+ bool voice = true,+ FakeDictation? dictation,+ }) async {+ final features = ExperimentalFeaturesStore(storage: MemoryKeyValueStore())+ ..value = {PhoneExperiment.swarm};+ account = ValueNotifier<String?>(me);+ final host = SwarmHost(+ accountId: () => account.value,+ account: account,+ request: fake.request,+ notices: SilentSystemNotices(),+ features: features,+ voice: voice+ ? () => VoiceInputController(+ transcriber: stt.call,+ recorder: recorder,+ language: ValueNotifier('en'),+ )+ : null,+ dictation: dictation,+ )..foreground = false;+ addTearDown(host.dispose);+ await tester.pumpWidget(+ MaterialApp(+ theme: grid.buildAppTheme(brightness: Brightness.dark),+ builder: (context, child) => SwarmScope(notifier: host, child: child!),+ home: const SwarmConversationPage(swarmId: tab, conversationId: 'g1'),+ ),+ );+ await tester.pumpAndSettle();+ await tester.tap(find.byKey(const ValueKey('swarm-thread-line-m1')));+ await tester.pumpAndSettle();+ expect(find.byType(SwarmThreadPage), findsOneWidget);+ return host;+ }++ Finder key(String value) => find.byKey(ValueKey(value));++ Finder inBox(Finder finder) =>+ find.descendant(of: key('swarm-thread-box'), matching: finder);++ /// A few frames, for the take's awaits to land. Never settling: the mic's+ /// bars move for as long as it listens.+ Future<void> frames(WidgetTester tester) async {+ for (var i = 0; i < 5; i++) {+ await tester.pump();+ }+ }++ Future<void> tapMic(WidgetTester tester) async {+ await tester.tap(inBox(key('swarm-mic')));+ await frames(tester);+ }++ String box(WidgetTester tester) => tester+ .widget<TextField>(+ find.descendant(+ of: key('swarm-thread-field'),+ matching: find.byType(TextField),+ ),+ )+ .controller!+ .text;++ /// The thread's draft as its chat keeps it, for the next time it opens.+ String kept(WidgetTester tester) => tester+ .widget<SwarmThreadPage>(find.byType(SwarmThreadPage))+ .chat+ .threadDraft('m1');++ /// The Swarm read again: [listed] or not, the person as [role].+ Future<void> reread(+ WidgetTester tester,+ SwarmHost host, {+ bool listed = true,+ String role = 'owner',+ }) async {+ fake.routes[parts] = (_) => read(listed: listed, role: role);+ fake.routes['/api/super-tabs/$tab/changes'] = (_) => {'reload': true};+ await host.session!.inbox.reader(tab)!.refresh();+ await frames(tester);+ }++ testWidgets('the thread\'s reply box has the main chat\'s mic, just left of '+ 'Send', (tester) async {+ await pump(tester);++ expect(inBox(key('swarm-mic')), findsOneWidget);+ final mic = tester.getCenter(inBox(key('swarm-mic')));+ final send = tester.getCenter(key('swarm-thread-send'));+ final attach = tester.getCenter(key('swarm-thread-attach'));+ expect(mic.dy, send.dy, reason: 'in Send\'s row');+ expect(mic.dx, lessThan(send.dx));+ expect(mic.dx, greaterThan(attach.dx));+ });++ testWidgets('with no voice input the reply box has no mic', (tester) async {+ await pump(tester, voice: false);++ expect(key('swarm-mic'), findsNothing);+ expect(key('swarm-thread-send'), findsOneWidget);+ });++ testWidgets('a second tap sends what was said as a reply in the thread, '+ 'not to the chat', (tester) async {+ await pump(tester);+ stt.replies.add('ship the fix today');++ await tapMic(tester);+ expect(recorder.recording, isTrue);+ expect(+ tester.widget<VoiceMicCore>(find.byType(VoiceMicCore)).face,+ VoiceMicFace.listening,+ );++ await tapMic(tester);+ await tester.pumpAndSettle();++ expect(recorder.recording, isFalse);+ expect(fake.commands.single, {+ 'type': 'message.send',+ 'conversationId': 'g1',+ 'text': 'ship the fix today',+ 'replyTo': 'm1',+ 'recipientIds': <String>[],+ });+ expect(box(tester), isEmpty);+ });++ testWidgets('what was typed goes with what is said', (tester) async {+ await pump(tester);+ await tester.enterText(key('swarm-thread-field'), 'Nova:');+ await tester.pump();+ stt.replies.add('ship the fix');++ await tapMic(tester);+ await tapMic(tester);+ await tester.pumpAndSettle();++ expect(fake.commands.single['text'], 'Nova: ship the fix');+ expect(fake.commands.single['replyTo'], 'm1');+ });++ testWidgets('with Also send ticked, a take goes to the chat too', (+ tester,+ ) async {+ await pump(tester);+ await tester.tap(key('swarm-thread-also-send'));+ await tester.pump();+ stt.replies.add('also for everyone');++ await tapMic(tester);+ await tapMic(tester);+ await tester.pumpAndSettle();++ expect(fake.commands.single['text'], 'also for everyone');+ expect(fake.commands.single['replyTo'], 'm1');+ expect(fake.commands.single['alsoSend'], isTrue);+ });++ // After #2471 a reply on its way no longer holds the box: a take that ends+ // meanwhile goes as the next reply, as Send would, and the first keeps its+ // own text.+ testWidgets('a take ending while a reply is still on its way goes as the '+ 'next reply, in order', (tester) async {+ await pump(tester);+ final first = Completer<Object?>();+ fake.routes[commands] = (_) =>+ first.isCompleted ? {'result': <String, dynamic>{}} : first.future;+ await tester.enterText(key('swarm-thread-field'), 'typed reply');+ await tester.pump();+ await tester.tap(key('swarm-thread-send'));+ await frames(tester);+ expect(fake.commands.single['text'], 'typed reply');+ expect(box(tester), isEmpty);++ stt.replies.add('spoken reply');+ await tapMic(tester);+ await tapMic(tester);+ await frames(tester);+ first.complete({'result': <String, dynamic>{}});+ await tester.pumpAndSettle();++ expect(+ [for (final c in fake.commands) (c['text'], c['replyTo'])],+ [('typed reply', 'm1'), ('spoken reply', 'm1')],+ );+ expect(box(tester), isEmpty);+ });++ testWidgets('a take not heard says so in the thread, and sends nothing', (+ tester,+ ) async {+ await pump(tester);++ await tapMic(tester);+ await tapMic(tester);+ await tester.pumpAndSettle();++ expect(fake.commands, isEmpty);+ expect(inBox(key('swarm-thread-voice-note')), findsOneWidget);+ });++ testWidgets('taken out of the chat mid-take: the microphone closes, and '+ 'nothing is transcribed, written or sent', (tester) async {+ final host = await pump(tester);+ stt.replies.add('the secret plan');+ await tapMic(tester);+ expect(recorder.recording, isTrue);++ await reread(tester, host, listed: false);++ expect(find.text('This chat is not open to you.'), findsOneWidget);+ expect(recorder.recording, isFalse);+ expect(recorder.cancels, 1);+ expect(stt.calls, isEmpty, reason: 'nothing leaves the phone');+ await tester.pumpAndSettle();+ expect(fake.commands, isEmpty);+ expect(find.text('the secret plan'), findsNothing);+ });++ testWidgets('words transcribed after access was lost land nowhere', (+ tester,+ ) async {+ final host = await pump(tester);+ stt.pending = Completer<String>();+ await tapMic(tester);+ await tapMic(tester);+ expect(stt.calls, hasLength(1), reason: 'the take is being transcribed');++ await reread(tester, host, listed: false);+ stt.pending!.complete('too late');+ await tester.pumpAndSettle();++ expect(fake.commands, isEmpty);+ expect(find.text('too late'), findsNothing);+ expect(kept(tester), isEmpty, reason: 'not kept for the thread either');+ });++ testWidgets('made read-only mid-take: the take goes with the box, and '+ 'nothing is sent', (tester) async {+ final host = await pump(tester);+ stt.replies.add('one more thing');+ await tapMic(tester);+ expect(recorder.recording, isTrue);++ await reread(tester, host, role: 'viewer');++ expect(key('swarm-thread-box'), findsNothing);+ expect(recorder.recording, isFalse);+ expect(stt.calls, isEmpty);+ await tester.pumpAndSettle();+ expect(fake.commands, isEmpty);+ });++ testWidgets('signed out mid-take: the microphone closes and nothing is '+ 'sent', (tester) async {+ await pump(tester);+ stt.replies.add('after sign out');+ await tapMic(tester);+ expect(recorder.recording, isTrue);++ account.value = null;+ await frames(tester);++ expect(recorder.recording, isFalse);+ expect(stt.calls, isEmpty);+ await tester.pumpAndSettle();+ expect(fake.commands, isEmpty);+ });++ testWidgets('another account signed in mid-take: the microphone closes '+ 'and nothing is sent', (tester) async {+ await pump(tester);+ stt.replies.add('meant for the first account');+ await tapMic(tester);+ expect(recorder.recording, isTrue);++ account.value = 'someone-else';+ await frames(tester);++ expect(recorder.recording, isFalse);+ expect(stt.calls, isEmpty);+ await tester.pumpAndSettle();+ expect(fake.commands, isEmpty);+ });++ testWidgets('covering the thread cancels its recording', (tester) async {+ await pump(tester);+ await tapMic(tester);+ expect(recorder.recording, isTrue);++ final context = tester.element(find.byType(SwarmThreadPage));+ unawaited(+ Navigator.of(context).push(+ MaterialPageRoute<void>(+ builder: (_) => const Scaffold(body: Text('Other screen')),+ ),+ ),+ );+ await tester.pump();+ await tester.pump(const Duration(seconds: 1));++ expect(recorder.recording, isFalse);+ expect(stt.calls, isEmpty);+ expect(fake.commands, isEmpty);+ });++ testWidgets('leaving the thread mid-take closes the microphone', (+ tester,+ ) async {+ await pump(tester);+ await tapMic(tester);+ expect(recorder.recording, isTrue);++ await tester.tap(find.bySemanticsLabel('Back'));+ await tester.pumpAndSettle();++ expect(find.byType(SwarmThreadPage), findsNothing);+ expect(recorder.recording, isFalse);+ expect(stt.calls, isEmpty);+ expect(fake.commands, isEmpty);+ });++ group('with the phone\'s own recognizer', () {+ late FakeDictation dictation;+ setUp(() => dictation = FakeDictation());++ testWidgets('words show in the reply box as they are said; the second '+ 'tap sends them as a reply', (tester) async {+ await pump(tester, dictation: dictation);++ await tapMic(tester);+ dictation.say('looks good to me');+ await frames(tester);+ expect(box(tester), 'looks good to me');+ expect(fake.commands, isEmpty);++ await tapMic(tester);+ await tester.pumpAndSettle();++ expect(dictation.stops, 1);+ expect(fake.commands.single['text'], 'looks good to me');+ expect(fake.commands.single['replyTo'], 'm1');+ expect(box(tester), isEmpty);+ });++ testWidgets('taken out of the chat while listening: the recognizer stops, '+ 'and what it still says lands nowhere', (tester) async {+ final host = await pump(tester, dictation: dictation);+ await tapMic(tester);+ dictation.say('half a');+ await frames(tester);+ final heard = dictation.heard.single;++ await reread(tester, host, listed: false);+ heard('half a secret');+ await tester.pumpAndSettle();++ expect(dictation.cancels, 1);+ expect(dictation.listening, isFalse);+ expect(find.text('This chat is not open to you.'), findsOneWidget);+ expect(find.textContaining('secret'), findsNothing);+ expect(find.textContaining('half a'), findsNothing);+ expect(kept(tester), isEmpty);+ expect(fake.commands, isEmpty);+ });+ });+}