Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

a11y(mobile): VoiceOver can cancel a take and hears a question open; two flaky tests made steady

dee94496ee22bb2
diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dart
index 751f104bd..0c7d32bdc 100644
--- a/mobile/lib/phone/terminal_page.dart
+++ b/mobile/lib/phone/terminal_page.dart
@@ -12,7 +12,8 @@ import 'package:flutter/services.dart';
import 'package:image_picker/image_picker.dart';
import 'package:lucide_icons_flutter/lucide_icons.dart';
import 'package:flutter/scheduler.dart';
-import 'package:flutter/semantics.dart' show CustomSemanticsAction;
+import 'package:flutter/semantics.dart'
+ show CustomSemanticsAction, SemanticsService;
import 'package:xterm/xterm.dart' show Terminal, TerminalKey, TerminalStyle;
import 'package:harness_mobile/logging/app_log.dart';
@@ -518,6 +519,20 @@ class _TerminalPageState extends State<TerminalPage>
if (!mounted) return;
final view = _questionWatcher?.view;
final open = view != null && view.answerable;
+ // VoiceOver hears it too, once per question: the keys appearing beside the mic and the
+ // terminal repainting say nothing to someone who cannot see them.
+ if (open && view.question != _questionAnnounced) {
+ _questionAnnounced = view.question;
+ unawaited(
+ SemanticsService.sendAnnouncement(
+ View.of(context),
+ 'Asking: ${view.question}',
+ Directionality.of(context),
+ ),
+ );
+ } else if (!open) {
+ _questionAnnounced = null;
+ }
setState(() {
// Cleared as the dialog goes, so the NEXT question raises the keyboard
// again even if it words itself identically.
@@ -527,6 +542,9 @@ class _TerminalPageState extends State<TerminalPage>
_raiseForQuestion();
}
+ /// The question VoiceOver was last told about, so a repaint of the same dialog is not said twice.
+ String? _questionAnnounced;
+
/// Raise the keyboard for the question on the pane, if it has not had its
/// one raise yet — see [_questionRaisedFor] and [_queueRaisedFor].
///
@@ -3434,69 +3452,73 @@ class _StatusLineState extends State<_StatusLine>
decoration: BoxDecoration(color: tty.green, shape: BoxShape.circle),
);
// A band the width of the screen, the output fading out above it as it does under the title —
- // a pill only as wide as its words cut the rows behind it in half.
- return DecoratedBox(
- decoration: BoxDecoration(
- gradient: LinearGradient(
- begin: Alignment.topCenter,
- end: Alignment.bottomCenter,
- colors: [tty.ground.withValues(alpha: 0), tty.ground, tty.ground],
- stops: const [0, 0.4, 1],
- ),
- ),
- child: Padding(
- padding: const EdgeInsets.fromLTRB(
- Tty.origin,
- 14,
- Tty.origin,
- _StatusLine.below,
+ // a pill only as wide as its words cut the rows behind it in half. A live region: what it says
+ // (a take not sent, a question to answer on screen) is read out as it changes.
+ return Semantics(
+ liveRegion: true,
+ child: DecoratedBox(
+ decoration: BoxDecoration(
+ gradient: LinearGradient(
+ begin: Alignment.topCenter,
+ end: Alignment.bottomCenter,
+ colors: [tty.ground.withValues(alpha: 0), tty.ground, tty.ground],
+ stops: const [0, 0.4, 1],
+ ),
),
- child: Row(
- mainAxisAlignment: MainAxisAlignment.center,
- children: [
- if (widget.dot) ...[
- SizedBox(
- width: 30,
- height: 6,
- child: AnimatedBuilder(
- animation: _clock,
- builder: (context, child) {
- // A glide over the first 60% of each beat, easing in and out, then a rest.
- final t = Curves.easeInOut.transform(
- (_clock.value / 0.6).clamp(0.0, 1.0),
- );
- final from = widget.glide < 0 ? 24.0 : 0.0;
- final x = widget.glide == 0
- ? 12.0
- : from + widget.glide * 24 * t;
- final fade = widget.glide == 0 ? 1.0 : 1 - (t * t);
- return Stack(
- children: [
- Positioned(
- left: x,
- top: 0,
- child: Opacity(opacity: fade, child: child),
- ),
- ],
- );
- },
- child: dot,
+ child: Padding(
+ padding: const EdgeInsets.fromLTRB(
+ Tty.origin,
+ 14,
+ Tty.origin,
+ _StatusLine.below,
+ ),
+ child: Row(
+ mainAxisAlignment: MainAxisAlignment.center,
+ children: [
+ if (widget.dot) ...[
+ SizedBox(
+ width: 30,
+ height: 6,
+ child: AnimatedBuilder(
+ animation: _clock,
+ builder: (context, child) {
+ // A glide over the first 60% of each beat, easing in and out, then a rest.
+ final t = Curves.easeInOut.transform(
+ (_clock.value / 0.6).clamp(0.0, 1.0),
+ );
+ final from = widget.glide < 0 ? 24.0 : 0.0;
+ final x = widget.glide == 0
+ ? 12.0
+ : from + widget.glide * 24 * t;
+ final fade = widget.glide == 0 ? 1.0 : 1 - (t * t);
+ return Stack(
+ children: [
+ Positioned(
+ left: x,
+ top: 0,
+ child: Opacity(opacity: fade, child: child),
+ ),
+ ],
+ );
+ },
+ child: dot,
+ ),
),
- ),
- const SizedBox(width: 8),
- ],
- Flexible(
- child: Text(
- widget.text,
- maxLines: 1,
- overflow: TextOverflow.ellipsis,
- style: tty.style(
- size: TtySize.meta,
- color: widget.color ?? tty.faint,
+ const SizedBox(width: 8),
+ ],
+ Flexible(
+ child: Text(
+ widget.text,
+ maxLines: 1,
+ overflow: TextOverflow.ellipsis,
+ style: tty.style(
+ size: TtySize.meta,
+ color: widget.color ?? tty.faint,
+ ),
),
),
- ),
- ],
+ ],
+ ),
),
),
);
diff --git a/mobile/lib/phone/voice_mic_button.dart b/mobile/lib/phone/voice_mic_button.dart
index 3e714f71e..5e014a9df 100644
--- a/mobile/lib/phone/voice_mic_button.dart
+++ b/mobile/lib/phone/voice_mic_button.dart
@@ -2,6 +2,7 @@ import 'dart:async';
import 'package:flutter/foundation.dart' show ValueListenable;
import 'package:flutter/material.dart';
+import 'package:flutter/semantics.dart' show CustomSemanticsAction;
import 'package:flutter/services.dart';
import 'package:harness_mobile/shared/theme/app_theme.dart';
@@ -144,10 +145,9 @@ class _VoiceMicButtonState extends State<VoiceMicButton> {
VoiceMicFace.talk || VoiceMicFace.sent =>
micHoldsToTalk ? 'Hold to talk to the harness' : 'Talk to the harness',
VoiceMicFace.starting => 'Cancel',
- VoiceMicFace.listening =>
- micHoldsToTalk
- ? 'Release to send'
- : 'Listening. Tap to send, swipe down to cancel',
+ // ⚠️ One word while the mic is open: VoiceOver reads the label aloud, and a sentence read
+ // into an open microphone lands in the take. Cancel is an action (below), not an instruction.
+ VoiceMicFace.listening => micHoldsToTalk ? 'Release to send' : 'Send',
VoiceMicFace.cancelling => 'Release to cancel',
VoiceMicFace.busy || VoiceMicFace.sending => 'Working',
VoiceMicFace.retry => 'Send again',
@@ -264,11 +264,18 @@ class _VoiceMicButtonState extends State<VoiceMicButton> {
@override
Widget build(BuildContext context) {
AppTheme.watch(context);
+ // VoiceOver keeps one-finger swipes for itself, so the swipe down that throws a take away
+ // never reached the mic: it is a Cancel action here, and the two-finger scrub (onDismiss).
+ final cancel = _face == VoiceMicFace.listening ? widget.onSwipeDown : null;
return Semantics(
button: true,
enabled: _live,
label: _semanticLabel,
onLongPressHint: widget.onLongPress == null ? null : 'Choose language',
+ onDismiss: cancel,
+ customSemanticsActions: cancel == null
+ ? null
+ : {const CustomSemanticsAction(label: 'Cancel'): cancel},
// ⚠️ **The slot is [VoiceMicButton.extent]; the hit area inside it is the
// larger [VoiceMicButton.touchExtent], spilling out on every side.** The
// [OverflowBox] is what allows a child bigger than its parent without the
diff --git a/mobile/test/session_content_search_test.dart b/mobile/test/session_content_search_test.dart
index 9a60162ff..658d5cf2c 100644
--- a/mobile/test/session_content_search_test.dart
+++ b/mobile/test/session_content_search_test.dart
@@ -84,9 +84,12 @@ void main() {
test('asks every machine once per pause in typing and keeps the best hit per harness', () async {
final asked = <(String, String)>[];
final gate = Completer<void>();
+ // A zero debounce, not 5ms: a timer due now always fires before [settle]'s 30ms one, where
+ // a 5ms one fired after it on a loaded machine (one full run in several). Four searches in
+ // a row still coalesce into one question — the restart happens before either timer is due.
final search = SessionContentSearch(
machines: () => ['m', 'n'],
- debounce: const Duration(milliseconds: 5),
+ debounce: Duration.zero,
ask: (machine, query, _) async {
asked.add((machine, query));
if (machine == 'n') await gate.future;
diff --git a/mobile/test/terminal_session_protocol_test.dart b/mobile/test/terminal_session_protocol_test.dart
index 41b881804..61076410d 100644
--- a/mobile/test/terminal_session_protocol_test.dart
+++ b/mobile/test/terminal_session_protocol_test.dart
@@ -210,24 +210,31 @@ void main() {
},
);
- test('silence after the open redials, resends, then gives up', () async {
- var redials = 0;
- final s = session(
- onOpenStalled: () async {
- redials++;
- return true;
- },
- resync: const Duration(milliseconds: 20),
- );
+ // On the fake clock, like the test below: on the real one a 20ms resync and a 120ms wait
+ // raced a loaded machine, and the resend landed after the check in one full run of several.
+ test('silence after the open redials, resends, then gives up', () {
+ fakeAsync((async) {
+ var redials = 0;
+ final s = session(
+ disposeAtEnd: false,
+ onOpenStalled: () async {
+ redials++;
+ return true;
+ },
+ resync: const Duration(milliseconds: 20),
+ );
- await s.open();
- await Future<void>.delayed(const Duration(milliseconds: 120));
+ s.open();
+ async.flushMicrotasks();
+ async.elapse(const Duration(milliseconds: 120));
- expect(redials, 1, reason: 'one forced redial per open');
- expect(wire.of('terminal_open').length, greaterThanOrEqualTo(2));
- expect(s.status, TerminalSessionStatus.error);
- expect(s.errorCode, 'TERMINAL_RESYNC_TIMEOUT');
- expect(s.errorMessage, contains('did not respond'));
+ expect(redials, 1, reason: 'one forced redial per open');
+ expect(wire.of('terminal_open').length, greaterThanOrEqualTo(2));
+ expect(s.status, TerminalSessionStatus.error);
+ expect(s.errorCode, 'TERMINAL_RESYNC_TIMEOUT');
+ expect(s.errorMessage, contains('did not respond'));
+ s.dispose();
+ });
});
test('a resend that cannot be sent either gives up at once', () {
diff --git a/mobile/test/voice_mic_fab_test.dart b/mobile/test/voice_mic_fab_test.dart
index b298e8f80..09ffc2a74 100644
--- a/mobile/test/voice_mic_fab_test.dart
+++ b/mobile/test/voice_mic_fab_test.dart
@@ -1,4 +1,7 @@
import 'package:flutter/material.dart';
+import 'package:harness_mobile/phone/voice_mic_face.dart';
+import 'package:harness_mobile/phone/voice_mic_button.dart';
+import 'package:flutter/rendering.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:harness_mobile/phone/voice_input_controller.dart';
import 'package:harness_mobile/phone/voice_mic_fab.dart';
@@ -137,4 +140,41 @@ void main() {
expect(recorder.starts, 0);
expect(frames, isEmpty);
});
+
+ testWidgets(
+ 'VoiceOver can cancel a take — the swipe down is VoiceOver\'s own',
+ (tester) async {
+ final handle = tester.ensureSemantics();
+ var cancelled = 0;
+ await tester.pumpWidget(
+ MaterialApp(
+ home: Scaffold(
+ body: Center(
+ child: VoiceMicButton(
+ face: VoiceMicFace.listening,
+ onPressed: () {},
+ onSwipeDown: () => cancelled++,
+ ),
+ ),
+ ),
+ ),
+ );
+ final node = tester.getSemantics(find.byType(VoiceMicButton));
+ // One word while the mic is open: a read-out sentence would land in the take.
+ expect(node.label, 'Send');
+ tester.binding.pipelineOwner.semanticsOwner!.performAction(
+ node.id,
+ SemanticsAction.customAction,
+ CustomSemanticsAction.getIdentifier(
+ const CustomSemanticsAction(label: 'Cancel'),
+ ),
+ );
+ tester.binding.pipelineOwner.semanticsOwner!.performAction(
+ node.id,
+ SemanticsAction.dismiss,
+ );
+ expect(cancelled, 2);
+ handle.dispose();
+ },
+ );
}