Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

a11y(mobile): VoiceOver can hear the agent and reach Find and a new harness

dee4bf9003923e1
diff --git a/mobile/lib/phone/terminal_page.dart b/mobile/lib/phone/terminal_page.dart
index 623c3f9e3..384f12a6f 100644
--- a/mobile/lib/phone/terminal_page.dart
+++ b/mobile/lib/phone/terminal_page.dart
@@ -12,6 +12,7 @@ import 'package:flutter/services.dart';
import 'package:image_picker/image_picker.dart';
import 'package:lucide_icons_flutter/lucide_icons.dart';
import 'package:flutter/scheduler.dart';
+import 'package:flutter/semantics.dart' show CustomSemanticsAction;
import 'package:xterm/xterm.dart' show Terminal, TerminalKey, TerminalStyle;
import 'package:harness_mobile/logging/app_log.dart';
@@ -1160,6 +1161,19 @@ class _TerminalPageState extends State<TerminalPage>
_closeSearch();
}
+ /// What VoiceOver reads for the terminal: its last few lines with anything on them, oldest first
+ /// — where the agent says what it did and what it wants. Empty before the first frame lands.
+ static String _lastLinesForVoiceOver(Terminal? terminal, {int count = 6}) {
+ if (terminal == null) return '';
+ final buffer = terminal.buffer;
+ final lines = <String>[];
+ for (var y = buffer.height - 1; y >= 0 && lines.length < count; y--) {
+ final text = buffer.lines[y].getText().trim();
+ if (text.isNotEmpty) lines.add(text);
+ }
+ return lines.reversed.join('\n');
+ }
+
/// A new agent, on the machine of the one on screen — the form slides in from the right, the way
/// a swipe left asks for.
///
@@ -1780,6 +1794,27 @@ class _TerminalPageState extends State<TerminalPage>
// and takes no hits; it only stops the Stack from inheriting a height
// that belongs to something being deliberately held still.
const SizedBox.expand(),
+ // ⚠️ **VoiceOver's way in, and the only one.** xterm draws the terminal without a
+ // single semantics node, and VoiceOver keeps one-finger swipes for itself — so without
+ // this, a VoiceOver user could neither hear what the agent last said nor reach Find or a
+ // new harness, which the sideways swipes are the only way to. One node under everything,
+ // painting nothing and taking no touches: the agent's name, the last lines on its screen,
+ // and the two swipes as actions (VoiceOver's rotor, "Actions").
+ Positioned.fill(
+ child: Semantics(
+ container: true,
+ label: '${agent?.displayName ?? widget.agentId}, terminal',
+ value: _lastLinesForVoiceOver(session?.terminal),
+ customSemanticsActions: widget.sideSwipes
+ ? {
+ const CustomSemanticsAction(label: 'Find'): _openSearch,
+ const CustomSemanticsAction(label: 'New harness'): () =>
+ unawaited(_newAgentHere()),
+ }
+ : null,
+ child: const SizedBox.expand(),
+ ),
+ ),
MediaQuery.removePadding(
context: context,
removeBottom: true,
diff --git a/mobile/test/focus_swipes_test.dart b/mobile/test/focus_swipes_test.dart
index 1548a9a0c..e932f4da9 100644
--- a/mobile/test/focus_swipes_test.dart
+++ b/mobile/test/focus_swipes_test.dart
@@ -1,4 +1,5 @@
import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:harness_mobile/auth/auth_session.dart';
import 'package:harness_mobile/core/config.dart';
@@ -146,6 +147,60 @@ void main() {
expect(pushed, hasLength(1));
expect(find.byType(TerminalSearchOverlay), findsNothing);
});
+
+ group('VoiceOver', () {
+ // VoiceOver keeps one-finger swipes for itself, and xterm draws no semantics: the terminal
+ // node is the only way it reaches the agent's words, Find and a new harness.
+ SemanticsNode terminalNode(WidgetTester tester) =>
+ tester.getSemantics(find.bySemanticsLabel(RegExp(r', terminal$')));
+
+ testWidgets('reads the agent\'s name and its last lines', (tester) async {
+ final handle = tester.ensureSemantics();
+ await pumpFocus(tester);
+ final node = terminalNode(tester);
+ expect(node.label, 'a, terminal');
+ expect(node.value, endsWith('output line 399'));
+ expect(node.value.split('\n'), hasLength(6));
+ handle.dispose();
+ });
+
+ testWidgets('Find is an action, and it opens Find', (tester) async {
+ final handle = tester.ensureSemantics();
+ await pumpFocus(tester);
+ final node = terminalNode(tester);
+ final find_ = node.getSemanticsData().customSemanticsActionIds!.map(
+ CustomSemanticsAction.getAction,
+ );
+ expect(find_.map((a) => a!.label), containsAll(['Find', 'New harness']));
+ tester.binding.pipelineOwner.semanticsOwner!.performAction(
+ node.id,
+ SemanticsAction.customAction,
+ CustomSemanticsAction.getIdentifier(
+ const CustomSemanticsAction(label: 'Find'),
+ ),
+ );
+ await tester.pump();
+ await tester.pump(const Duration(milliseconds: 400));
+ expect(find.byType(TerminalSearchOverlay), findsOneWidget);
+ handle.dispose();
+ });
+
+ testWidgets('New harness is an action, and it opens the form', (
+ tester,
+ ) async {
+ final handle = tester.ensureSemantics();
+ final pushed = await pumpFocus(tester);
+ tester.binding.pipelineOwner.semanticsOwner!.performAction(
+ terminalNode(tester).id,
+ SemanticsAction.customAction,
+ CustomSemanticsAction.getIdentifier(
+ const CustomSemanticsAction(label: 'New harness'),
+ ),
+ );
+ expect(pushed, hasLength(1));
+ handle.dispose();
+ });
+ });
}
class _Pushes extends NavigatorObserver {