Skip to content

A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.

TalkBack: agents, tabs, sheets and the island say their state in words

Claude06cf6f470486
diff --git a/android/app/src/main/kotlin/com/coucou/android/core/SpokenLabels.kt b/android/app/src/main/kotlin/com/coucou/android/core/SpokenLabels.kt
new file mode 100644
index 00000000..9408308a
--- /dev/null
+++ b/android/app/src/main/kotlin/com/coucou/android/core/SpokenLabels.kt
@@ -0,0 +1,32 @@
+package com.coucou.android.core
+
+/**
+ * What TalkBack says for the things that are drawn rather than written (a state dot, a Mochi, a badge), so the state of an agent
+ * is never only a colour. Pure: the words come from the string resources, this only joins them in one fixed shape.
+ */
+object SpokenLabels {
+ private fun lowerFirst(s: String) = s.replaceFirstChar { it.lowercaseChar() }
+
+ /** "Claude Code, waiting for your OK. Editing files": the name, the state in words, then the line when it says something more. */
+ fun agent(name: String, state: String, line: String?): String {
+ val who = name.trim()
+ val st = state.trim()
+ val extra = line?.trim().orEmpty().takeIf { it.isNotEmpty() && !it.equals(st, ignoreCase = true) }
+ val head = when {
+ who.isEmpty() -> st
+ st.isEmpty() -> who
+ else -> "$who, ${lowerFirst(st)}"
+ }
+ return if (extra == null) head else "$head. $extra"
+ }
+
+ /** "Approval needed: Claude Code wants to run a command". [wants] is the sheet's own sentence ("Wants to run a command"). */
+ fun approval(title: String, name: String, wants: String): String {
+ val who = name.trim()
+ val sentence = if (who.isEmpty()) wants.trim() else "$who ${lowerFirst(wants.trim())}"
+ return "${title.trim()}: $sentence"
+ }
+
+ /** A tab that carries the alert dot says why: "Home, a request is waiting". */
+ fun tab(label: String, alert: Boolean, alertText: String): String = if (alert) alertText else label
+}
diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt b/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt
index 501a88a3..81b7da52 100644
--- a/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt
+++ b/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt
@@ -23,10 +23,14 @@ import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.Role
+import androidx.compose.ui.semantics.contentDescription
+import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.unit.dp
import com.coucou.android.R
import com.coucou.android.core.BarFit
import com.coucou.android.core.IconKind
+import com.coucou.android.core.Spacing
+import com.coucou.android.core.SpokenLabels
import com.coucou.android.core.Tab
import com.coucou.android.core.TypeScale
import com.coucou.android.mochi.BotState
@@ -52,6 +56,7 @@ private fun Tab.label() = when (this) {
@Composable
fun BottomBar(tabs: List<Tab>, selected: Tab?, onSelect: (Tab) -> Unit, alert: Boolean, modifier: Modifier = Modifier) {
val t = tokens()
+ val homeAlert = stringResource(R.string.tab_home_alert)
Row(
// At least 64 dp, taller when the font is large, so labels are never cut.
modifier.fillMaxWidth().padding(horizontal = 24.dp).height(IntrinsicSize.Min).heightIn(min = 64.dp)
@@ -60,14 +65,16 @@ fun BottomBar(tabs: List<Tab>, selected: Tab?, onSelect: (Tab) -> Unit, alert: B
) {
for (tab in tabs) {
val on = tab == selected
+ val label = stringResource(tab.label())
Box(
- Modifier.weight(1f).fillMaxHeight().clip(CircleShape).background(if (on) t.panel2.c() else Color.Transparent)
- .selectable(selected = on, role = Role.Tab, onClick = { onSelect(tab) }),
+ Modifier.weight(1f).fillMaxHeight().heightIn(min = Spacing.MIN_TOUCH.dp).clip(CircleShape).background(if (on) t.panel2.c() else Color.Transparent)
+ .selectable(selected = on, role = Role.Tab, onClick = { onSelect(tab) })
+ .semantics { contentDescription = SpokenLabels.tab(label, alert && tab == Tab.HOME, homeAlert) },
contentAlignment = Alignment.Center,
) {
Column(Modifier.padding(horizontal = BarFit.ITEM_SIDE_PADDING_DP.dp), horizontalAlignment = Alignment.CenterHorizontally) {
CoucouIcon(tab.icon(), tint = if (on) t.text.c() else t.textDim.c(), size = 22.dp)
- Text(stringResource(tab.label()), style = TypeScale.LABEL.style(if (on) t.text.c() else t.textDim.c()), maxLines = 1, softWrap = false)
+ Text(label, style = TypeScale.LABEL.style(if (on) t.text.c() else t.textDim.c()), maxLines = 1, softWrap = false)
}
if (alert && tab == Tab.HOME) {
val amber = MochiConst.STATES[BotState.APPROVAL]?.color?.let(::rgbColor) ?: Color(0xFFF5A524)
diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt b/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt
index f919aec8..62ea9b8a 100644
--- a/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt
+++ b/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt
@@ -22,6 +22,7 @@ import androidx.compose.ui.draw.clip
import androidx.compose.ui.graphics.Color
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.Role
+import androidx.compose.ui.semantics.contentDescription
import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
@@ -34,6 +35,7 @@ import com.coucou.android.core.HomeText
import com.coucou.android.core.IconKind
import com.coucou.android.core.IslandSurface
import com.coucou.android.core.Pills
+import com.coucou.android.core.SpokenLabels
import com.coucou.android.core.Spacing
import com.coucou.android.core.TypeScale
import com.coucou.android.link.SessionInfo
@@ -95,7 +97,7 @@ fun HeroCard(
Row(verticalAlignment = Alignment.CenterVertically) {
StateDot(linkDot)
Spacer(Modifier.width(8.dp))
- Text(linkText, Modifier.weight(1f), style = TypeScale.LABEL.style(dim), maxLines = 1, overflow = TextOverflow.Ellipsis)
+ Text(linkText, Modifier.weight(1f), style = TypeScale.LABEL.style(dim), maxLines = 2, overflow = TextOverflow.Ellipsis)
if (running > 0) Text(stringResource(R.string.home_running, running), style = TypeScale.LABEL.style(dim), maxLines = 1)
}
if (focus == null) {
@@ -105,11 +107,15 @@ fun HeroCard(
Text(stringResource(R.string.home_empty_hint), style = TypeScale.SECONDARY.style(dim), maxLines = 3)
}
} else {
- Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(14.dp)) {
+ val pillName = Pills.byId(focus.pillId)?.name.orEmpty()
+ val name = focus.agent.ifBlank { pillName }.ifBlank { stringResource(R.string.app_name) }
+ val spoken = SpokenLabels.agent(name, stringResource(stateLabel(state)), HomeText.line(focus))
+ Row(
+ Modifier.semantics(mergeDescendants = true) { contentDescription = spoken },
+ verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(14.dp),
+ ) {
MochiView(engine, Modifier.size(108.dp).then(touch))
Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(2.dp)) {
- val pillName = Pills.byId(focus.pillId)?.name.orEmpty()
- val name = focus.agent.ifBlank { pillName }.ifBlank { stringResource(R.string.app_name) }
Text(name, style = TypeScale.TITLE.style(white), maxLines = 2, overflow = TextOverflow.Ellipsis)
Text(
if (pillName.isNotBlank() && pillName != name) pillName else stringResource(R.string.agent_kind),
@@ -152,15 +158,18 @@ fun AgentRows(others: List<SessionInfo>, engineFor: (SessionInfo) -> MochiEngine
if (i > 0) RowDivider()
val name = s.agent.ifBlank { Pills.byId(s.pillId)?.name.orEmpty() }
val state = stringResource(stateLabel(s.state))
+ val line = HomeText.line(s)
+ // The dot is only a colour: the state is said in words, with the line when it adds something.
+ val spoken = SpokenLabels.agent(name, state, line)
Row(
Modifier.fillMaxWidth().heightIn(min = 64.dp).clickable(role = Role.Button) { onPick(s) }
- .padding(horizontal = Spacing.INSIDE.dp, vertical = 12.dp).semantics(mergeDescendants = true) {},
+ .padding(horizontal = Spacing.INSIDE.dp, vertical = 12.dp).semantics(mergeDescendants = true) { contentDescription = spoken },
horizontalArrangement = Arrangement.spacedBy(Spacing.GAP.dp), verticalAlignment = Alignment.CenterVertically,
) {
MochiView(engineFor(s), Modifier.size(40.dp))
Column(Modifier.weight(1f)) {
Text(name, style = TypeScale.BODY.style(t.text.c()).copy(fontWeight = FontWeight.SemiBold), maxLines = 1, overflow = TextOverflow.Ellipsis)
- Text(HomeText.line(s) ?: state, style = TypeScale.SECONDARY.style(t.textDim.c()), maxLines = 1, overflow = TextOverflow.Ellipsis)
+ Text(line ?: state, style = TypeScale.SECONDARY.style(t.textDim.c()), maxLines = 2, overflow = TextOverflow.Ellipsis)
}
StateDot(stateColor(s.state))
}
@@ -200,7 +209,7 @@ fun RecentPanel(decisions: List<Decision>, onAll: () -> Unit) {
horizontalArrangement = Arrangement.spacedBy(Spacing.GAP.dp), verticalAlignment = Alignment.CenterVertically,
) {
StateDot(if (d.allowed) StatusColors.online else t.danger.c())
- Text(HomeText.recent(d.allowed, d.tool), Modifier.weight(1f), style = TypeScale.BODY.style(t.text.c()), maxLines = 1, overflow = TextOverflow.Ellipsis)
+ Text(HomeText.recent(d.allowed, d.tool), Modifier.weight(1f), style = TypeScale.BODY.style(t.text.c()), maxLines = 2, overflow = TextOverflow.Ellipsis)
Text(whenText, style = TypeScale.SECONDARY.style(t.textDim.c()), maxLines = 1)
}
}
diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt b/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt
index 7b8e9703..67c09d23 100644
--- a/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt
+++ b/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt
@@ -62,6 +62,9 @@ import androidx.compose.ui.res.stringResource
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.font.FontWeight
import androidx.compose.ui.text.style.TextOverflow
+import androidx.compose.ui.semantics.LiveRegionMode
+import androidx.compose.ui.semantics.liveRegion
+import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.unit.Dp
import androidx.compose.ui.unit.dp
import androidx.compose.ui.unit.sp
@@ -319,7 +322,11 @@ class IslandOverlay(
Modifier.fillMaxWidth().padding(top = topInset).alpha(open.coerceIn(0.0, 1.0).toFloat()),
contentAlignment = Alignment.TopCenter,
) {
- Box(Modifier.requiredWidth(px(targetW.toDouble())).wrapContentHeight(Alignment.Top, unbounded = true)) {
+ // TalkBack reads what the island shows when it appears: a request at once, the rest when it is quiet.
+ Box(
+ Modifier.requiredWidth(px(targetW.toDouble())).wrapContentHeight(Alignment.Top, unbounded = true)
+ .semantics { liveRegion = if (s.kind == IslandSpec.Kind.APPROVAL) LiveRegionMode.Assertive else LiveRegionMode.Polite },
+ ) {
when (s.kind) {
IslandSpec.Kind.WORKING -> WorkingStrip(s)
IslandSpec.Kind.APPROVAL -> RequestCard(s)
diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt b/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt
index 42fc2501..bb9eb2b1 100644
--- a/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt
+++ b/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt
@@ -43,6 +43,10 @@ import androidx.compose.ui.graphics.Color
import androidx.compose.ui.input.pointer.pointerInput
import androidx.compose.ui.res.stringResource
import androidx.compose.ui.semantics.Role
+import androidx.compose.ui.semantics.contentDescription
+import androidx.compose.ui.semantics.paneTitle
+import androidx.compose.ui.semantics.role
+import androidx.compose.ui.semantics.semantics
import androidx.compose.ui.text.font.FontFamily
import androidx.compose.ui.text.style.TextDecoration
import androidx.compose.ui.unit.dp
@@ -59,6 +63,7 @@ import com.coucou.android.link.AskedOption
import com.coucou.android.link.QuestionRequest
import com.coucou.android.core.Radii
import com.coucou.android.core.Spacing
+import com.coucou.android.core.SpokenLabels
import com.coucou.android.core.TypeScale
import com.coucou.android.link.ApprovalRequest
import com.coucou.android.mochi.BotState
@@ -73,15 +78,17 @@ import kotlinx.coroutines.delay
* panel itself never reach the scrim.
*/
@Composable
-fun BottomSheetHost(onDismiss: () -> Unit, wash: Color? = null, content: @Composable ColumnScope.() -> Unit) {
+fun BottomSheetHost(onDismiss: () -> Unit, wash: Color? = null, announce: String? = null, content: @Composable ColumnScope.() -> Unit) {
BackHandler(onBack = onDismiss)
+ val closeLabel = stringResource(R.string.action_close)
var shown by remember { mutableStateOf(false) }
LaunchedEffect(Unit) { shown = true }
val shape = RoundedCornerShape(topStart = Radii.SHEET.dp, topEnd = Radii.SHEET.dp)
Box(Modifier.fillMaxSize()) {
Box(
Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.55f))
- .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null, onClick = onDismiss),
+ .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null, onClick = onDismiss)
+ .semantics { contentDescription = closeLabel; role = Role.Button },
)
AnimatedVisibility(
visible = shown, modifier = Modifier.align(Alignment.BottomCenter),
@@ -91,6 +98,10 @@ fun BottomSheetHost(onDismiss: () -> Unit, wash: Color? = null, content: @Compos
Modifier.fillMaxWidth().clip(shape).background(Color(IslandSurface.PANEL)).border(1.dp, Color.White.copy(alpha = 0.08f), shape)
.then(if (wash != null) Modifier.drawBehind { drawWash(wash, 0.45f) } else Modifier)
.pointerInput(Unit) { detectTapGestures { } }
+ // A large font can make the sheet taller than the screen: it scrolls, so Deny and Allow are always reachable.
+ .verticalScroll(rememberScrollState())
+ // TalkBack announces the sheet when it rises, so a request is heard without searching for it.
+ .semantics { if (announce != null) paneTitle = announce }
.padding(start = 20.dp, end = 20.dp, top = 10.dp, bottom = 20.dp),
content = {
Box(Modifier.align(Alignment.CenterHorizontally).size(width = 36.dp, height = 4.dp).clip(RoundedCornerShape(2.dp)).background(Color.White.copy(alpha = 0.2f)))
@@ -128,11 +139,12 @@ fun ApprovalSheet(model: AppModel, r: ApprovalRequest, onAllow: (ApprovalRequest
val wash = MochiConst.STATES[BotState.APPROVAL]?.color?.let(::rgbColor)
val name = model.agentName(r.pillId)
val kind = Pills.byId(r.pillId)?.name.orEmpty()
- BottomSheetHost(onDismiss, wash) {
+ val announce = SpokenLabels.approval(stringResource(R.string.approval_title), name, stringResource(R.string.approval_wants, ApprovalSheetPlan.action(r.tool)))
+ BottomSheetHost(onDismiss, wash, announce) {
Row(horizontalArrangement = Arrangement.spacedBy(14.dp), verticalAlignment = Alignment.CenterVertically) {
SheetMochi(BotState.APPROVAL, agentColour(model, r.pillId))
Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(2.dp)) {
- Text(if (kind.isNotBlank() && kind != name) "$name · $kind" else name, style = TypeScale.SECONDARY.style(dim), maxLines = 1)
+ Text(if (kind.isNotBlank() && kind != name) "$name · $kind" else name, style = TypeScale.SECONDARY.style(dim), maxLines = 2)
Text(stringResource(R.string.approval_wants, ApprovalSheetPlan.action(r.tool)), style = TypeScale.TITLE.style(white), maxLines = 3)
}
}
@@ -190,7 +202,8 @@ fun QuestionSheet(model: AppModel, pillId: String, onDismiss: () -> Unit, onSend
val dim = Color(IslandSurface.TEXT_DIM)
val wash = MochiConst.STATES[BotState.QUESTION]?.color?.let(::rgbColor)
val request = model.questionFor(pillId)
- BottomSheetHost(onDismiss, wash) {
+ val announce = stringResource(R.string.question_title, model.agentName(pillId))
+ BottomSheetHost(onDismiss, wash, announce) {
if (request == null) {
Row(horizontalArrangement = Arrangement.spacedBy(14.dp), verticalAlignment = Alignment.CenterVertically) {
SheetMochi(BotState.QUESTION, agentColour(model, pillId))
diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xml
index c265f533..16372f2a 100644
--- a/android/app/src/main/res/values/strings.xml
+++ b/android/app/src/main/res/values/strings.xml
@@ -136,6 +136,7 @@
<string name="tab_home">Home</string>
<string name="tab_chat">Chat</string>
<string name="tab_settings">Settings</string>
+ <string name="tab_home_alert">Home, a request is waiting</string>
<string name="design_note">The building blocks of the new look. No other screen uses them yet.</string>
<string name="pair_clipboard">Paste from clipboard</string>
<string name="notif_unlock">Unlock to see the request</string>
diff --git a/android/app/src/test/kotlin/com/coucou/android/SpokenLabelsTest.kt b/android/app/src/test/kotlin/com/coucou/android/SpokenLabelsTest.kt
new file mode 100644
index 00000000..c9e1a16c
--- /dev/null
+++ b/android/app/src/test/kotlin/com/coucou/android/SpokenLabelsTest.kt
@@ -0,0 +1,34 @@
+package com.coucou.android
+
+import com.coucou.android.core.SpokenLabels
+import org.junit.Assert.assertEquals
+import org.junit.Test
+
+class SpokenLabelsTest {
+ @Test fun anAgentIsSpokenWithItsStateInWords() {
+ assertEquals("Claude Code, waiting for your OK", SpokenLabels.agent("Claude Code", "Waiting for your OK", null))
+ assertEquals("Claude Code, working. Editing files", SpokenLabels.agent("Claude Code", "Working", "Editing files"))
+ }
+
+ @Test fun aLineThatRepeatsTheStateIsNotSaidTwice() {
+ assertEquals("Codex, working", SpokenLabels.agent("Codex", "Working", "Working"))
+ assertEquals("Codex, working", SpokenLabels.agent("Codex", "Working", "working"))
+ assertEquals("Codex, working", SpokenLabels.agent("Codex", "Working", " "))
+ }
+
+ @Test fun aMissingNameOrStateStillGivesASentence() {
+ assertEquals("Waiting for your OK", SpokenLabels.agent("", "Waiting for your OK", null))
+ assertEquals("Claude Code", SpokenLabels.agent("Claude Code", "", null))
+ assertEquals("", SpokenLabels.agent("", "", null))
+ }
+
+ @Test fun anApprovalNamesTheAgentAndTheAction() {
+ assertEquals("Approval needed: Claude Code wants to run a command", SpokenLabels.approval("Approval needed", "Claude Code", "Wants to run a command"))
+ assertEquals("Approval needed: Wants to run a command", SpokenLabels.approval("Approval needed", " ", "Wants to run a command"))
+ }
+
+ @Test fun aTabWithTheAlertDotSaysWhy() {
+ assertEquals("Home", SpokenLabels.tab("Home", false, "Home, a request is waiting"))
+ assertEquals("Home, a request is waiting", SpokenLabels.tab("Home", true, "Home, a request is waiting"))
+ }
+}
diff --git a/android/i18n/app-strings.json b/android/i18n/app-strings.json
index 5e11ec13..d1109a96 100644
--- a/android/i18n/app-strings.json
+++ b/android/i18n/app-strings.json
@@ -136,6 +136,7 @@
"tab_home": "Home",
"tab_chat": "Chat",
"tab_settings": "Settings",
+ "tab_home_alert": "Home, a request is waiting",
"design_note": "The building blocks of the new look. No other screen uses them yet.",
"pair_clipboard": "Paste from clipboard",
"notif_unlock": "Unlock to see the request",