A tiny friend in your Mac's notch and on your iPhone that keeps an eye on your AI coding agents: Claude Code, Codex, Cursor, Gemini CLI, Antigravity and more. Approve from the notch or your Lock Screen.
Swift ai-agentsanthropicantigravityclaudeclaude-codecodexcursordynamic-islandgemini-cliiosiphonlinuxmacosmacos-appmenubar-appnotchopen-sourceswiftswiftuiwindows
TalkBack: agents, tabs, sheets and the island say their state in words
Claude06cf6f470486
diff --git a/android/app/src/main/kotlin/com/coucou/android/core/SpokenLabels.kt b/android/app/src/main/kotlin/com/coucou/android/core/SpokenLabels.ktnew file mode 100644index 00000000..9408308a--- /dev/null+++ b/android/app/src/main/kotlin/com/coucou/android/core/SpokenLabels.kt@@ -0,0 +1,32 @@+package com.coucou.android.core++/**+ * What TalkBack says for the things that are drawn rather than written (a state dot, a Mochi, a badge), so the state of an agent+ * is never only a colour. Pure: the words come from the string resources, this only joins them in one fixed shape.+ */+object SpokenLabels {+ private fun lowerFirst(s: String) = s.replaceFirstChar { it.lowercaseChar() }++ /** "Claude Code, waiting for your OK. Editing files": the name, the state in words, then the line when it says something more. */+ fun agent(name: String, state: String, line: String?): String {+ val who = name.trim()+ val st = state.trim()+ val extra = line?.trim().orEmpty().takeIf { it.isNotEmpty() && !it.equals(st, ignoreCase = true) }+ val head = when {+ who.isEmpty() -> st+ st.isEmpty() -> who+ else -> "$who, ${lowerFirst(st)}"+ }+ return if (extra == null) head else "$head. $extra"+ }++ /** "Approval needed: Claude Code wants to run a command". [wants] is the sheet's own sentence ("Wants to run a command"). */+ fun approval(title: String, name: String, wants: String): String {+ val who = name.trim()+ val sentence = if (who.isEmpty()) wants.trim() else "$who ${lowerFirst(wants.trim())}"+ return "${title.trim()}: $sentence"+ }++ /** A tab that carries the alert dot says why: "Home, a request is waiting". */+ fun tab(label: String, alert: Boolean, alertText: String): String = if (alert) alertText else label+}diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt b/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.ktindex 501a88a3..81b7da52 100644--- a/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt+++ b/android/app/src/main/kotlin/com/coucou/android/ui/BottomBar.kt@@ -23,10 +23,14 @@ import androidx.compose.ui.draw.clipimport androidx.compose.ui.graphics.Colorimport androidx.compose.ui.res.stringResourceimport androidx.compose.ui.semantics.Role+import androidx.compose.ui.semantics.contentDescription+import androidx.compose.ui.semantics.semanticsimport androidx.compose.ui.unit.dpimport com.coucou.android.Rimport com.coucou.android.core.BarFitimport com.coucou.android.core.IconKind+import com.coucou.android.core.Spacing+import com.coucou.android.core.SpokenLabelsimport com.coucou.android.core.Tabimport com.coucou.android.core.TypeScaleimport com.coucou.android.mochi.BotState@@ -52,6 +56,7 @@ private fun Tab.label() = when (this) {@Composablefun BottomBar(tabs: List<Tab>, selected: Tab?, onSelect: (Tab) -> Unit, alert: Boolean, modifier: Modifier = Modifier) {val t = tokens()+ val homeAlert = stringResource(R.string.tab_home_alert)Row(// At least 64 dp, taller when the font is large, so labels are never cut.modifier.fillMaxWidth().padding(horizontal = 24.dp).height(IntrinsicSize.Min).heightIn(min = 64.dp)@@ -60,14 +65,16 @@ fun BottomBar(tabs: List<Tab>, selected: Tab?, onSelect: (Tab) -> Unit, alert: B) {for (tab in tabs) {val on = tab == selected+ val label = stringResource(tab.label())Box(- Modifier.weight(1f).fillMaxHeight().clip(CircleShape).background(if (on) t.panel2.c() else Color.Transparent)- .selectable(selected = on, role = Role.Tab, onClick = { onSelect(tab) }),+ Modifier.weight(1f).fillMaxHeight().heightIn(min = Spacing.MIN_TOUCH.dp).clip(CircleShape).background(if (on) t.panel2.c() else Color.Transparent)+ .selectable(selected = on, role = Role.Tab, onClick = { onSelect(tab) })+ .semantics { contentDescription = SpokenLabels.tab(label, alert && tab == Tab.HOME, homeAlert) },contentAlignment = Alignment.Center,) {Column(Modifier.padding(horizontal = BarFit.ITEM_SIDE_PADDING_DP.dp), horizontalAlignment = Alignment.CenterHorizontally) {CoucouIcon(tab.icon(), tint = if (on) t.text.c() else t.textDim.c(), size = 22.dp)- Text(stringResource(tab.label()), style = TypeScale.LABEL.style(if (on) t.text.c() else t.textDim.c()), maxLines = 1, softWrap = false)+ Text(label, style = TypeScale.LABEL.style(if (on) t.text.c() else t.textDim.c()), maxLines = 1, softWrap = false)}if (alert && tab == Tab.HOME) {val amber = MochiConst.STATES[BotState.APPROVAL]?.color?.let(::rgbColor) ?: Color(0xFFF5A524)diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt b/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.ktindex f919aec8..62ea9b8a 100644--- a/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt+++ b/android/app/src/main/kotlin/com/coucou/android/ui/HomePanel.kt@@ -22,6 +22,7 @@ import androidx.compose.ui.draw.clipimport androidx.compose.ui.graphics.Colorimport androidx.compose.ui.res.stringResourceimport androidx.compose.ui.semantics.Role+import androidx.compose.ui.semantics.contentDescriptionimport androidx.compose.ui.semantics.semanticsimport androidx.compose.ui.text.font.FontWeightimport androidx.compose.ui.text.style.TextOverflow@@ -34,6 +35,7 @@ import com.coucou.android.core.HomeTextimport com.coucou.android.core.IconKindimport com.coucou.android.core.IslandSurfaceimport com.coucou.android.core.Pills+import com.coucou.android.core.SpokenLabelsimport com.coucou.android.core.Spacingimport com.coucou.android.core.TypeScaleimport com.coucou.android.link.SessionInfo@@ -95,7 +97,7 @@ fun HeroCard(Row(verticalAlignment = Alignment.CenterVertically) {StateDot(linkDot)Spacer(Modifier.width(8.dp))- Text(linkText, Modifier.weight(1f), style = TypeScale.LABEL.style(dim), maxLines = 1, overflow = TextOverflow.Ellipsis)+ Text(linkText, Modifier.weight(1f), style = TypeScale.LABEL.style(dim), maxLines = 2, overflow = TextOverflow.Ellipsis)if (running > 0) Text(stringResource(R.string.home_running, running), style = TypeScale.LABEL.style(dim), maxLines = 1)}if (focus == null) {@@ -105,11 +107,15 @@ fun HeroCard(Text(stringResource(R.string.home_empty_hint), style = TypeScale.SECONDARY.style(dim), maxLines = 3)}} else {- Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(14.dp)) {+ val pillName = Pills.byId(focus.pillId)?.name.orEmpty()+ val name = focus.agent.ifBlank { pillName }.ifBlank { stringResource(R.string.app_name) }+ val spoken = SpokenLabels.agent(name, stringResource(stateLabel(state)), HomeText.line(focus))+ Row(+ Modifier.semantics(mergeDescendants = true) { contentDescription = spoken },+ verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(14.dp),+ ) {MochiView(engine, Modifier.size(108.dp).then(touch))Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(2.dp)) {- val pillName = Pills.byId(focus.pillId)?.name.orEmpty()- val name = focus.agent.ifBlank { pillName }.ifBlank { stringResource(R.string.app_name) }Text(name, style = TypeScale.TITLE.style(white), maxLines = 2, overflow = TextOverflow.Ellipsis)Text(if (pillName.isNotBlank() && pillName != name) pillName else stringResource(R.string.agent_kind),@@ -152,15 +158,18 @@ fun AgentRows(others: List<SessionInfo>, engineFor: (SessionInfo) -> MochiEngineif (i > 0) RowDivider()val name = s.agent.ifBlank { Pills.byId(s.pillId)?.name.orEmpty() }val state = stringResource(stateLabel(s.state))+ val line = HomeText.line(s)+ // The dot is only a colour: the state is said in words, with the line when it adds something.+ val spoken = SpokenLabels.agent(name, state, line)Row(Modifier.fillMaxWidth().heightIn(min = 64.dp).clickable(role = Role.Button) { onPick(s) }- .padding(horizontal = Spacing.INSIDE.dp, vertical = 12.dp).semantics(mergeDescendants = true) {},+ .padding(horizontal = Spacing.INSIDE.dp, vertical = 12.dp).semantics(mergeDescendants = true) { contentDescription = spoken },horizontalArrangement = Arrangement.spacedBy(Spacing.GAP.dp), verticalAlignment = Alignment.CenterVertically,) {MochiView(engineFor(s), Modifier.size(40.dp))Column(Modifier.weight(1f)) {Text(name, style = TypeScale.BODY.style(t.text.c()).copy(fontWeight = FontWeight.SemiBold), maxLines = 1, overflow = TextOverflow.Ellipsis)- Text(HomeText.line(s) ?: state, style = TypeScale.SECONDARY.style(t.textDim.c()), maxLines = 1, overflow = TextOverflow.Ellipsis)+ Text(line ?: state, style = TypeScale.SECONDARY.style(t.textDim.c()), maxLines = 2, overflow = TextOverflow.Ellipsis)}StateDot(stateColor(s.state))}@@ -200,7 +209,7 @@ fun RecentPanel(decisions: List<Decision>, onAll: () -> Unit) {horizontalArrangement = Arrangement.spacedBy(Spacing.GAP.dp), verticalAlignment = Alignment.CenterVertically,) {StateDot(if (d.allowed) StatusColors.online else t.danger.c())- Text(HomeText.recent(d.allowed, d.tool), Modifier.weight(1f), style = TypeScale.BODY.style(t.text.c()), maxLines = 1, overflow = TextOverflow.Ellipsis)+ Text(HomeText.recent(d.allowed, d.tool), Modifier.weight(1f), style = TypeScale.BODY.style(t.text.c()), maxLines = 2, overflow = TextOverflow.Ellipsis)Text(whenText, style = TypeScale.SECONDARY.style(t.textDim.c()), maxLines = 1)}}diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt b/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.ktindex 7b8e9703..67c09d23 100644--- a/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt+++ b/android/app/src/main/kotlin/com/coucou/android/ui/IslandOverlay.kt@@ -62,6 +62,9 @@ import androidx.compose.ui.res.stringResourceimport androidx.compose.ui.text.font.FontFamilyimport androidx.compose.ui.text.font.FontWeightimport androidx.compose.ui.text.style.TextOverflow+import androidx.compose.ui.semantics.LiveRegionMode+import androidx.compose.ui.semantics.liveRegion+import androidx.compose.ui.semantics.semanticsimport androidx.compose.ui.unit.Dpimport androidx.compose.ui.unit.dpimport androidx.compose.ui.unit.sp@@ -319,7 +322,11 @@ class IslandOverlay(Modifier.fillMaxWidth().padding(top = topInset).alpha(open.coerceIn(0.0, 1.0).toFloat()),contentAlignment = Alignment.TopCenter,) {- Box(Modifier.requiredWidth(px(targetW.toDouble())).wrapContentHeight(Alignment.Top, unbounded = true)) {+ // TalkBack reads what the island shows when it appears: a request at once, the rest when it is quiet.+ Box(+ Modifier.requiredWidth(px(targetW.toDouble())).wrapContentHeight(Alignment.Top, unbounded = true)+ .semantics { liveRegion = if (s.kind == IslandSpec.Kind.APPROVAL) LiveRegionMode.Assertive else LiveRegionMode.Polite },+ ) {when (s.kind) {IslandSpec.Kind.WORKING -> WorkingStrip(s)IslandSpec.Kind.APPROVAL -> RequestCard(s)diff --git a/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt b/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.ktindex 42fc2501..bb9eb2b1 100644--- a/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt+++ b/android/app/src/main/kotlin/com/coucou/android/ui/Sheets.kt@@ -43,6 +43,10 @@ import androidx.compose.ui.graphics.Colorimport androidx.compose.ui.input.pointer.pointerInputimport androidx.compose.ui.res.stringResourceimport androidx.compose.ui.semantics.Role+import androidx.compose.ui.semantics.contentDescription+import androidx.compose.ui.semantics.paneTitle+import androidx.compose.ui.semantics.role+import androidx.compose.ui.semantics.semanticsimport androidx.compose.ui.text.font.FontFamilyimport androidx.compose.ui.text.style.TextDecorationimport androidx.compose.ui.unit.dp@@ -59,6 +63,7 @@ import com.coucou.android.link.AskedOptionimport com.coucou.android.link.QuestionRequestimport com.coucou.android.core.Radiiimport com.coucou.android.core.Spacing+import com.coucou.android.core.SpokenLabelsimport com.coucou.android.core.TypeScaleimport com.coucou.android.link.ApprovalRequestimport com.coucou.android.mochi.BotState@@ -73,15 +78,17 @@ import kotlinx.coroutines.delay* panel itself never reach the scrim.*/@Composable-fun BottomSheetHost(onDismiss: () -> Unit, wash: Color? = null, content: @Composable ColumnScope.() -> Unit) {+fun BottomSheetHost(onDismiss: () -> Unit, wash: Color? = null, announce: String? = null, content: @Composable ColumnScope.() -> Unit) {BackHandler(onBack = onDismiss)+ val closeLabel = stringResource(R.string.action_close)var shown by remember { mutableStateOf(false) }LaunchedEffect(Unit) { shown = true }val shape = RoundedCornerShape(topStart = Radii.SHEET.dp, topEnd = Radii.SHEET.dp)Box(Modifier.fillMaxSize()) {Box(Modifier.fillMaxSize().background(Color.Black.copy(alpha = 0.55f))- .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null, onClick = onDismiss),+ .clickable(interactionSource = remember { MutableInteractionSource() }, indication = null, onClick = onDismiss)+ .semantics { contentDescription = closeLabel; role = Role.Button },)AnimatedVisibility(visible = shown, modifier = Modifier.align(Alignment.BottomCenter),@@ -91,6 +98,10 @@ fun BottomSheetHost(onDismiss: () -> Unit, wash: Color? = null, content: @ComposModifier.fillMaxWidth().clip(shape).background(Color(IslandSurface.PANEL)).border(1.dp, Color.White.copy(alpha = 0.08f), shape).then(if (wash != null) Modifier.drawBehind { drawWash(wash, 0.45f) } else Modifier).pointerInput(Unit) { detectTapGestures { } }+ // A large font can make the sheet taller than the screen: it scrolls, so Deny and Allow are always reachable.+ .verticalScroll(rememberScrollState())+ // TalkBack announces the sheet when it rises, so a request is heard without searching for it.+ .semantics { if (announce != null) paneTitle = announce }.padding(start = 20.dp, end = 20.dp, top = 10.dp, bottom = 20.dp),content = {Box(Modifier.align(Alignment.CenterHorizontally).size(width = 36.dp, height = 4.dp).clip(RoundedCornerShape(2.dp)).background(Color.White.copy(alpha = 0.2f)))@@ -128,11 +139,12 @@ fun ApprovalSheet(model: AppModel, r: ApprovalRequest, onAllow: (ApprovalRequestval wash = MochiConst.STATES[BotState.APPROVAL]?.color?.let(::rgbColor)val name = model.agentName(r.pillId)val kind = Pills.byId(r.pillId)?.name.orEmpty()- BottomSheetHost(onDismiss, wash) {+ val announce = SpokenLabels.approval(stringResource(R.string.approval_title), name, stringResource(R.string.approval_wants, ApprovalSheetPlan.action(r.tool)))+ BottomSheetHost(onDismiss, wash, announce) {Row(horizontalArrangement = Arrangement.spacedBy(14.dp), verticalAlignment = Alignment.CenterVertically) {SheetMochi(BotState.APPROVAL, agentColour(model, r.pillId))Column(Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(2.dp)) {- Text(if (kind.isNotBlank() && kind != name) "$name · $kind" else name, style = TypeScale.SECONDARY.style(dim), maxLines = 1)+ Text(if (kind.isNotBlank() && kind != name) "$name · $kind" else name, style = TypeScale.SECONDARY.style(dim), maxLines = 2)Text(stringResource(R.string.approval_wants, ApprovalSheetPlan.action(r.tool)), style = TypeScale.TITLE.style(white), maxLines = 3)}}@@ -190,7 +202,8 @@ fun QuestionSheet(model: AppModel, pillId: String, onDismiss: () -> Unit, onSendval dim = Color(IslandSurface.TEXT_DIM)val wash = MochiConst.STATES[BotState.QUESTION]?.color?.let(::rgbColor)val request = model.questionFor(pillId)- BottomSheetHost(onDismiss, wash) {+ val announce = stringResource(R.string.question_title, model.agentName(pillId))+ BottomSheetHost(onDismiss, wash, announce) {if (request == null) {Row(horizontalArrangement = Arrangement.spacedBy(14.dp), verticalAlignment = Alignment.CenterVertically) {SheetMochi(BotState.QUESTION, agentColour(model, pillId))diff --git a/android/app/src/main/res/values/strings.xml b/android/app/src/main/res/values/strings.xmlindex c265f533..16372f2a 100644--- a/android/app/src/main/res/values/strings.xml+++ b/android/app/src/main/res/values/strings.xml@@ -136,6 +136,7 @@<string name="tab_home">Home</string><string name="tab_chat">Chat</string><string name="tab_settings">Settings</string>+ <string name="tab_home_alert">Home, a request is waiting</string><string name="design_note">The building blocks of the new look. No other screen uses them yet.</string><string name="pair_clipboard">Paste from clipboard</string><string name="notif_unlock">Unlock to see the request</string>diff --git a/android/app/src/test/kotlin/com/coucou/android/SpokenLabelsTest.kt b/android/app/src/test/kotlin/com/coucou/android/SpokenLabelsTest.ktnew file mode 100644index 00000000..c9e1a16c--- /dev/null+++ b/android/app/src/test/kotlin/com/coucou/android/SpokenLabelsTest.kt@@ -0,0 +1,34 @@+package com.coucou.android++import com.coucou.android.core.SpokenLabels+import org.junit.Assert.assertEquals+import org.junit.Test++class SpokenLabelsTest {+ @Test fun anAgentIsSpokenWithItsStateInWords() {+ assertEquals("Claude Code, waiting for your OK", SpokenLabels.agent("Claude Code", "Waiting for your OK", null))+ assertEquals("Claude Code, working. Editing files", SpokenLabels.agent("Claude Code", "Working", "Editing files"))+ }++ @Test fun aLineThatRepeatsTheStateIsNotSaidTwice() {+ assertEquals("Codex, working", SpokenLabels.agent("Codex", "Working", "Working"))+ assertEquals("Codex, working", SpokenLabels.agent("Codex", "Working", "working"))+ assertEquals("Codex, working", SpokenLabels.agent("Codex", "Working", " "))+ }++ @Test fun aMissingNameOrStateStillGivesASentence() {+ assertEquals("Waiting for your OK", SpokenLabels.agent("", "Waiting for your OK", null))+ assertEquals("Claude Code", SpokenLabels.agent("Claude Code", "", null))+ assertEquals("", SpokenLabels.agent("", "", null))+ }++ @Test fun anApprovalNamesTheAgentAndTheAction() {+ assertEquals("Approval needed: Claude Code wants to run a command", SpokenLabels.approval("Approval needed", "Claude Code", "Wants to run a command"))+ assertEquals("Approval needed: Wants to run a command", SpokenLabels.approval("Approval needed", " ", "Wants to run a command"))+ }++ @Test fun aTabWithTheAlertDotSaysWhy() {+ assertEquals("Home", SpokenLabels.tab("Home", false, "Home, a request is waiting"))+ assertEquals("Home, a request is waiting", SpokenLabels.tab("Home", true, "Home, a request is waiting"))+ }+}diff --git a/android/i18n/app-strings.json b/android/i18n/app-strings.jsonindex 5e11ec13..d1109a96 100644--- a/android/i18n/app-strings.json+++ b/android/i18n/app-strings.json@@ -136,6 +136,7 @@"tab_home": "Home","tab_chat": "Chat","tab_settings": "Settings",+ "tab_home_alert": "Home, a request is waiting","design_note": "The building blocks of the new look. No other screen uses them yet.","pair_clipboard": "Paste from clipboard","notif_unlock": "Unlock to see the request",