Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

feat(screen): read the screen with MediaProjection when accessibility is gone

booboosui82f08db4e023
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 62f9200..898eabc 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -73,6 +73,13 @@
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
+ <!--
+ The screen-capture fallback: MediaProjection, for when a ROM has
+ revoked accessibility and the assistant would otherwise be blind.
+ Added to the body's types so a capture can outlive a screen that
+ cannot be read the normal way. See screen/ScreenCapture.
+ -->
+ <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" />
<!-- Battery optimisation. Doze is what holds up the pending todos — they are
AlarmManager alarms, so with the screen off a non-exempt app's reminders
arrive late. Exempting is done from Settings' per-app request screen
@@ -145,7 +152,7 @@
android:name=".BodyCoreService"
android:label="@string/app_name"
android:exported="false"
- android:foregroundServiceType="specialUse|microphone">
+ android:foregroundServiceType="specialUse|microphone|mediaProjection">
<property
android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"
android:value="conversational device assistant: on-device ball, voice turn, reminders" />
diff --git a/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt b/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt
index 7fc3c23..a86b98a 100644
--- a/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt
+++ b/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt
@@ -127,17 +127,11 @@ class BodyCoreService : Service() {
private var screen: ScreenController? = null
/**
- * The screen capability, for callers that can carry on without it.
- *
- * **Callers that cannot must go through here too, and let it throw.** A
- * brain that asked to tap and got an exception it can read is being told
- * the truth; a brain that asked to tap and got a silent no-op will try
- * again, and again, and describe the device as broken.
+ * The screen capability, for callers that can carry on without it, and for
+ * the dispatcher — which turns its absence into a sentence rather than a
+ * no-op. See `CommandDispatcher.requireScreen`.
*/
- private fun screenOrThrow(): ScreenController = screen ?: throw IllegalStateException(
- "accessibility is off — screen.* needs Andee's accessibility service, " +
- "which the system has to have enabled (Settings → Accessibility → Andee)"
- )
+ private fun screenOrNull(): ScreenController? = screen
private lateinit var dispatcher: CommandDispatcher
private var wsServer: BodyWsServer? = null
@@ -198,6 +192,9 @@ class BodyCoreService : Service() {
override fun onCreate() {
super.onCreate()
+ // Before the notification, because the notification's own content is
+ // built from what this service can currently do.
+ capture = net.kuafuai.andee.screen.ScreenCapture(this)
// First, and within the ~5 s the system allows: `startForegroundService`
// owes a notification. Ours is load-bearing twice over — it is what
// keeps this process alive when the accessibility service is taken
@@ -583,9 +580,11 @@ class BodyCoreService : Service() {
dogMotion = net.kuafuai.andee.dog.DogMotion(dogLink, dogSense)
// A *provider*, not the controller: the screen half can be taken away
// and handed back at any time, and the dispatcher outlives both. See
- // [screenOrThrow], which is what a call with no accessibility hits.
+ // `CommandDispatcher.requireScreen`, which turns its absence into a
+ // sentence the brain can read. The second provider is the
+ // accessibility-free way to *look* at the screen — see [captureFallback].
dispatcher = CommandDispatcher(
- ::screenOrThrow, window, ripple, marker,
+ ::screenOrNull, ::captureFallback, window, ripple, marker,
net.kuafuai.andee.device.DeviceInfoController(this),
net.kuafuai.andee.device.DeviceActionsController(this),
net.kuafuai.andee.device.DeviceCommsController(this),
@@ -1316,6 +1315,108 @@ class BodyCoreService : Service() {
}
}
+ /**
+ * Take a screen-capture consent, and start capturing.
+ *
+ * Called with the result of `MediaProjectionManager.createScreenCaptureIntent()`,
+ * which only an Activity can launch — see `ui/SelfCheckActivity`.
+ *
+ * @return whether the screen can be read the non-accessibility way from now
+ * on. False is "the user said no, or the platform did", and both are the
+ * same thing to every caller.
+ */
+ fun offerScreenCapture(resultCode: Int, data: Intent?): Boolean {
+ if (resultCode != android.app.Activity.RESULT_OK || data == null) {
+ postNotification()
+ return false
+ }
+ // The type goes on **before** the projection is built, not after. From
+ // Android 14 `getMediaProjection` refuses an app whose foreground
+ // service does not already claim `mediaProjection`, and adding the type
+ // is only legal while we are visible — which is now, by definition:
+ // this result came back from an Activity.
+ captureWanted = true
+ promoteForWhileInUse()
+ val started = if (::capture.isInitialized) capture.start(resultCode, data) else false
+ if (!started) captureWanted = false
+ postNotification()
+ return started
+ }
+
+ /** Whether a capture can be taken without accessibility right now. */
+ fun hasScreenCapture(): Boolean = ::capture.isInitialized && capture.isAvailable()
+
+ /**
+ * A screenshot from the fallback path, in the shape the `screen.screenshot`
+ * tool returns.
+ *
+ * **Deliberately not [ScreenController.screenshot].** That one settles the
+ * screen by fingerprinting the accessibility tree, draws an aim grid for a
+ * tap that is about to happen, piggybacks the element list and logs a
+ * `shot_id` for the grounding record — every one of which needs
+ * accessibility, and all of which exist to serve *acting* on the screen.
+ * There is no acting without accessibility, so pretending to the brain that
+ * it holds a picture it can aim with is worse than handing it fewer fields.
+ * It gets the pixels and a sentence saying what it does not have.
+ *
+ * **Blocking — call it from a worker.**
+ */
+ fun captureFallback(reason: String): JSONObject? {
+ if (!hasScreenCapture()) return null
+ val bmp = capture.grab() ?: return null
+ var out: android.graphics.Bitmap? = null
+ return try {
+ val w = bmp.width
+ val h = bmp.height
+ // Same budget as the accessibility path, and for the same reason —
+ // see `ScreenController.SHOT_MAX_SIDE`: the raw 1600x2560 frame is
+ // ~765 KB of base64, and the vision tokens are billed whether or not
+ // the extra pixels help. The number is shared rather than copied so
+ // the two paths cannot drift into different picture sizes for the
+ // same tool.
+ val scale = net.kuafuai.andee.screen.ScreenCapture.SHOT_MAX_SIDE.toFloat() / maxOf(w, h)
+ out = if (scale < 1f) {
+ android.graphics.Bitmap.createScaledBitmap(
+ bmp,
+ (w * scale).toInt().coerceAtLeast(1),
+ (h * scale).toInt().coerceAtLeast(1),
+ true,
+ )
+ } else {
+ bmp
+ }
+ val bos = java.io.ByteArrayOutputStream()
+ out.compress(android.graphics.Bitmap.CompressFormat.PNG, 100, bos)
+ val bytes = bos.toByteArray()
+ JSONObject()
+ .put(
+ "png_base64",
+ android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP),
+ )
+ // The *source* dimensions, like the accessibility path reports:
+ // every coordinate in this protocol is 0-1000 relative, so the
+ // model needs the shape of the screen, not of the PNG.
+ .put("width", w)
+ .put("height", h)
+ .put("bytes", bytes.size)
+ // Both of these are the honest half of this result. A brain that
+ // has been handed a picture for weeks without being told it
+ // cannot aim with this one will ask for a tap and be confused by
+ // the refusal.
+ .put(
+ net.kuafuai.andee.screen.ScreenCapture.SOURCE,
+ net.kuafuai.andee.screen.ScreenCapture.SOURCE_MEDIA_PROJECTION,
+ )
+ .put("note", net.kuafuai.andee.screen.ScreenCapture.PROJECTION_NOTE)
+ } catch (t: Throwable) {
+ android.util.Log.w(TAG, "capture fallback ($reason) failed", t)
+ null
+ } finally {
+ if (out != null && out !== bmp) out.recycle()
+ bmp.recycle()
+ }
+ }
+
/**
* The package in front, from the shim.
*
@@ -1378,6 +1479,10 @@ class BodyCoreService : Service() {
// the ROM killed the process outright) — a revoked accessibility
// service no longer comes through here at all, which is the point.
screen = null
+ // Before the window goes and the process may be torn down: a projection
+ // left running shows the system's "screen is being captured" chrome to
+ // the user for a service that is no longer there.
+ if (::capture.isInitialized) capture.release()
instanceRef.compareAndSet(this, null)
// Last, and belt-and-braces: the notification is the one thing that
// must not outlive the body, or the shade keeps promising a ball that
@@ -1434,40 +1539,87 @@ class BodyCoreService : Service() {
}
/**
- * Add the microphone type to the foreground service, now that we can.
+ * The screen's pixels, for when accessibility is not there to be borrowed —
+ * see [ScreenCapture]. Owned here rather than by the controller because the
+ * whole point is that it outlives one.
+ */
+ private lateinit var capture: net.kuafuai.andee.screen.ScreenCapture
+
+ /**
+ * A capture consent has been granted, whether or not the projection is
+ * built yet.
+ *
+ * Read by [whileInUseType] and separate from `capture.isAvailable()` for
+ * one reason: the foreground service type has to be claimed *before*
+ * `getMediaProjection` is called on Android 14, so there is a moment where
+ * we are entitled to the type and do not yet hold a projection. Without
+ * this the type would be added one step too late and the capture would be
+ * refused on exactly the version that needs it most.
+ */
+ @Volatile
+ private var captureWanted = false
+
+ /**
+ * Add every while-in-use type we are now entitled to.
*
* Only legal from a context where the app is visible, which is why the
- * caller is `ui/SelfCheckActivity` — the launcher icon — and nothing else.
+ * callers are the ones they are: `ui/SelfCheckActivity` — the launcher —
+ * for the microphone, and the same Activity again when it comes back with a
+ * capture permission. The platform refuses to *create* a
+ * microphone- or mediaProjection-typed foreground service from the
+ * background, and every other path into this service is background.
+ *
+ * Composed rather than written out because the entitlement changes at
+ * runtime in two independent directions: record permission revoked, or a
+ * capture consent that Android 14 makes per-session.
+ */
+ fun promoteForWhileInUse() {
+ val type = whileInUseType() ?: return
+ runCatching {
+ startForeground(NOTIFY_BODY, buildNotification(), type)
+ }.onFailure {
+ android.util.Log.w(TAG, "while-in-use promotion refused (type=$type)", it)
+ }
+ }
+
+ private fun whileInUseType(): Int? {
+ var t = 0
+ if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) ==
+ android.content.pm.PackageManager.PERMISSION_GRANTED
+ ) {
+ t = t or android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
+ }
+ if (::capture.isInitialized && (captureWanted || capture.isAvailable())) {
+ t = t or android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION
+ }
+ if (t == 0) return null
+ // `SPECIAL_USE` is API 34 and reading the constant below it would be a
+ // NoSuchFieldError rather than a default, so it is added last and only
+ // where it exists. Above 34 it has to be in the set: the type is part of
+ // the service's declared identity there, not a label.
+ return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
+ t or android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE
+ } else {
+ t
+ }
+ }
+
+ /**
+ * Keep the microphone type on the foreground service, now that we can.
+ *
* The payoff is the a11y-free case: with accessibility off, the wake word
* and the talk button are exactly what still works, and the mic type is
* what keeps them working on a device whose only remaining capability is
- * this service.
- *
- * A no-op below API 34 (there is no type check to satisfy) and a no-op when
- * the record permission is missing (the platform would refuse, and there is
- * nothing to gain by asking).
+ * this service. A no-op when the record permission is missing — the
+ * platform would refuse, and there is nothing to gain by asking.
*/
fun promoteForMicrophone() {
- val granted = checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) ==
+ if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) !=
android.content.pm.PackageManager.PERMISSION_GRANTED
- if (!granted) return
- // `SPECIAL_USE` is API 34 and reading the constant below it would be a
- // NoSuchFieldError, not a default — so the type is composed rather than
- // written out. Below 34 the microphone flag alone is the whole point of
- // the call anyway; that is the version range where an app with only an
- // accessibility service to its name loses the mic the moment that grant
- // goes, which is the case this exists for.
- val type = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
- android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE or
- android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
- } else {
- android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE
- }
- runCatching {
- startForeground(NOTIFY_BODY, buildNotification(), type)
- }.onFailure {
- android.util.Log.w(TAG, "microphone promotion refused", it)
+ ) {
+ return
}
+ promoteForWhileInUse()
}
private fun ensureChannel() {
@@ -1552,6 +1704,31 @@ class BodyCoreService : Service() {
fix,
).build()
)
+ // A second way out, and only while this one is the only one left.
+ //
+ // Accessibility cannot be granted by the app, but the projection
+ // consent can at least be *asked for* — and being able to look is
+ // the whole of what a blind assistant can still do. Offered only
+ // when it is not already held, because a row that does nothing is
+ // a row the user learns to ignore, including the one above it.
+ val hasCapture = hasScreenCapture()
+ if (!hasCapture) {
+ val see = PendingIntent.getActivity(
+ this,
+ REQ_GRANT_CAPTURE,
+ Intent(this, net.kuafuai.andee.ui.SelfCheckActivity::class.java)
+ .putExtra(net.kuafuai.andee.ui.SelfCheckActivity.EXTRA_SCREEN_CAPTURE, true)
+ .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK),
+ PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,
+ )
+ b.addAction(
+ Notification.Action.Builder(
+ null,
+ AppLocale.str(this, R.string.body_notify_action_see),
+ see,
+ ).build()
+ )
+ }
}
return b.build()
}
@@ -2588,6 +2765,7 @@ class BodyCoreService : Service() {
private const val NOTIFY_BODY = 0x0B0D
private const val REQ_OPEN_APP = 0x0B0D
private const val REQ_FIX_A11Y = 0x0B0E
+ private const val REQ_GRANT_CAPTURE = 0x0B0F
/**
* Begin the body, or hand an already-running one what it was asked for.
diff --git a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
index 1285372..2ee1551 100644
--- a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
+++ b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
@@ -71,7 +71,33 @@ class CommandDispatcher(
* the honest reply to `screen.*`: accessibility is off. See
* `BodyCoreService.screenOrThrow`.
*/
- private val screen: () -> ScreenController,
+ /**
+ * The screen half, as a *provider* — not the controller itself.
+ *
+ * It has to be a callback because the capability is revocable at runtime:
+ * a Chinese ROM can take accessibility back while this object is alive and
+ * mid-conversation, and it can hand it back later. Holding a
+ * [ScreenController] here would mean holding one built around a dead
+ * binding, and `dispatchGesture` against that reports `completed: true`
+ * while nothing moves — the worst possible answer, because the brain
+ * believes it and tells the user it did something.
+ *
+ * **Named for what it does when there is nothing to hand out: it throws.**
+ * A `screen.*` call on a device whose accessibility has been revoked has to
+ * say so rather than no-op — a silent success is a lie the brain repeats to
+ * the user. See `BodyCoreService.screenOrThrow`.
+ */
+ private val screenOrNull: () -> ScreenController?,
+ /**
+ * The screen's pixels when [requireScreen] cannot be had — MediaProjection,
+ * wired to `BodyCoreService.captureFallback`. Null when even that is
+ * unavailable, which is why the one tool that can use it checks.
+ *
+ * Only reading, never acting: it returns a picture with no aim grid and no
+ * element list, because both of those exist to serve a tap and there are no
+ * taps on this path. See [net.kuafuai.andee.screen.ScreenCapture].
+ */
+ private val captureFallback: (String) -> JSONObject?,
private val window: FloatingWindowUi,
private val ripple: EdgeRippleUi,
private val marker: net.kuafuai.andee.ui.TapMarkerUi,
@@ -167,6 +193,17 @@ class CommandDispatcher(
/** Leak guard, not a guess about thinking time — see [setBusy]. */
private val endDriving = Runnable { setBusy(false) }
+ /**
+ * The screen half, or a sentence saying why there isn't one.
+ *
+ * Every `screen.*` call but [screenshot][dispatch] goes through here, and
+ * the throw is the whole contract: an assistant that cannot tap has to
+ * *say* it cannot tap. A `screen.tap` that returned ok while nothing moved
+ * would be repeated to the user as something that happened.
+ */
+ private fun requireScreen(): ScreenController =
+ screenOrNull() ?: throw IllegalStateException(SCREEN_UNAVAILABLE)
+
/** Written on the main thread, read by worker threads: card already folded. */
@Volatile
private var compacted = false
@@ -397,7 +434,7 @@ class CommandDispatcher(
appContext, R.string.cmd_vault_filling, e.label, secretLabel(field),
)
)
- screen().typeText(value, id = null, secret = true)
+ requireScreen().typeText(value, id = null, secret = true)
}
"device.notifications" -> {
@@ -778,7 +815,7 @@ class CommandDispatcher(
"screen.tap" -> {
passthroughForGesture {
- screen().tapAim(
+ requireScreen().tapAim(
on = params?.optString("on")?.ifBlank { null },
x = params?.optInt("x", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },
y = params?.optInt("y", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },
@@ -794,7 +831,7 @@ class CommandDispatcher(
// and e-numbers cover every node the list shows.
val eid = params?.optString("eid").orEmpty()
require(eid.isNotEmpty()) { "screen.tap_id requires string eid (e.g. \"e7\", from get_screen_element's element list)" }
- passthroughForGesture { screen().tapByEid(eid) }
+ passthroughForGesture { requireScreen().tapByEid(eid) }
}
"screen.swipe" -> {
@@ -804,13 +841,13 @@ class CommandDispatcher(
val x2 = params.getInt("x2");
val y2 = params.getInt("y2")
val dur = params.optLong("duration_ms", 300L)
- passthroughForGesture { screen().swipeNorm(x1, y1, x2, y2, dur) }
+ passthroughForGesture { requireScreen().swipeNorm(x1, y1, x2, y2, dur) }
}
"screen.long_press" -> {
val dur = params?.optLong("duration_ms", 600L) ?: 600L
passthroughForGesture {
- screen().longPressAim(
+ requireScreen().longPressAim(
on = params?.optString("on")?.ifBlank { null },
x = params?.optInt("x", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },
y = params?.optInt("y", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },
@@ -824,21 +861,36 @@ class CommandDispatcher(
"screen.long_press_id" -> {
val eid = params?.optString("eid").orEmpty()
require(eid.isNotEmpty()) { "screen.long_press_id requires string eid" }
- passthroughForGesture { screen().longPressByEid(eid, 600L) }
+ passthroughForGesture { requireScreen().longPressByEid(eid, 600L) }
}
"screen.type" -> {
val text = params?.optString("text").orEmpty()
val id = params?.optString("id")?.ifEmpty { null }
- screen().typeText(text, id)
+ requireScreen().typeText(text, id)
}
"screen.submit_input" -> {
val id = params?.optString("id")?.ifEmpty { null }
- screen().submitInput(id)
+ requireScreen().submitInput(id)
}
- "screen.screenshot" -> blankedForCapture { screen().screenshot() }
+ // The one `screen.*` tool that survives accessibility being revoked.
+ // Everything else here needs the tree or a gesture, and gestures
+ // have no non-root substitute — but "what is on this screen?" is a
+ // question that needs neither, and a device assistant that goes
+ // blind the moment a ROM takes a grant back is worse than one that
+ // can still read the screen out loud.
+ //
+ // `look_region` deliberately does *not* take this path: it exists to
+ // zoom a target so the next call can aim at it, and there is no next
+ // call without accessibility. A brain that wants a closer look can
+ // crop the full shot itself.
+ "screen.screenshot" -> blankedForCapture {
+ screenOrNull()?.screenshot()
+ ?: captureFallback("tool")
+ ?: throw IllegalStateException(SCREEN_UNAVAILABLE)
+ }
"screen.look_region" -> blankedForCapture {
requireNotNull(params) { "screen.look_region requires params" }
@@ -847,9 +899,9 @@ class CommandDispatcher(
// guessing which one the caller meant.
val cell = params.optString("cell")?.ifBlank { null }
if (cell != null) {
- screen().zoomAim(cell, params.optString("part")?.ifBlank { null })
+ requireScreen().zoomAim(cell, params.optString("part")?.ifBlank { null })
} else {
- screen().lookRegion(
+ requireScreen().lookRegion(
params.optInt("x", Int.MIN_VALUE),
params.optInt("y", Int.MIN_VALUE),
params.optInt("w", 0),
@@ -858,7 +910,7 @@ class CommandDispatcher(
}
}
- "screen.ui_tree" -> screen().dumpUiTree(
+ "screen.ui_tree" -> requireScreen().dumpUiTree(
verbose = params?.optBoolean("verbose", false) ?: false,
withShot = params?.optBoolean("with_shot", ToolSchemas.UI_TREE_WITH_SHOT_DEFAULT)
?: ToolSchemas.UI_TREE_WITH_SHOT_DEFAULT,
@@ -869,7 +921,7 @@ class CommandDispatcher(
// the only shots left are the ones on self-drawn panes, where
// pixels are all the brain has. ScreenController stays unaware
// of the overlay — it just calls what it is handed.
- capture = { reason -> blankedForCapture { screen().screenshot(reason) } },
+ capture = { reason -> blankedForCapture { requireScreen().screenshot(reason) } },
)
// Blocking the worker here is the point — the brain asked to wait,
@@ -882,17 +934,17 @@ class CommandDispatcher(
require(!asked.isNaN() && asked > 0) { "screen.wait requires positive seconds" }
val waitMs = (asked * 1000).toLong().coerceAtMost(WAIT_MAX_MS)
Thread.sleep(waitMs)
- screen().dumpUiTree(
+ requireScreen().dumpUiTree(
verbose = false,
withShot = true,
- capture = { reason -> blankedForCapture { screen().screenshot(reason) } },
+ capture = { reason -> blankedForCapture { requireScreen().screenshot(reason) } },
).put("waited_ms", waitMs)
}
"screen.global" -> {
val action = params?.optString("action").orEmpty()
require(action.isNotEmpty()) { "screen.global requires string action" }
- screen().globalAction(action)
+ requireScreen().globalAction(action)
}
// ---- note.* : the notebook -------------------------------------
@@ -1380,7 +1432,7 @@ class CommandDispatcher(
*/
private fun awaitUncovered() {
val deadline = System.currentTimeMillis() + UNCOVER_BUDGET_MS
- while (!screen().activeWindowVisible()) {
+ while (!requireScreen().activeWindowVisible()) {
if (System.currentTimeMillis() >= deadline) return
runCatching { Thread.sleep(UNCOVER_POLL_MS) }
}
@@ -1594,6 +1646,20 @@ class CommandDispatcher(
}
private companion object {
+ /**
+ * What the brain is told when it asks for the screen and there is no
+ * accessibility to ask through.
+ *
+ * Written as a sentence rather than a code because it is read by a
+ * model and, through it, by a person: it has to name the thing that is
+ * off and where it lives, or the answer is "something went wrong" and
+ * the user has nothing to act on. `screen.screenshot` does *not* use
+ * this — it has a fallback; see the `screen.screenshot` branch.
+ */
+ const val SCREEN_UNAVAILABLE =
+ "accessibility is off — Andee can't touch or read the screen without " +
+ "it. It has to be switched on in Settings → Accessibility → Andee."
+
/**
* Ceiling for `screen.wait`. The hub drops a request at 30s, and the
* dump plus screenshot afterwards are part of the same request — so
diff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenCapture.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenCapture.kt
new file mode 100644
index 0000000..f5edb5f
--- /dev/null
+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenCapture.kt
@@ -0,0 +1,247 @@
+package net.kuafuai.andee.screen
+
+import android.app.Activity
+import android.content.Context
+import android.content.Intent
+import android.graphics.Bitmap
+import android.graphics.PixelFormat
+import android.hardware.display.DisplayManager
+import android.hardware.display.VirtualDisplay
+import android.media.Image
+import android.media.ImageReader
+import android.media.projection.MediaProjection
+import android.media.projection.MediaProjectionManager
+import android.os.Handler
+import android.os.Looper
+import android.util.Log
+import android.view.WindowManager
+
+/**
+ * The screen's pixels, without accessibility.
+ *
+ * **Why this exists.** Everything else about reading a screen went through the
+ * accessibility service: `rootInActiveWindow` for the element list,
+ * `takeScreenshot` for the picture. So when a Chinese ROM revoked Andee's
+ * accessibility — the failure this whole split is about — the assistant could
+ * still talk, still remember, still be reached, and was completely **blind**.
+ * "What is this screen?" is the one question a device assistant answers with no
+ * ability to touch anything, and it was the one it could not answer.
+ *
+ * `MediaProjection` is the platform's other way in: one consent dialog, granted
+ * by the user, and after that the app can capture the display with no
+ * accessibility service anywhere in the picture.
+ *
+ * ## What this is not
+ *
+ * **It is not a replacement, and it must not be dressed up as one.** The
+ * `screen.screenshot` the brain normally gets is deliberately *rich*: an aim
+ * grid drawn for a finger it is about to move, the element list piggybacked on
+ * it, and a `shot_id` tying the picture to whatever it taps next. All three of
+ * those serve tapping, and there is no tapping without accessibility —
+ * `dispatchGesture` has no non-root substitute. So the honest shape of a
+ * projection capture is "here are the pixels, at this size": enough to answer
+ * a question about what is on screen, and nothing to aim with. The caller says
+ * which source it used ([SOURCE]) so the brain is never told it is holding a
+ * picture it can act on.
+ *
+ * ## The consent, and why it is the caller's problem
+ *
+ * `MediaProjectionManager.createScreenCaptureIntent()` must be launched **from
+ * an Activity** and its result handed back here — see
+ * `ui/SelfCheckActivity.requestScreenCapture`. There is no way to ask for this
+ * from a service, and no way to ask silently: since Android 14 the grant is
+ * per-session, so a device that reboots needs the user again. That is a real
+ * cost of this fallback and the reason it is a fallback.
+ */
+class ScreenCapture(private val context: Context) {
+
+ private val lock = Any()
+ private var projection: MediaProjection? = null
+ private var display: VirtualDisplay? = null
+ private var reader: ImageReader? = null
+ private var width = 0
+ private var height = 0
+
+ /** Whether a capture will work right now. Free to call. */
+ fun isAvailable(): Boolean = synchronized(lock) { reader != null && projection != null }
+
+ /**
+ * Take the consent result, and build a capture from it.
+ *
+ * Returns false rather than throwing: the two ways this fails — the user
+ * refused, or the platform refused to hand over a projection for a result
+ * code we liked — are both "no capture today", and neither is a fault the
+ * caller can act on differently.
+ */
+ fun start(resultCode: Int, data: Intent?): Boolean {
+ if (resultCode != Activity.RESULT_OK || data == null) {
+ Log.i(TAG, "screen capture not granted (resultCode=$resultCode)")
+ return false
+ }
+ val mgr = context.getSystemService(Context.MEDIA_PROJECTION_SERVICE)
+ as? MediaProjectionManager ?: return false
+ val fresh = runCatching { mgr.getMediaProjection(resultCode, data) }
+ .onFailure { Log.w(TAG, "getMediaProjection refused", it) }
+ .getOrNull() ?: return false
+ // Registered before anything is created from it. Android 14 requires a
+ // callback (an app that stops the projection without one is killed),
+ // and without a callback here a projection the *user* stops — from the
+ // privacy chip — would leave us holding a dead one and reporting
+ // "available" forever.
+ fresh.registerCallback(object : MediaProjection.Callback() {
+ override fun onStop() {
+ Log.i(TAG, "projection stopped by the system or the user")
+ release()
+ }
+ }, Handler(Looper.getMainLooper()))
+
+ val bounds = context.getSystemService(WindowManager::class.java)
+ .currentWindowMetrics.bounds
+ val w = bounds.width()
+ val h = bounds.height()
+ val dpi = context.resources.displayMetrics.densityDpi
+ synchronized(lock) {
+ releaseLocked()
+ val r = ImageReader.newInstance(w, h, PixelFormat.RGBA_8888, IMAGE_BUFFER)
+ val d = runCatching {
+ fresh.createVirtualDisplay(
+ "andee-capture", w, h, dpi,
+ DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
+ r.surface, null, Handler(Looper.getMainLooper()),
+ )
+ }.onFailure { Log.w(TAG, "createVirtualDisplay failed", it) }.getOrNull()
+ if (d == null) {
+ runCatching { r.close() }
+ runCatching { fresh.stop() }
+ return false
+ }
+ projection = fresh
+ display = d
+ reader = r
+ width = w
+ height = h
+ }
+ return true
+ }
+
+ /**
+ * The display as it is now, or null if it could not be read.
+ *
+ * **Blocking — call it from a worker, never the main thread.** The first
+ * frame after a projection starts is not there yet (the display needs a
+ * beat to produce one), so this waits up to [timeoutMs] for an image rather
+ * than returning an empty screen that the brain would describe as blank.
+ */
+ fun grab(timeoutMs: Long = GRAB_TIMEOUT_MS): Bitmap? {
+ val r: ImageReader
+ val w: Int
+ val h: Int
+ synchronized(lock) {
+ r = reader ?: return null
+ w = width
+ h = height
+ }
+ val image = awaitImage(r, timeoutMs) ?: run {
+ Log.w(TAG, "no frame within ${timeoutMs}ms")
+ return null
+ }
+ return try {
+ image.use { toBitmap(it, w, h) }
+ } catch (t: Throwable) {
+ Log.w(TAG, "frame could not be read", t)
+ null
+ }
+ }
+
+ private fun awaitImage(r: ImageReader, timeoutMs: Long): Image? {
+ r.acquireLatestImage()?.let { return it }
+ // Poll rather than listen: ImageReader.OnImageAvailableListener fires on
+ // a handler we do not own, and a latch here would have to be armed
+ // before the frame arrived to not lose it. Polling at this interval
+ // costs nothing next to the frame the display is producing.
+ val deadline = System.currentTimeMillis() + timeoutMs
+ while (System.currentTimeMillis() < deadline) {
+ Thread.sleep(POLL_MS)
+ r.acquireLatestImage()?.let { return it }
+ }
+ return null
+ }
+
+ /**
+ * One `Image` as an ARGB bitmap.
+ *
+ * The row stride is not `width * 4`: the producer pads rows, so the buffer
+ * is copied into a bitmap that is *at least* as wide as the padding
+ * demands and then cropped. Copying straight into `w x h` would shear the
+ * image diagonally, which is the kind of wrong that still looks like a
+ * screenshot until someone reads text off it.
+ */
+ private fun toBitmap(image: Image, w: Int, h: Int): Bitmap {
+ val plane = image.planes[0]
+ val pixelStride = plane.pixelStride
+ val rowStride = plane.rowStride
+ val rowPadding = rowStride - pixelStride * w
+ val padded = Bitmap.createBitmap(
+ w + rowPadding / pixelStride,
+ h,
+ Bitmap.Config.ARGB_8888,
+ )
+ padded.copyPixelsFromBuffer(plane.buffer)
+ if (padded.width == w) return padded
+ val exact = Bitmap.createBitmap(padded, 0, 0, w, h)
+ padded.recycle()
+ return exact
+ }
+
+ fun release() {
+ synchronized(lock) { releaseLocked() }
+ }
+
+ private fun releaseLocked() {
+ runCatching { reader?.close() }
+ runCatching { display?.release() }
+ runCatching { projection?.stop() }
+ reader = null
+ display = null
+ projection = null
+ }
+
+ companion object {
+ private const val TAG = "ScreenCapture"
+
+ /**
+ * Longest side of the PNG the brain is handed, whatever the source.
+ *
+ * Shared with `ScreenController`, which had the number to itself until
+ * the projection fallback arrived — two capture paths with two different
+ * picture sizes for the same tool is a difference the brain would have
+ * to learn for no reason.
+ */
+ const val SHOT_MAX_SIDE = 1280
+
+ /** How many frames the reader may hold before the producer stalls. */
+ private const val IMAGE_BUFFER = 2
+
+ /** A frame is not instant after `start`; this is how long we give it. */
+ private const val GRAB_TIMEOUT_MS = 2_000L
+ private const val POLL_MS = 25L
+
+ /**
+ * Which source a capture came from, in the tool result.
+ *
+ * Spelled out rather than implied, because the two are not
+ * interchangeable: an `accessibility` shot is one the brain may aim
+ * with, a `media_projection` one is only to look at.
+ */
+ const val SOURCE = "source"
+ const val SOURCE_ACCESSIBILITY = "accessibility"
+ const val SOURCE_MEDIA_PROJECTION = "media_projection"
+
+ /** What the brain is told when a projection shot is all it gets. */
+ const val PROJECTION_NOTE =
+ "Captured with MediaProjection: this is the raw screen, with no aim " +
+ "grid and no element list — both of those need Andee's accessibility " +
+ "service, and so does tapping. You can describe what you see; you " +
+ "cannot act on it until accessibility is back on."
+ }
+}
diff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt
index 3557a83..907c9f3 100644
--- a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt
+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt
@@ -3050,8 +3050,15 @@ class ScreenController(
/** The relative coordinate space the brain speaks: 0..1000 both axes. */
private const val NORM_MAX = 1000
- /** Screenshot long side after downscale (generic_bridge value). */
- private const val SHOT_MAX_SIDE = 1280
+ /**
+ * Screenshot long side after downscale (generic_bridge value).
+ *
+ * Owned by [ScreenCapture] and read from there, not copied: the
+ * MediaProjection fallback hands the brain pictures through the same
+ * tool, and a difference in size between the two sources is a
+ * difference the brain would have to learn for no reason.
+ */
+ private const val SHOT_MAX_SIDE = ScreenCapture.SHOT_MAX_SIDE
/** Aim-grid columns. Letters run A.. — keep it ≤ 26. */
private const val GRID_COLS = 8
diff --git a/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt b/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt
index 21a61ff..b7ce7d4 100644
--- a/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt
+++ b/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt
@@ -4,6 +4,7 @@ import android.graphics.drawable.ColorDrawable
import android.os.Bundle
import android.provider.Settings
import android.widget.ImageView
+import androidx.activity.result.contract.ActivityResultContracts
import net.kuafuai.andee.BodyCoreService
import net.kuafuai.andee.ScreenBodyService
import net.kuafuai.andee.config.FactoryReset
@@ -110,6 +111,19 @@ import net.kuafuai.andee.i18n.AppLocale
*/
class SelfCheckActivity : StageActivity() {
+ companion object {
+ /**
+ * Start-intent extra: ask for the screen-capture consent as soon as
+ * this page is up.
+ *
+ * Set by the body's notification action, which is the only way in —
+ * `MediaProjectionManager.createScreenCaptureIntent()` needs an
+ * Activity, and this is the only one the app has. See
+ * [requestScreenCapture].
+ */
+ const val EXTRA_SCREEN_CAPTURE = "screen_capture"
+ }
+
/**
* The card. Built here on the service-off path only, placed on [stage], and
* gone with the Activity.
@@ -143,6 +157,13 @@ class SelfCheckActivity : StageActivity() {
override fun onCreate(savedInstanceState: Bundle?) {
super.onCreate(savedInstanceState)
+ // The notification's 允许看屏幕 action lands here — see
+ // [requestScreenCapture]. Asked before the branch below, because the
+ // point of the action is that the page it opens is a means to an end.
+ if (intent?.getBooleanExtra(EXTRA_SCREEN_CAPTURE, false) == true) {
+ requestScreenCapture()
+ }
+
// A working assistant ⇒ this icon means what every other icon on the
// home screen means: open the app. That is the *same request* a
// long-press on the ball makes, so it is the same call — one entry point
@@ -326,6 +347,68 @@ class SelfCheckActivity : StageActivity() {
* icon-as-app entry finishes before the card is ever built, and a lifecycle
* callback arriving afterwards is not a good moment to throw.
*/
+ /**
+ * The screen-capture consent, and what the answer goes to.
+ *
+ * Registered as a field rather than from [onCreate] because the lifecycle
+ * requires the callback to exist before the Activity is started, and this
+ * page is started by an icon tap — there is no earlier moment.
+ *
+ * The grant goes to the *body*, not to this Activity: the capture has to
+ * outlive this page, and on Android 14 it has to be re-asked for every
+ * session, so the thing that owns it is the thing that is still running
+ * tomorrow. Nothing is announced here either — the body's own notification
+ * is what reports this state, and a second voice describing the same fact
+ * is how two surfaces start disagreeing.
+ */
+ private val captureConsent = registerForActivityResult(
+ ActivityResultContracts.StartActivityForResult(),
+ ) { result ->
+ val granted = BodyCoreService.get()
+ ?.offerScreenCapture(result.resultCode, result.data) == true
+ android.util.Log.i("Body", "screen capture consent: granted=$granted")
+ }
+
+ /**
+ * Ask the user to let Andee read the screen without accessibility.
+ *
+ * **Only reachable while the assistant is blind**, which is the only time
+ * it is worth a system dialog: with accessibility on, `takeScreenshot` is
+ * richer, free and already granted — an aim grid the brain can tap from and
+ * the element list beside it — and asking for a second, weaker way in would
+ * be asking the user to make a decision with no benefit.
+ *
+ * The caller is the body's notification action, `Intent` extra
+ * [EXTRA_SCREEN_CAPTURE]. Failing to launch is not worth a card: the
+ * platform simply has no such screen on some ROMs, and the notification
+ * will go on offering it.
+ */
+ private fun requestScreenCapture() {
+ val mgr = getSystemService(android.media.projection.MediaProjectionManager::class.java)
+ ?: return
+ runCatching { captureConsent.launch(mgr.createScreenCaptureIntent()) }
+ .onFailure {
+ android.util.Log.w("Body", "screen capture consent would not launch", it)
+ }
+ }
+
+ /**
+ * The other half of [EXTRA_SCREEN_CAPTURE], and not a formality.
+ *
+ * This page is `singleTop`, and its two callers both routinely arrive while
+ * it is already the top instance: the notification action is tapped from
+ * the shade with the page up, and the launcher icon does the same. On that
+ * path `onCreate` does not run, so an extra read only there would be a
+ * button that works the first time and silently stops working afterwards —
+ * and the first time is the one where the user had not yet refused.
+ * [setIntent] keeps `intent` honest for anything that reads it later.
+ */
+ override fun onNewIntent(intent: android.content.Intent) {
+ super.onNewIntent(intent)
+ setIntent(intent)
+ if (intent.getBooleanExtra(EXTRA_SCREEN_CAPTURE, false)) requestScreenCapture()
+ }
+
override fun onResume() {
super.onResume()
if (!::ui.isInitialized) return
diff --git a/app/src/main/res/values-en/strings.xml b/app/src/main/res/values-en/strings.xml
index 1228a5f..4f7fc90 100644
--- a/app/src/main/res/values-en/strings.xml
+++ b/app/src/main/res/values-en/strings.xml
@@ -361,6 +361,7 @@
<string name="body_notify_text_ok">Screen control is up.</string>
<string name="body_notify_text_no_screen">Accessibility was switched off, so I can\'t reach the screen. Tap the action to get it back.</string>
<string name="body_notify_action_fix">Get it back</string>
+ <string name="body_notify_action_see">Let me see the screen</string>
<string name="cmd_vault_filling">Typing the “%2$s” for %1$s…</string>
<string name="cmd_ask_yes">OK</string>
<string name="cmd_ask_no">No thanks</string>
diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml
index 1fde1b1..e5c3933 100644
--- a/app/src/main/res/values/strings.xml
+++ b/app/src/main/res/values/strings.xml
@@ -398,6 +398,7 @@
<string name="body_notify_text_ok">屏幕操作已就绪。</string>
<string name="body_notify_text_no_screen">无障碍被系统关掉了,我现在碰不到屏幕。点右边接回来。</string>
<string name="body_notify_action_fix">接回来</string>
+ <string name="body_notify_action_see">允许看屏幕</string>
<!-- net/CommandDispatcher.kt — the three lines from it a user can see. -->
<string name="cmd_vault_filling">正在填入「%1$s」的%2$s…</string>