Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。
feat(screen): read the screen with MediaProjection when accessibility is gone
booboosui82f08db4e023
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xmlindex 62f9200..898eabc 100644--- a/app/src/main/AndroidManifest.xml+++ b/app/src/main/AndroidManifest.xml@@ -73,6 +73,13 @@<uses-permission android:name="android.permission.FOREGROUND_SERVICE" /><uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" /><uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />+ <!--+ The screen-capture fallback: MediaProjection, for when a ROM has+ revoked accessibility and the assistant would otherwise be blind.+ Added to the body's types so a capture can outlive a screen that+ cannot be read the normal way. See screen/ScreenCapture.+ -->+ <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MEDIA_PROJECTION" /><!-- Battery optimisation. Doze is what holds up the pending todos — they areAlarmManager alarms, so with the screen off a non-exempt app's remindersarrive late. Exempting is done from Settings' per-app request screen@@ -145,7 +152,7 @@android:name=".BodyCoreService"android:label="@string/app_name"android:exported="false"- android:foregroundServiceType="specialUse|microphone">+ android:foregroundServiceType="specialUse|microphone|mediaProjection"><propertyandroid:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"android:value="conversational device assistant: on-device ball, voice turn, reminders" />diff --git a/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt b/app/src/main/java/net/kuafuai/andee/BodyCoreService.ktindex 7fc3c23..a86b98a 100644--- a/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt+++ b/app/src/main/java/net/kuafuai/andee/BodyCoreService.kt@@ -127,17 +127,11 @@ class BodyCoreService : Service() {private var screen: ScreenController? = null/**- * The screen capability, for callers that can carry on without it.- *- * **Callers that cannot must go through here too, and let it throw.** A- * brain that asked to tap and got an exception it can read is being told- * the truth; a brain that asked to tap and got a silent no-op will try- * again, and again, and describe the device as broken.+ * The screen capability, for callers that can carry on without it, and for+ * the dispatcher — which turns its absence into a sentence rather than a+ * no-op. See `CommandDispatcher.requireScreen`.*/- private fun screenOrThrow(): ScreenController = screen ?: throw IllegalStateException(- "accessibility is off — screen.* needs Andee's accessibility service, " +- "which the system has to have enabled (Settings → Accessibility → Andee)"- )+ private fun screenOrNull(): ScreenController? = screenprivate lateinit var dispatcher: CommandDispatcherprivate var wsServer: BodyWsServer? = null@@ -198,6 +192,9 @@ class BodyCoreService : Service() {override fun onCreate() {super.onCreate()+ // Before the notification, because the notification's own content is+ // built from what this service can currently do.+ capture = net.kuafuai.andee.screen.ScreenCapture(this)// First, and within the ~5 s the system allows: `startForegroundService`// owes a notification. Ours is load-bearing twice over — it is what// keeps this process alive when the accessibility service is taken@@ -583,9 +580,11 @@ class BodyCoreService : Service() {dogMotion = net.kuafuai.andee.dog.DogMotion(dogLink, dogSense)// A *provider*, not the controller: the screen half can be taken away// and handed back at any time, and the dispatcher outlives both. See- // [screenOrThrow], which is what a call with no accessibility hits.+ // `CommandDispatcher.requireScreen`, which turns its absence into a+ // sentence the brain can read. The second provider is the+ // accessibility-free way to *look* at the screen — see [captureFallback].dispatcher = CommandDispatcher(- ::screenOrThrow, window, ripple, marker,+ ::screenOrNull, ::captureFallback, window, ripple, marker,net.kuafuai.andee.device.DeviceInfoController(this),net.kuafuai.andee.device.DeviceActionsController(this),net.kuafuai.andee.device.DeviceCommsController(this),@@ -1316,6 +1315,108 @@ class BodyCoreService : Service() {}}+ /**+ * Take a screen-capture consent, and start capturing.+ *+ * Called with the result of `MediaProjectionManager.createScreenCaptureIntent()`,+ * which only an Activity can launch — see `ui/SelfCheckActivity`.+ *+ * @return whether the screen can be read the non-accessibility way from now+ * on. False is "the user said no, or the platform did", and both are the+ * same thing to every caller.+ */+ fun offerScreenCapture(resultCode: Int, data: Intent?): Boolean {+ if (resultCode != android.app.Activity.RESULT_OK || data == null) {+ postNotification()+ return false+ }+ // The type goes on **before** the projection is built, not after. From+ // Android 14 `getMediaProjection` refuses an app whose foreground+ // service does not already claim `mediaProjection`, and adding the type+ // is only legal while we are visible — which is now, by definition:+ // this result came back from an Activity.+ captureWanted = true+ promoteForWhileInUse()+ val started = if (::capture.isInitialized) capture.start(resultCode, data) else false+ if (!started) captureWanted = false+ postNotification()+ return started+ }++ /** Whether a capture can be taken without accessibility right now. */+ fun hasScreenCapture(): Boolean = ::capture.isInitialized && capture.isAvailable()++ /**+ * A screenshot from the fallback path, in the shape the `screen.screenshot`+ * tool returns.+ *+ * **Deliberately not [ScreenController.screenshot].** That one settles the+ * screen by fingerprinting the accessibility tree, draws an aim grid for a+ * tap that is about to happen, piggybacks the element list and logs a+ * `shot_id` for the grounding record — every one of which needs+ * accessibility, and all of which exist to serve *acting* on the screen.+ * There is no acting without accessibility, so pretending to the brain that+ * it holds a picture it can aim with is worse than handing it fewer fields.+ * It gets the pixels and a sentence saying what it does not have.+ *+ * **Blocking — call it from a worker.**+ */+ fun captureFallback(reason: String): JSONObject? {+ if (!hasScreenCapture()) return null+ val bmp = capture.grab() ?: return null+ var out: android.graphics.Bitmap? = null+ return try {+ val w = bmp.width+ val h = bmp.height+ // Same budget as the accessibility path, and for the same reason —+ // see `ScreenController.SHOT_MAX_SIDE`: the raw 1600x2560 frame is+ // ~765 KB of base64, and the vision tokens are billed whether or not+ // the extra pixels help. The number is shared rather than copied so+ // the two paths cannot drift into different picture sizes for the+ // same tool.+ val scale = net.kuafuai.andee.screen.ScreenCapture.SHOT_MAX_SIDE.toFloat() / maxOf(w, h)+ out = if (scale < 1f) {+ android.graphics.Bitmap.createScaledBitmap(+ bmp,+ (w * scale).toInt().coerceAtLeast(1),+ (h * scale).toInt().coerceAtLeast(1),+ true,+ )+ } else {+ bmp+ }+ val bos = java.io.ByteArrayOutputStream()+ out.compress(android.graphics.Bitmap.CompressFormat.PNG, 100, bos)+ val bytes = bos.toByteArray()+ JSONObject()+ .put(+ "png_base64",+ android.util.Base64.encodeToString(bytes, android.util.Base64.NO_WRAP),+ )+ // The *source* dimensions, like the accessibility path reports:+ // every coordinate in this protocol is 0-1000 relative, so the+ // model needs the shape of the screen, not of the PNG.+ .put("width", w)+ .put("height", h)+ .put("bytes", bytes.size)+ // Both of these are the honest half of this result. A brain that+ // has been handed a picture for weeks without being told it+ // cannot aim with this one will ask for a tap and be confused by+ // the refusal.+ .put(+ net.kuafuai.andee.screen.ScreenCapture.SOURCE,+ net.kuafuai.andee.screen.ScreenCapture.SOURCE_MEDIA_PROJECTION,+ )+ .put("note", net.kuafuai.andee.screen.ScreenCapture.PROJECTION_NOTE)+ } catch (t: Throwable) {+ android.util.Log.w(TAG, "capture fallback ($reason) failed", t)+ null+ } finally {+ if (out != null && out !== bmp) out.recycle()+ bmp.recycle()+ }+ }+/*** The package in front, from the shim.*@@ -1378,6 +1479,10 @@ class BodyCoreService : Service() {// the ROM killed the process outright) — a revoked accessibility// service no longer comes through here at all, which is the point.screen = null+ // Before the window goes and the process may be torn down: a projection+ // left running shows the system's "screen is being captured" chrome to+ // the user for a service that is no longer there.+ if (::capture.isInitialized) capture.release()instanceRef.compareAndSet(this, null)// Last, and belt-and-braces: the notification is the one thing that// must not outlive the body, or the shade keeps promising a ball that@@ -1434,40 +1539,87 @@ class BodyCoreService : Service() {}/**- * Add the microphone type to the foreground service, now that we can.+ * The screen's pixels, for when accessibility is not there to be borrowed —+ * see [ScreenCapture]. Owned here rather than by the controller because the+ * whole point is that it outlives one.+ */+ private lateinit var capture: net.kuafuai.andee.screen.ScreenCapture++ /**+ * A capture consent has been granted, whether or not the projection is+ * built yet.+ *+ * Read by [whileInUseType] and separate from `capture.isAvailable()` for+ * one reason: the foreground service type has to be claimed *before*+ * `getMediaProjection` is called on Android 14, so there is a moment where+ * we are entitled to the type and do not yet hold a projection. Without+ * this the type would be added one step too late and the capture would be+ * refused on exactly the version that needs it most.+ */+ @Volatile+ private var captureWanted = false++ /**+ * Add every while-in-use type we are now entitled to.** Only legal from a context where the app is visible, which is why the- * caller is `ui/SelfCheckActivity` — the launcher icon — and nothing else.+ * callers are the ones they are: `ui/SelfCheckActivity` — the launcher —+ * for the microphone, and the same Activity again when it comes back with a+ * capture permission. The platform refuses to *create* a+ * microphone- or mediaProjection-typed foreground service from the+ * background, and every other path into this service is background.+ *+ * Composed rather than written out because the entitlement changes at+ * runtime in two independent directions: record permission revoked, or a+ * capture consent that Android 14 makes per-session.+ */+ fun promoteForWhileInUse() {+ val type = whileInUseType() ?: return+ runCatching {+ startForeground(NOTIFY_BODY, buildNotification(), type)+ }.onFailure {+ android.util.Log.w(TAG, "while-in-use promotion refused (type=$type)", it)+ }+ }++ private fun whileInUseType(): Int? {+ var t = 0+ if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) ==+ android.content.pm.PackageManager.PERMISSION_GRANTED+ ) {+ t = t or android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE+ }+ if (::capture.isInitialized && (captureWanted || capture.isAvailable())) {+ t = t or android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MEDIA_PROJECTION+ }+ if (t == 0) return null+ // `SPECIAL_USE` is API 34 and reading the constant below it would be a+ // NoSuchFieldError rather than a default, so it is added last and only+ // where it exists. Above 34 it has to be in the set: the type is part of+ // the service's declared identity there, not a label.+ return if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {+ t or android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE+ } else {+ t+ }+ }++ /**+ * Keep the microphone type on the foreground service, now that we can.+ ** The payoff is the a11y-free case: with accessibility off, the wake word* and the talk button are exactly what still works, and the mic type is* what keeps them working on a device whose only remaining capability is- * this service.- *- * A no-op below API 34 (there is no type check to satisfy) and a no-op when- * the record permission is missing (the platform would refuse, and there is- * nothing to gain by asking).+ * this service. A no-op when the record permission is missing — the+ * platform would refuse, and there is nothing to gain by asking.*/fun promoteForMicrophone() {- val granted = checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) ==+ if (checkSelfPermission(android.Manifest.permission.RECORD_AUDIO) !=android.content.pm.PackageManager.PERMISSION_GRANTED- if (!granted) return- // `SPECIAL_USE` is API 34 and reading the constant below it would be a- // NoSuchFieldError, not a default — so the type is composed rather than- // written out. Below 34 the microphone flag alone is the whole point of- // the call anyway; that is the version range where an app with only an- // accessibility service to its name loses the mic the moment that grant- // goes, which is the case this exists for.- val type = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {- android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE or- android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE- } else {- android.content.pm.ServiceInfo.FOREGROUND_SERVICE_TYPE_MICROPHONE- }- runCatching {- startForeground(NOTIFY_BODY, buildNotification(), type)- }.onFailure {- android.util.Log.w(TAG, "microphone promotion refused", it)+ ) {+ return}+ promoteForWhileInUse()}private fun ensureChannel() {@@ -1552,6 +1704,31 @@ class BodyCoreService : Service() {fix,).build())+ // A second way out, and only while this one is the only one left.+ //+ // Accessibility cannot be granted by the app, but the projection+ // consent can at least be *asked for* — and being able to look is+ // the whole of what a blind assistant can still do. Offered only+ // when it is not already held, because a row that does nothing is+ // a row the user learns to ignore, including the one above it.+ val hasCapture = hasScreenCapture()+ if (!hasCapture) {+ val see = PendingIntent.getActivity(+ this,+ REQ_GRANT_CAPTURE,+ Intent(this, net.kuafuai.andee.ui.SelfCheckActivity::class.java)+ .putExtra(net.kuafuai.andee.ui.SelfCheckActivity.EXTRA_SCREEN_CAPTURE, true)+ .addFlags(Intent.FLAG_ACTIVITY_NEW_TASK),+ PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE,+ )+ b.addAction(+ Notification.Action.Builder(+ null,+ AppLocale.str(this, R.string.body_notify_action_see),+ see,+ ).build()+ )+ }}return b.build()}@@ -2588,6 +2765,7 @@ class BodyCoreService : Service() {private const val NOTIFY_BODY = 0x0B0Dprivate const val REQ_OPEN_APP = 0x0B0Dprivate const val REQ_FIX_A11Y = 0x0B0E+ private const val REQ_GRANT_CAPTURE = 0x0B0F/*** Begin the body, or hand an already-running one what it was asked for.diff --git a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.ktindex 1285372..2ee1551 100644--- a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt+++ b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt@@ -71,7 +71,33 @@ class CommandDispatcher(* the honest reply to `screen.*`: accessibility is off. See* `BodyCoreService.screenOrThrow`.*/- private val screen: () -> ScreenController,+ /**+ * The screen half, as a *provider* — not the controller itself.+ *+ * It has to be a callback because the capability is revocable at runtime:+ * a Chinese ROM can take accessibility back while this object is alive and+ * mid-conversation, and it can hand it back later. Holding a+ * [ScreenController] here would mean holding one built around a dead+ * binding, and `dispatchGesture` against that reports `completed: true`+ * while nothing moves — the worst possible answer, because the brain+ * believes it and tells the user it did something.+ *+ * **Named for what it does when there is nothing to hand out: it throws.**+ * A `screen.*` call on a device whose accessibility has been revoked has to+ * say so rather than no-op — a silent success is a lie the brain repeats to+ * the user. See `BodyCoreService.screenOrThrow`.+ */+ private val screenOrNull: () -> ScreenController?,+ /**+ * The screen's pixels when [requireScreen] cannot be had — MediaProjection,+ * wired to `BodyCoreService.captureFallback`. Null when even that is+ * unavailable, which is why the one tool that can use it checks.+ *+ * Only reading, never acting: it returns a picture with no aim grid and no+ * element list, because both of those exist to serve a tap and there are no+ * taps on this path. See [net.kuafuai.andee.screen.ScreenCapture].+ */+ private val captureFallback: (String) -> JSONObject?,private val window: FloatingWindowUi,private val ripple: EdgeRippleUi,private val marker: net.kuafuai.andee.ui.TapMarkerUi,@@ -167,6 +193,17 @@ class CommandDispatcher(/** Leak guard, not a guess about thinking time — see [setBusy]. */private val endDriving = Runnable { setBusy(false) }+ /**+ * The screen half, or a sentence saying why there isn't one.+ *+ * Every `screen.*` call but [screenshot][dispatch] goes through here, and+ * the throw is the whole contract: an assistant that cannot tap has to+ * *say* it cannot tap. A `screen.tap` that returned ok while nothing moved+ * would be repeated to the user as something that happened.+ */+ private fun requireScreen(): ScreenController =+ screenOrNull() ?: throw IllegalStateException(SCREEN_UNAVAILABLE)+/** Written on the main thread, read by worker threads: card already folded. */@Volatileprivate var compacted = false@@ -397,7 +434,7 @@ class CommandDispatcher(appContext, R.string.cmd_vault_filling, e.label, secretLabel(field),))- screen().typeText(value, id = null, secret = true)+ requireScreen().typeText(value, id = null, secret = true)}"device.notifications" -> {@@ -778,7 +815,7 @@ class CommandDispatcher("screen.tap" -> {passthroughForGesture {- screen().tapAim(+ requireScreen().tapAim(on = params?.optString("on")?.ifBlank { null },x = params?.optInt("x", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },y = params?.optInt("y", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },@@ -794,7 +831,7 @@ class CommandDispatcher(// and e-numbers cover every node the list shows.val eid = params?.optString("eid").orEmpty()require(eid.isNotEmpty()) { "screen.tap_id requires string eid (e.g. \"e7\", from get_screen_element's element list)" }- passthroughForGesture { screen().tapByEid(eid) }+ passthroughForGesture { requireScreen().tapByEid(eid) }}"screen.swipe" -> {@@ -804,13 +841,13 @@ class CommandDispatcher(val x2 = params.getInt("x2");val y2 = params.getInt("y2")val dur = params.optLong("duration_ms", 300L)- passthroughForGesture { screen().swipeNorm(x1, y1, x2, y2, dur) }+ passthroughForGesture { requireScreen().swipeNorm(x1, y1, x2, y2, dur) }}"screen.long_press" -> {val dur = params?.optLong("duration_ms", 600L) ?: 600LpassthroughForGesture {- screen().longPressAim(+ requireScreen().longPressAim(on = params?.optString("on")?.ifBlank { null },x = params?.optInt("x", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },y = params?.optInt("y", Int.MIN_VALUE)?.takeIf { it != Int.MIN_VALUE },@@ -824,21 +861,36 @@ class CommandDispatcher("screen.long_press_id" -> {val eid = params?.optString("eid").orEmpty()require(eid.isNotEmpty()) { "screen.long_press_id requires string eid" }- passthroughForGesture { screen().longPressByEid(eid, 600L) }+ passthroughForGesture { requireScreen().longPressByEid(eid, 600L) }}"screen.type" -> {val text = params?.optString("text").orEmpty()val id = params?.optString("id")?.ifEmpty { null }- screen().typeText(text, id)+ requireScreen().typeText(text, id)}"screen.submit_input" -> {val id = params?.optString("id")?.ifEmpty { null }- screen().submitInput(id)+ requireScreen().submitInput(id)}- "screen.screenshot" -> blankedForCapture { screen().screenshot() }+ // The one `screen.*` tool that survives accessibility being revoked.+ // Everything else here needs the tree or a gesture, and gestures+ // have no non-root substitute — but "what is on this screen?" is a+ // question that needs neither, and a device assistant that goes+ // blind the moment a ROM takes a grant back is worse than one that+ // can still read the screen out loud.+ //+ // `look_region` deliberately does *not* take this path: it exists to+ // zoom a target so the next call can aim at it, and there is no next+ // call without accessibility. A brain that wants a closer look can+ // crop the full shot itself.+ "screen.screenshot" -> blankedForCapture {+ screenOrNull()?.screenshot()+ ?: captureFallback("tool")+ ?: throw IllegalStateException(SCREEN_UNAVAILABLE)+ }"screen.look_region" -> blankedForCapture {requireNotNull(params) { "screen.look_region requires params" }@@ -847,9 +899,9 @@ class CommandDispatcher(// guessing which one the caller meant.val cell = params.optString("cell")?.ifBlank { null }if (cell != null) {- screen().zoomAim(cell, params.optString("part")?.ifBlank { null })+ requireScreen().zoomAim(cell, params.optString("part")?.ifBlank { null })} else {- screen().lookRegion(+ requireScreen().lookRegion(params.optInt("x", Int.MIN_VALUE),params.optInt("y", Int.MIN_VALUE),params.optInt("w", 0),@@ -858,7 +910,7 @@ class CommandDispatcher(}}- "screen.ui_tree" -> screen().dumpUiTree(+ "screen.ui_tree" -> requireScreen().dumpUiTree(verbose = params?.optBoolean("verbose", false) ?: false,withShot = params?.optBoolean("with_shot", ToolSchemas.UI_TREE_WITH_SHOT_DEFAULT)?: ToolSchemas.UI_TREE_WITH_SHOT_DEFAULT,@@ -869,7 +921,7 @@ class CommandDispatcher(// the only shots left are the ones on self-drawn panes, where// pixels are all the brain has. ScreenController stays unaware// of the overlay — it just calls what it is handed.- capture = { reason -> blankedForCapture { screen().screenshot(reason) } },+ capture = { reason -> blankedForCapture { requireScreen().screenshot(reason) } },)// Blocking the worker here is the point — the brain asked to wait,@@ -882,17 +934,17 @@ class CommandDispatcher(require(!asked.isNaN() && asked > 0) { "screen.wait requires positive seconds" }val waitMs = (asked * 1000).toLong().coerceAtMost(WAIT_MAX_MS)Thread.sleep(waitMs)- screen().dumpUiTree(+ requireScreen().dumpUiTree(verbose = false,withShot = true,- capture = { reason -> blankedForCapture { screen().screenshot(reason) } },+ capture = { reason -> blankedForCapture { requireScreen().screenshot(reason) } },).put("waited_ms", waitMs)}"screen.global" -> {val action = params?.optString("action").orEmpty()require(action.isNotEmpty()) { "screen.global requires string action" }- screen().globalAction(action)+ requireScreen().globalAction(action)}// ---- note.* : the notebook -------------------------------------@@ -1380,7 +1432,7 @@ class CommandDispatcher(*/private fun awaitUncovered() {val deadline = System.currentTimeMillis() + UNCOVER_BUDGET_MS- while (!screen().activeWindowVisible()) {+ while (!requireScreen().activeWindowVisible()) {if (System.currentTimeMillis() >= deadline) returnrunCatching { Thread.sleep(UNCOVER_POLL_MS) }}@@ -1594,6 +1646,20 @@ class CommandDispatcher(}private companion object {+ /**+ * What the brain is told when it asks for the screen and there is no+ * accessibility to ask through.+ *+ * Written as a sentence rather than a code because it is read by a+ * model and, through it, by a person: it has to name the thing that is+ * off and where it lives, or the answer is "something went wrong" and+ * the user has nothing to act on. `screen.screenshot` does *not* use+ * this — it has a fallback; see the `screen.screenshot` branch.+ */+ const val SCREEN_UNAVAILABLE =+ "accessibility is off — Andee can't touch or read the screen without " ++ "it. It has to be switched on in Settings → Accessibility → Andee."+/*** Ceiling for `screen.wait`. The hub drops a request at 30s, and the* dump plus screenshot afterwards are part of the same request — sodiff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenCapture.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenCapture.ktnew file mode 100644index 0000000..f5edb5f--- /dev/null+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenCapture.kt@@ -0,0 +1,247 @@+package net.kuafuai.andee.screen++import android.app.Activity+import android.content.Context+import android.content.Intent+import android.graphics.Bitmap+import android.graphics.PixelFormat+import android.hardware.display.DisplayManager+import android.hardware.display.VirtualDisplay+import android.media.Image+import android.media.ImageReader+import android.media.projection.MediaProjection+import android.media.projection.MediaProjectionManager+import android.os.Handler+import android.os.Looper+import android.util.Log+import android.view.WindowManager++/**+ * The screen's pixels, without accessibility.+ *+ * **Why this exists.** Everything else about reading a screen went through the+ * accessibility service: `rootInActiveWindow` for the element list,+ * `takeScreenshot` for the picture. So when a Chinese ROM revoked Andee's+ * accessibility — the failure this whole split is about — the assistant could+ * still talk, still remember, still be reached, and was completely **blind**.+ * "What is this screen?" is the one question a device assistant answers with no+ * ability to touch anything, and it was the one it could not answer.+ *+ * `MediaProjection` is the platform's other way in: one consent dialog, granted+ * by the user, and after that the app can capture the display with no+ * accessibility service anywhere in the picture.+ *+ * ## What this is not+ *+ * **It is not a replacement, and it must not be dressed up as one.** The+ * `screen.screenshot` the brain normally gets is deliberately *rich*: an aim+ * grid drawn for a finger it is about to move, the element list piggybacked on+ * it, and a `shot_id` tying the picture to whatever it taps next. All three of+ * those serve tapping, and there is no tapping without accessibility —+ * `dispatchGesture` has no non-root substitute. So the honest shape of a+ * projection capture is "here are the pixels, at this size": enough to answer+ * a question about what is on screen, and nothing to aim with. The caller says+ * which source it used ([SOURCE]) so the brain is never told it is holding a+ * picture it can act on.+ *+ * ## The consent, and why it is the caller's problem+ *+ * `MediaProjectionManager.createScreenCaptureIntent()` must be launched **from+ * an Activity** and its result handed back here — see+ * `ui/SelfCheckActivity.requestScreenCapture`. There is no way to ask for this+ * from a service, and no way to ask silently: since Android 14 the grant is+ * per-session, so a device that reboots needs the user again. That is a real+ * cost of this fallback and the reason it is a fallback.+ */+class ScreenCapture(private val context: Context) {++ private val lock = Any()+ private var projection: MediaProjection? = null+ private var display: VirtualDisplay? = null+ private var reader: ImageReader? = null+ private var width = 0+ private var height = 0++ /** Whether a capture will work right now. Free to call. */+ fun isAvailable(): Boolean = synchronized(lock) { reader != null && projection != null }++ /**+ * Take the consent result, and build a capture from it.+ *+ * Returns false rather than throwing: the two ways this fails — the user+ * refused, or the platform refused to hand over a projection for a result+ * code we liked — are both "no capture today", and neither is a fault the+ * caller can act on differently.+ */+ fun start(resultCode: Int, data: Intent?): Boolean {+ if (resultCode != Activity.RESULT_OK || data == null) {+ Log.i(TAG, "screen capture not granted (resultCode=$resultCode)")+ return false+ }+ val mgr = context.getSystemService(Context.MEDIA_PROJECTION_SERVICE)+ as? MediaProjectionManager ?: return false+ val fresh = runCatching { mgr.getMediaProjection(resultCode, data) }+ .onFailure { Log.w(TAG, "getMediaProjection refused", it) }+ .getOrNull() ?: return false+ // Registered before anything is created from it. Android 14 requires a+ // callback (an app that stops the projection without one is killed),+ // and without a callback here a projection the *user* stops — from the+ // privacy chip — would leave us holding a dead one and reporting+ // "available" forever.+ fresh.registerCallback(object : MediaProjection.Callback() {+ override fun onStop() {+ Log.i(TAG, "projection stopped by the system or the user")+ release()+ }+ }, Handler(Looper.getMainLooper()))++ val bounds = context.getSystemService(WindowManager::class.java)+ .currentWindowMetrics.bounds+ val w = bounds.width()+ val h = bounds.height()+ val dpi = context.resources.displayMetrics.densityDpi+ synchronized(lock) {+ releaseLocked()+ val r = ImageReader.newInstance(w, h, PixelFormat.RGBA_8888, IMAGE_BUFFER)+ val d = runCatching {+ fresh.createVirtualDisplay(+ "andee-capture", w, h, dpi,+ DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,+ r.surface, null, Handler(Looper.getMainLooper()),+ )+ }.onFailure { Log.w(TAG, "createVirtualDisplay failed", it) }.getOrNull()+ if (d == null) {+ runCatching { r.close() }+ runCatching { fresh.stop() }+ return false+ }+ projection = fresh+ display = d+ reader = r+ width = w+ height = h+ }+ return true+ }++ /**+ * The display as it is now, or null if it could not be read.+ *+ * **Blocking — call it from a worker, never the main thread.** The first+ * frame after a projection starts is not there yet (the display needs a+ * beat to produce one), so this waits up to [timeoutMs] for an image rather+ * than returning an empty screen that the brain would describe as blank.+ */+ fun grab(timeoutMs: Long = GRAB_TIMEOUT_MS): Bitmap? {+ val r: ImageReader+ val w: Int+ val h: Int+ synchronized(lock) {+ r = reader ?: return null+ w = width+ h = height+ }+ val image = awaitImage(r, timeoutMs) ?: run {+ Log.w(TAG, "no frame within ${timeoutMs}ms")+ return null+ }+ return try {+ image.use { toBitmap(it, w, h) }+ } catch (t: Throwable) {+ Log.w(TAG, "frame could not be read", t)+ null+ }+ }++ private fun awaitImage(r: ImageReader, timeoutMs: Long): Image? {+ r.acquireLatestImage()?.let { return it }+ // Poll rather than listen: ImageReader.OnImageAvailableListener fires on+ // a handler we do not own, and a latch here would have to be armed+ // before the frame arrived to not lose it. Polling at this interval+ // costs nothing next to the frame the display is producing.+ val deadline = System.currentTimeMillis() + timeoutMs+ while (System.currentTimeMillis() < deadline) {+ Thread.sleep(POLL_MS)+ r.acquireLatestImage()?.let { return it }+ }+ return null+ }++ /**+ * One `Image` as an ARGB bitmap.+ *+ * The row stride is not `width * 4`: the producer pads rows, so the buffer+ * is copied into a bitmap that is *at least* as wide as the padding+ * demands and then cropped. Copying straight into `w x h` would shear the+ * image diagonally, which is the kind of wrong that still looks like a+ * screenshot until someone reads text off it.+ */+ private fun toBitmap(image: Image, w: Int, h: Int): Bitmap {+ val plane = image.planes[0]+ val pixelStride = plane.pixelStride+ val rowStride = plane.rowStride+ val rowPadding = rowStride - pixelStride * w+ val padded = Bitmap.createBitmap(+ w + rowPadding / pixelStride,+ h,+ Bitmap.Config.ARGB_8888,+ )+ padded.copyPixelsFromBuffer(plane.buffer)+ if (padded.width == w) return padded+ val exact = Bitmap.createBitmap(padded, 0, 0, w, h)+ padded.recycle()+ return exact+ }++ fun release() {+ synchronized(lock) { releaseLocked() }+ }++ private fun releaseLocked() {+ runCatching { reader?.close() }+ runCatching { display?.release() }+ runCatching { projection?.stop() }+ reader = null+ display = null+ projection = null+ }++ companion object {+ private const val TAG = "ScreenCapture"++ /**+ * Longest side of the PNG the brain is handed, whatever the source.+ *+ * Shared with `ScreenController`, which had the number to itself until+ * the projection fallback arrived — two capture paths with two different+ * picture sizes for the same tool is a difference the brain would have+ * to learn for no reason.+ */+ const val SHOT_MAX_SIDE = 1280++ /** How many frames the reader may hold before the producer stalls. */+ private const val IMAGE_BUFFER = 2++ /** A frame is not instant after `start`; this is how long we give it. */+ private const val GRAB_TIMEOUT_MS = 2_000L+ private const val POLL_MS = 25L++ /**+ * Which source a capture came from, in the tool result.+ *+ * Spelled out rather than implied, because the two are not+ * interchangeable: an `accessibility` shot is one the brain may aim+ * with, a `media_projection` one is only to look at.+ */+ const val SOURCE = "source"+ const val SOURCE_ACCESSIBILITY = "accessibility"+ const val SOURCE_MEDIA_PROJECTION = "media_projection"++ /** What the brain is told when a projection shot is all it gets. */+ const val PROJECTION_NOTE =+ "Captured with MediaProjection: this is the raw screen, with no aim " ++ "grid and no element list — both of those need Andee's accessibility " ++ "service, and so does tapping. You can describe what you see; you " ++ "cannot act on it until accessibility is back on."+ }+}diff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.ktindex 3557a83..907c9f3 100644--- a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt@@ -3050,8 +3050,15 @@ class ScreenController(/** The relative coordinate space the brain speaks: 0..1000 both axes. */private const val NORM_MAX = 1000- /** Screenshot long side after downscale (generic_bridge value). */- private const val SHOT_MAX_SIDE = 1280+ /**+ * Screenshot long side after downscale (generic_bridge value).+ *+ * Owned by [ScreenCapture] and read from there, not copied: the+ * MediaProjection fallback hands the brain pictures through the same+ * tool, and a difference in size between the two sources is a+ * difference the brain would have to learn for no reason.+ */+ private const val SHOT_MAX_SIDE = ScreenCapture.SHOT_MAX_SIDE/** Aim-grid columns. Letters run A.. — keep it ≤ 26. */private const val GRID_COLS = 8diff --git a/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt b/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.ktindex 21a61ff..b7ce7d4 100644--- a/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt+++ b/app/src/main/java/net/kuafuai/andee/ui/SelfCheckActivity.kt@@ -4,6 +4,7 @@ import android.graphics.drawable.ColorDrawableimport android.os.Bundleimport android.provider.Settingsimport android.widget.ImageView+import androidx.activity.result.contract.ActivityResultContractsimport net.kuafuai.andee.BodyCoreServiceimport net.kuafuai.andee.ScreenBodyServiceimport net.kuafuai.andee.config.FactoryReset@@ -110,6 +111,19 @@ import net.kuafuai.andee.i18n.AppLocale*/class SelfCheckActivity : StageActivity() {+ companion object {+ /**+ * Start-intent extra: ask for the screen-capture consent as soon as+ * this page is up.+ *+ * Set by the body's notification action, which is the only way in —+ * `MediaProjectionManager.createScreenCaptureIntent()` needs an+ * Activity, and this is the only one the app has. See+ * [requestScreenCapture].+ */+ const val EXTRA_SCREEN_CAPTURE = "screen_capture"+ }+/*** The card. Built here on the service-off path only, placed on [stage], and* gone with the Activity.@@ -143,6 +157,13 @@ class SelfCheckActivity : StageActivity() {override fun onCreate(savedInstanceState: Bundle?) {super.onCreate(savedInstanceState)+ // The notification's 允许看屏幕 action lands here — see+ // [requestScreenCapture]. Asked before the branch below, because the+ // point of the action is that the page it opens is a means to an end.+ if (intent?.getBooleanExtra(EXTRA_SCREEN_CAPTURE, false) == true) {+ requestScreenCapture()+ }+// A working assistant ⇒ this icon means what every other icon on the// home screen means: open the app. That is the *same request* a// long-press on the ball makes, so it is the same call — one entry point@@ -326,6 +347,68 @@ class SelfCheckActivity : StageActivity() {* icon-as-app entry finishes before the card is ever built, and a lifecycle* callback arriving afterwards is not a good moment to throw.*/+ /**+ * The screen-capture consent, and what the answer goes to.+ *+ * Registered as a field rather than from [onCreate] because the lifecycle+ * requires the callback to exist before the Activity is started, and this+ * page is started by an icon tap — there is no earlier moment.+ *+ * The grant goes to the *body*, not to this Activity: the capture has to+ * outlive this page, and on Android 14 it has to be re-asked for every+ * session, so the thing that owns it is the thing that is still running+ * tomorrow. Nothing is announced here either — the body's own notification+ * is what reports this state, and a second voice describing the same fact+ * is how two surfaces start disagreeing.+ */+ private val captureConsent = registerForActivityResult(+ ActivityResultContracts.StartActivityForResult(),+ ) { result ->+ val granted = BodyCoreService.get()+ ?.offerScreenCapture(result.resultCode, result.data) == true+ android.util.Log.i("Body", "screen capture consent: granted=$granted")+ }++ /**+ * Ask the user to let Andee read the screen without accessibility.+ *+ * **Only reachable while the assistant is blind**, which is the only time+ * it is worth a system dialog: with accessibility on, `takeScreenshot` is+ * richer, free and already granted — an aim grid the brain can tap from and+ * the element list beside it — and asking for a second, weaker way in would+ * be asking the user to make a decision with no benefit.+ *+ * The caller is the body's notification action, `Intent` extra+ * [EXTRA_SCREEN_CAPTURE]. Failing to launch is not worth a card: the+ * platform simply has no such screen on some ROMs, and the notification+ * will go on offering it.+ */+ private fun requestScreenCapture() {+ val mgr = getSystemService(android.media.projection.MediaProjectionManager::class.java)+ ?: return+ runCatching { captureConsent.launch(mgr.createScreenCaptureIntent()) }+ .onFailure {+ android.util.Log.w("Body", "screen capture consent would not launch", it)+ }+ }++ /**+ * The other half of [EXTRA_SCREEN_CAPTURE], and not a formality.+ *+ * This page is `singleTop`, and its two callers both routinely arrive while+ * it is already the top instance: the notification action is tapped from+ * the shade with the page up, and the launcher icon does the same. On that+ * path `onCreate` does not run, so an extra read only there would be a+ * button that works the first time and silently stops working afterwards —+ * and the first time is the one where the user had not yet refused.+ * [setIntent] keeps `intent` honest for anything that reads it later.+ */+ override fun onNewIntent(intent: android.content.Intent) {+ super.onNewIntent(intent)+ setIntent(intent)+ if (intent.getBooleanExtra(EXTRA_SCREEN_CAPTURE, false)) requestScreenCapture()+ }+override fun onResume() {super.onResume()if (!::ui.isInitialized) returndiff --git a/app/src/main/res/values-en/strings.xml b/app/src/main/res/values-en/strings.xmlindex 1228a5f..4f7fc90 100644--- a/app/src/main/res/values-en/strings.xml+++ b/app/src/main/res/values-en/strings.xml@@ -361,6 +361,7 @@<string name="body_notify_text_ok">Screen control is up.</string><string name="body_notify_text_no_screen">Accessibility was switched off, so I can\'t reach the screen. Tap the action to get it back.</string><string name="body_notify_action_fix">Get it back</string>+ <string name="body_notify_action_see">Let me see the screen</string><string name="cmd_vault_filling">Typing the “%2$s” for %1$s…</string><string name="cmd_ask_yes">OK</string><string name="cmd_ask_no">No thanks</string>diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xmlindex 1fde1b1..e5c3933 100644--- a/app/src/main/res/values/strings.xml+++ b/app/src/main/res/values/strings.xml@@ -398,6 +398,7 @@<string name="body_notify_text_ok">屏幕操作已就绪。</string><string name="body_notify_text_no_screen">无障碍被系统关掉了,我现在碰不到屏幕。点右边接回来。</string><string name="body_notify_action_fix">接回来</string>+ <string name="body_notify_action_see">允许看屏幕</string><!-- net/CommandDispatcher.kt — the three lines from it a user can see. --><string name="cmd_vault_filling">正在填入「%1$s」的%2$s…</string>