Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。
fix(screen): cap tree walks, and prefetch them on API 33+
booboosuie47bc4d52e59
diff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.ktindex a366a50..43b1938 100644--- a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt@@ -604,7 +604,7 @@ class ScreenController(* worth far more.*/private fun findLiveNode(entry: EidEntry): IntArray? {- val root = runCatching { service.rootInActiveWindow }.getOrNull() ?: return null+ val root = runCatching { activeRoot() }.getOrNull() ?: return nulltry {if (root.packageName?.toString() != entry.pkg) return null// Strong path: resource id narrows it down, but WeChat reuses ids@@ -638,23 +638,25 @@ class ScreenController(matches.forEach { it.recycle() }}}- // Weak path: walk for a node with the exact same bounds.- return findNodeByBounds(root, entry.bounds)+ // Weak path: walk for a node with the exact same bounds. Budgeted:+ // a miss falls back to the remembered centre, which is fine.+ return findNodeByBounds(root, entry.bounds, WalkBudget(SEARCH_BUDGET_MS))} finally {root.recycle()}}- private fun findNodeByBounds(node: AccessibilityNodeInfo, bounds: String): IntArray? {+ private fun findNodeByBounds(node: AccessibilityNodeInfo, bounds: String, budget: WalkBudget): IntArray? {val r = Rect()runCatching { node.getBoundsInScreen(r) }if (r.width() > 0 && r.height() > 0 && "${r.left},${r.top},${r.right},${r.bottom}" == bounds) {return intArrayOf(r.left, r.top, r.right, r.bottom)}for (i in 0 until node.childCount) {- val child = runCatching { node.getChild(i) }.getOrNull() ?: continue+ if (budget.spent()) return null+ val child = runCatching { childOf(node, i) }.getOrNull() ?: continuetry {- val hit = findNodeByBounds(child, bounds)+ val hit = findNodeByBounds(child, bounds, budget)if (hit != null) return hit} finally {child.recycle()@@ -696,11 +698,12 @@ class ScreenController(* taken after a fixed beat, answers "did it land" without any of that.*/private fun screenFingerprint(): JSONObject {- val root = runCatching { service.rootInActiveWindow }.getOrNull()+ val root = runCatching { activeRoot() }.getOrNull()?: return JSONObject().put("pkg", "").put("sig", 0)return try {val f = FingerprintStats()- collectFingerprint(root, f, 0)+ val budget = WalkBudget(FINGERPRINT_BUDGET_MS)+ collectFingerprint(root, f, 0, budget)// Window COUNT is part of the fingerprint on purpose: WeChat's// Moments like/comment bar is a full-screen overlay window that// changes nothing in the active window's node tree — measured in@@ -710,6 +713,9 @@ class ScreenController(JSONObject().put("pkg", root.packageName?.toString().orEmpty()).put("sig", f.sig() * 31 + winCount)+ // Two cut fingerprints stop at different nodes, so their+ // signatures differ whether or not the screen did.+ .apply { if (budget.cut) put("cut", true) }} finally {root.recycle()}@@ -734,13 +740,14 @@ class ScreenController(fun sig(): Int = count * 31 + hash}- private fun collectFingerprint(node: AccessibilityNodeInfo, f: FingerprintStats, depth: Int) {+ private fun collectFingerprint(node: AccessibilityNodeInfo, f: FingerprintStats, depth: Int, budget: WalkBudget) {if (depth > 18 || f.count > 900) return // cap: fingerprint, not a dumpf.visit(node)for (i in 0 until node.childCount) {- val c = runCatching { node.getChild(i) }.getOrNull() ?: continue+ if (budget.spent()) return+ val c = runCatching { childOf(node, i) }.getOrNull() ?: continuetry {- collectFingerprint(c, f, depth + 1)+ collectFingerprint(c, f, depth + 1, budget)} finally {c.recycle()}@@ -779,6 +786,14 @@ class ScreenController(val changed = before.optString("pkg") != after.optString("pkg") ||before.optInt("sig") != after.optInt("sig")result.put("screen_changed", changed)+ if (before.optBoolean("cut") || after.optBoolean("cut")) {+ result.put(+ "screen_changed_note",+ "This app answers accessibility too slowly to compare the screen before and " ++ "after, so screen_changed is a guess — judge whether the tap landed from the " ++ "attached screenshot.",+ )+ }result.put("now_pkg", after.optString("pkg"))val tapX = result.optInt("x")val tapY = result.optInt("y")@@ -810,7 +825,7 @@ class ScreenController(// screenshot above already carries the pixels; this carries the// e-numbers. One tap response = full next-step context.runCatching {- val tree = dumpOnce(false)+ val tree = dumpOnce(false, WalkBudget(ATTACHED_TREE_MS))if (tree.optInt("nodes_useful", 0) > 0) {rebuildIndexAndList(tree)val elements = tree.optString("elements")@@ -819,7 +834,11 @@ class ScreenController(.put("elements_pkg", tree.optString("pkg")).put("elements_hint","This is the POST-TAP screen. Chain your next tap_screen_element / " +- "type_text directly from these e-numbers — do NOT re-dump first.")+ "type_text directly from these e-numbers — do NOT re-dump first." ++ if (tree.optBoolean("walk_cut")) {+ " The list is PARTIAL: this app answered too slowly to read it " ++ "all, so what is in the screenshot but not here was not reached."+ } else "")}}}@@ -1131,14 +1150,15 @@ class ScreenController(private fun readTextFromTree(want: String): String {if (want.isEmpty()) return ""val r = service.rootInActiveWindow ?: return ""+ val budget = WalkBudget(SEARCH_BUDGET_MS)try {val stack = ArrayDeque<AccessibilityNodeInfo>()stack.add(r)- while (stack.isNotEmpty()) {+ while (stack.isNotEmpty() && !budget.spent()) {val n = stack.removeFirst()val t = n.text?.toString().orEmpty()if (t == want) return t- for (i in 0 until n.childCount) n.getChild(i)?.let { stack.add(it) }+ for (i in 0 until n.childCount) childOf(n, i)?.let { stack.add(it) }}} finally {r.recycle()@@ -1233,11 +1253,17 @@ class ScreenController(// genuinely animated — the shot is taken anyway and `still_moving`// says so, rather than pretending.runCatching {- var prev = screenFingerprint().optInt("sig", -1)+ val first = screenFingerprint()+ // A slow app cannot be fingerprinted to a stable answer, and each+ // try would cost a full budget — take the picture as it is.+ if (first.optBoolean("cut")) return@runCatching+ var prev = first.optInt("sig", -1)var tries = 0while (tries < STABLE_MAX_TRIES) {Thread.sleep(STABLE_POLL_MS)- val sig = screenFingerprint().optInt("sig", -2)+ val again = screenFingerprint()+ if (again.optBoolean("cut")) return@runCatching+ val sig = again.optInt("sig", -2)if (sig == prev) return@runCatchingprev = sigtries++@@ -1366,7 +1392,7 @@ class ScreenController(// it comes back empty and costs nothing..also {runCatching {- val tree = dumpOnce(false)+ val tree = dumpOnce(false, WalkBudget(ATTACHED_TREE_MS))if (tree.optInt("nodes_useful", 0) > 0) {rebuildIndexAndList(tree)val elements = tree.optString("elements")@@ -1376,7 +1402,11 @@ class ScreenController(.put("aim_hint","Prefer tap_id(e) / type(id) with these elements — " +"system-computed, always accurate. Pixel-aiming " +- "from the image is the fallback, not the default.")+ "from the image is the fallback, not the default." ++ if (tree.optBoolean("walk_cut")) {+ " The list is PARTIAL: this app answered too slowly " ++ "to read it all, so aim at what is missing from the image."+ } else "")}}}@@ -1561,14 +1591,17 @@ class ScreenController(capture: (String) -> JSONObject = { screenshot(it) },): JSONObject {val ts = System.currentTimeMillis()- var out = dumpOnce(verbose)+ val budget = WalkBudget(TREE_BUDGET_MS)+ val firstStart = SystemClock.uptimeMillis()+ var out = dumpOnce(verbose, budget)+ val firstMs = SystemClock.uptimeMillis() - firstStartvar waited = 0L- while (!verbose && out.optInt("nodes_useful", 0) == 0 && waited < SETTLE_BUDGET_MS) {+ while (!verbose && out.optInt("nodes_useful", 0) == 0 && waited < SETTLE_BUDGET_MS && !budget.spent()) {val age = System.currentTimeMillis() - lastContentEventMs.get()if (lastContentEventMs.get() != 0L && age >= QUIET_WINDOW_MS) breaktry { Thread.sleep(SETTLE_POLL_MS) } catch (_: InterruptedException) { break }waited += SETTLE_POLL_MS- out = dumpOnce(verbose)+ out = dumpOnce(verbose, budget)}// Stability pass — the half the old loop never covered: "has content"// is not "has stopped moving". A screen mid-animation (list settling,@@ -1579,13 +1612,20 @@ class ScreenController(// still is exactly one extra pass — and that pass is also what// guarantees the eid index the brain is about to read was built from// the settled geometry, not a mid-flight one.- if (!verbose && out.optInt("nodes_useful", 0) > 0) {+ //+ // Skipped for a slow app: each re-dump is another full walk, and a+ // re-dump cut short by the budget is kept out of `out` — a partial+ // tree must never replace a whole one just because it came second.+ if (!verbose && out.optInt("nodes_useful", 0) > 0 &&+ !out.optBoolean("walk_cut") && firstMs < STABLE_SKIP_MS+ ) {var prevSig = treeSignature(out)var stableTries = 0while (waited < SETTLE_BUDGET_MS && stableTries < STABLE_MAX_TRIES) {try { Thread.sleep(STABLE_POLL_MS) } catch (_: InterruptedException) { break }waited += STABLE_POLL_MS- val again = dumpOnce(verbose)+ val again = dumpOnce(verbose, budget)+ if (again.optBoolean("walk_cut")) breakval sig = treeSignature(again)if (sig == prevSig) {out = again@@ -1667,8 +1707,15 @@ class ScreenController(// so the caller can blank our overlay first — see CommandDispatcher.val treeEmpty = out.optInt("nodes_useful", 0) == 0 ||out.optString("pkg") == service.packageName- if (!verbose && (withShot || treeEmpty)) {- val reason = if (withShot) "with_shot" else "auto_empty"+ // A walk the budget cut short is the same case in part: whatever it+ // did not reach exists only in the pixels.+ val walkCut = out.optBoolean("walk_cut")+ if (!verbose && (withShot || treeEmpty || walkCut)) {+ val reason = when {+ withShot -> "with_shot"+ treeEmpty -> "auto_empty"+ else -> "auto_partial"+ }runCatching { out.put("shot", capture(reason)) }.onFailure { out.put("shot_error", it.message ?: "capture failed") }if (!withShot) out.put("shot_auto", true)@@ -1697,11 +1744,11 @@ class ScreenController(return sb.toString()}- private fun dumpOnce(verbose: Boolean): JSONObject {- val active = service.rootInActiveWindow+ private fun dumpOnce(verbose: Boolean, budget: WalkBudget): JSONObject {+ val active = activeRoot()val out = if (active != null) {try {- val stats = Stats()+ val stats = Stats(budget)val body = buildNode(active, verbose, stats) ?: JSONObject().put("empty", true)withStats(body, active.packageName?.toString() ?: "", stats)} finally {@@ -1716,13 +1763,13 @@ class ScreenController(// window (WeChat's mini-program / web-view pannes do exactly this).// Also entered when the "active" window turned out to be a status-bar// or other system shell during an app transition.- if (!verbose && out.optInt("nodes_useful", 0) == 0) tryOtherWindows(out)+ if (!verbose && out.optInt("nodes_useful", 0) == 0 && !budget.spent()) tryOtherWindows(out, budget)// A status bar (SystemUI) can be the active window mid-transition and// hand us ~27 "useful" status icons — real nodes, wrong screen. The// user-facing app is always a better source when it has ANY content.// Only run this correction when we did NOT just come from// tryOtherWindows with a genuinely better window.- if (!verbose && out.optString("pkg") == "com.android.systemui") tryOtherWindows(out)+ if (!verbose && out.optString("pkg") == "com.android.systemui" && !budget.spent()) tryOtherWindows(out, budget)return out}@@ -1740,7 +1787,7 @@ class ScreenController(* was found, so the next person debugging this (or the model) can see which* window existed and which one had content, instead of a bare `{}`.*/- private fun tryOtherWindows(out: JSONObject) {+ private fun tryOtherWindows(out: JSONObject, budget: WalkBudget) {val windows = runCatching { service.windows }.getOrNull()if (windows == null) {out.put("windows_seen", 0)@@ -1755,8 +1802,9 @@ class ScreenController(var bestUseful = out.optInt("nodes_useful", 0)try {for (w in windows) {- val stats = Stats()- val root = runCatching { w.root }.getOrNull()+ if (budget.spent()) break+ val stats = Stats(budget)+ val root = runCatching { rootOf(w) }.getOrNull()// Read everything we need off the node *before* recycling it.val pkg = if (root != null) runCatching { root.packageName?.toString() }.getOrNull().orEmpty() else ""val body = if (root != null) {@@ -2058,6 +2106,7 @@ class ScreenController(body.put("nodes_kept", stats.kept)body.put("nodes_useful", stats.useful)stats.surface?.let { body.put("surface", shortenClass(it, false)) }+ if (stats.budget.cut) body.put("walk_cut", true)if (source != null) body.put("source", source)return body}@@ -2071,13 +2120,53 @@ class ScreenController(* [useful] — nodes carrying something you can act on (text / desc / id)* [surface]— first self-rendering view found, if any*/- private class Stats {+ private class Stats(val budget: WalkBudget = WalkBudget.UNLIMITED) {var total = 0var kept = 0var useful = 0var surface: String? = null}+ /**+ * Wall-clock cap on one tree walk. Every `getChild` is a binder round trip+ * answered on the *target app's* UI thread, so an app busy animating+ * answers each one late — 红果's 福利 page measured 551 nodes in 135 s,+ * ~245 ms a node — and a walk with no cap outlives the brain's tool deadline while+ * still holding the app's attention. A cut walk reports itself+ * (`walk_cut`) so the caller can say the list is partial, not absent.+ */+ private class WalkBudget(ms: Long) {+ private val until = if (ms == Long.MAX_VALUE) Long.MAX_VALUE else SystemClock.uptimeMillis() + ms+ var cut = false+ private set++ fun spent(): Boolean {+ if (!cut && SystemClock.uptimeMillis() >= until) cut = true+ return cut+ }++ companion object {+ val UNLIMITED get() = WalkBudget(Long.MAX_VALUE)+ }+ }++ /**+ * API 33+ lets one round trip carry up to 50 descendants back with the+ * node asked for. `UNINTERRUPTIBLE` is the half that matters for a walk:+ * by default the app abandons the prefetch the moment our next request+ * arrives, and a depth-first walk sends the next request immediately —+ * so most nodes came back one IPC each. Older versions keep the plain call.+ */+ private fun childOf(node: AccessibilityNodeInfo, i: Int): AccessibilityNodeInfo? =+ if (android.os.Build.VERSION.SDK_INT >= 33) node.getChild(i, PREFETCH_WALK) else node.getChild(i)++ private fun activeRoot(): AccessibilityNodeInfo? =+ if (android.os.Build.VERSION.SDK_INT >= 33) service.getRootInActiveWindow(PREFETCH_WALK)+ else service.rootInActiveWindow++ private fun rootOf(w: AccessibilityWindowInfo): AccessibilityNodeInfo? =+ if (android.os.Build.VERSION.SDK_INT >= 33) w.getRoot(PREFETCH_WALK) else w.root+/*** Tell the brain what it is looking at when there is nothing to act on, in* its language rather than ours: not "empty tree" but "this is a surface you@@ -2204,6 +2293,19 @@ class ScreenController("line covers it.",)}+ // Independent of the verdict above: a cut walk can still have produced+ // a perfectly good partial list, and the model needs both facts.+ if (out.optBoolean("walk_cut")) {+ out.put(+ "walk_note",+ "This app answered accessibility too slowly (it is busy animating), so reading " ++ "stopped after ${TREE_BUDGET_MS / 1000}s with only part of the screen listed. " ++ "The e-numbers above are real and tappable; anything you see in $eyes but not " ++ "in the list was simply not reached — aim at it with tap_by_coordinates. " ++ "Calling this tool again will be just as slow. Nothing is broken; do not tell " ++ "the user you cannot see the screen.",+ )+ }}/**@@ -2306,7 +2408,8 @@ class ScreenController(val kids = mutableListOf<JSONObject>()for (i in 0 until node.childCount) {- val child = node.getChild(i) ?: continue+ if (st != null && st.budget.spent()) break+ val child = childOf(node, i) ?: continuetry {buildNode(child, verbose, stats)?.let { kids.add(it) }} finally {@@ -2464,6 +2567,23 @@ class ScreenController(private const val STABLE_MAX_TRIES = 3private const val QUIET_WINDOW_MS = 1200L+ /** See [childOf]. Compile-time ints, only ever passed on API 33+. */+ @android.annotation.SuppressLint("InlinedApi")+ private const val PREFETCH_WALK = AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_HYBRID or+ AccessibilityNodeInfo.FLAG_PREFETCH_UNINTERRUPTIBLE++ /**+ * Walk caps — see [WalkBudget]. A whole `get_screen_element`, re-dumps+ * included, fits in [TREE_BUDGET_MS]; a first pass slower than+ * [STABLE_SKIP_MS] skips the stability re-dumps, which on a slow app+ * would cost a full walk each to learn what the screenshot shows anyway.+ */+ private const val TREE_BUDGET_MS = 8_000L+ private const val STABLE_SKIP_MS = 2_000L+ private const val ATTACHED_TREE_MS = 4_000L+ private const val FINGERPRINT_BUDGET_MS = 1_500L+ private const val SEARCH_BUDGET_MS = 1_500L+/** The relative coordinate space the brain speaks: 0..1000 both axes. */private const val NORM_MAX = 1000