Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。
fix(screen): snap near-miss taps onto the one small shape beside them, and map 0-1000 against the whole display
booboosui70c9a86340bd
diff --git a/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt b/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.ktindex 22385ed..1b7d87b 100644--- a/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt+++ b/app/src/main/java/net/kuafuai/andee/net/ToolSchemas.kt@@ -229,6 +229,12 @@ object ToolSchemas {Use this only when the target has no e-number: self-drawn panes(mini-program, WebView, game), images, pure-visual content. If aget_screen_element list exists for this screen, use tap_screen_element.++ When the app hides its element tree and your point lands on blank+ background right beside one small isolated shape (an icon, a "··"+ button), the device taps that shape's centre instead and says so with+ `snapped_from` / `snapped_to`. Check after_shot as usual — if the shape+ was not what you meant, aim again further away from it.""".trimIndent(),required = emptyList(),props = mapOf(diff --git a/app/src/main/java/net/kuafuai/andee/screen/GroundingLog.kt b/app/src/main/java/net/kuafuai/andee/screen/GroundingLog.ktindex 9be0e09..21abd6b 100644--- a/app/src/main/java/net/kuafuai/andee/screen/GroundingLog.kt+++ b/app/src/main/java/net/kuafuai/andee/screen/GroundingLog.kt@@ -141,8 +141,8 @@ object GroundingLog {note(JSONObject().put("ev", "session")- .put("screen_w", app.resources.displayMetrics.widthPixels)- .put("screen_h", app.resources.displayMetrics.heightPixels)+ .put("screen_w", ScreenController.displaySize(app).x)+ .put("screen_h", ScreenController.displaySize(app).y).put("orientation",if (app.resources.configuration.orientation ==diff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.ktindex be6858e..637def1 100644--- a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt@@ -72,8 +72,12 @@ class ScreenController(// independent. Values outside 0-1000 are REJECTED, not clamped — a silent// clamp to the edge is a mis-tap the model can't even see happening.- private val screenW: Int get() = service.resources.displayMetrics.widthPixels- private val screenH: Int get() = service.resources.displayMetrics.heightPixels+ // The whole display, not displayMetrics: heightPixels leaves out the+ // navigation bar (2523 of 2560 on this tablet), while screenshots and+ // injected gestures both live in the full display. Mapping 0-1000 against+ // the short height put every coordinate tap ~1.4% too high.+ private val screenW: Int get() = displaySize(service).x+ private val screenH: Int get() = displaySize(service).yprivate fun Int.toPixel(max: Int): Int = this * max / NORM_MAX@@ -86,7 +90,7 @@ class ScreenController(return v}- fun tapNorm(nx: Int, ny: Int): JSONObject {+ fun tapNorm(nx: Int, ny: Int, beforePx: FloatArray? = null): JSONObject {requireNorm(nx, "x"); requireNorm(ny, "y")val w = screenW; val h = screenHval px = nx.toPixel(w); val py = ny.toPixel(h)@@ -99,7 +103,7 @@ class ScreenController(GroundingLog.noteTapCoord(nx, ny, px, py, w, h, hold = false, refused = true, pkg = lastPkg())}GroundingLog.noteTapCoord(nx, ny, px, py, w, h, hold = false, refused = false, pkg = lastPkg())- return tap(px, py).put("norm", "$nx,$ny")+ return tap(px, py, beforePx).put("norm", "$nx,$ny")}fun longPressNorm(nx: Int, ny: Int, durationMs: Long): JSONObject {@@ -266,7 +270,177 @@ class ScreenController(fun tapAim(on: String?, x: Int?, y: Int?, cell: String?, part: String?): JSONObject {val a = resolveAim(on, x, y, cell, part)- return tapNorm(a.nx, a.ny).put("aimed_at", a.label)+ val s = if (treeBlind()) snapToShape(a.nx, a.ny) else null+ val nx = s?.nx ?: a.nx+ val ny = s?.ny ?: a.ny+ val r = tapNorm(nx, ny, s?.signature).put("aimed_at", a.label)+ if (s?.moved == true) {+ r.put("snapped_from", "${a.nx},${a.ny}")+ .put("snapped_to", "$nx,$ny")+ .put(+ "snap_note",+ "Your point was on blank background ${s.distPx} px from one small isolated " ++ "shape, so the device tapped that shape's centre instead. If that shape " ++ "was not your target, aim again further from it."+ )+ }+ return r+ }++ private class Snap(val nx: Int, val ny: Int, val moved: Boolean, val distPx: Int, val signature: FloatArray?)++ /** A tree of a handful of nodes says nothing about what is on screen — see [BLIND_TREE_NODES]. */+ private fun treeBlind(): Boolean {+ val root = runCatching { activeRoot() }.getOrNull() ?: return false+ return try {+ countNodes(root, BLIND_TREE_NODES + 1) <= BLIND_TREE_NODES+ } finally {+ root.recycle()+ }+ }++ private fun countNodes(node: AccessibilityNodeInfo, cap: Int): Int {+ var n = 1+ for (i in 0 until node.childCount) {+ if (n >= cap) break+ val c = runCatching { childOf(node, i) }.getOrNull() ?: continue+ try {+ n += countNodes(c, cap - n)+ } finally {+ c.recycle()+ }+ }+ return n+ }++ /**+ * Pull a coordinate tap that landed on blank background onto the one small+ * shape beside it.+ *+ * Measured on WeChat Moments with the tree withheld: the "··" button is+ * ~70×44 px, and four aimed taps in a row missed it — the last one inside+ * its column and 2 px above its top edge. Vision models read *which* thing+ * well and *where exactly* poorly, and a miss by a few pixels onto blank+ * space does nothing at all. So when the point is on uniform background and+ * exactly one compact shape sits within [SNAP_RADIUS_DP], the device taps+ * that shape's centre. Anything less clear-cut — the point is on content,+ * nothing is near, two things are near, the near thing is large or runs out+ * of the search patch — leaves the tap where it was aimed.+ *+ * Only for blind trees: with a real tree the brain has e-numbers and does+ * not need this. The same frame doubles as the before-image for the pixel+ * comparison in [tap], so snapping costs no extra capture there.+ */+ private fun snapToShape(nx: Int, ny: Int): Snap? {+ val mask = ownArea()+ val density = service.resources.displayMetrics.density+ val radius = (SNAP_RADIUS_DP * density).toInt()+ val minSide = (SNAP_MIN_DP * density).toInt().coerceAtLeast(4)+ val maxSide = (SNAP_MAX_DP * density).toInt()+ val half = radius + maxSide+ return captureFrame { hw ->+ val sig = signatureOf(hw, mask)+ val w = hw.width; val h = hw.height+ val tx = nx.toPixel(w); val ty = ny.toPixel(h)+ val unmoved = Snap(nx, ny, false, 0, sig)+ val x0 = (tx - half).coerceAtLeast(0)+ val y0 = (ty - half).coerceAtLeast(0)+ val x1 = (tx + half).coerceAtMost(w)+ val y1 = (ty + half).coerceAtMost(h)+ val pw = x1 - x0; val ph = y1 - y0+ if (pw <= 0 || ph <= 0) return@captureFrame unmoved+ val crop = Bitmap.createBitmap(hw, x0, y0, pw, ph)+ val soft = if (crop.config == Bitmap.Config.HARDWARE) {+ crop.copy(Bitmap.Config.ARGB_8888, false).also { crop.recycle() }+ } else crop+ val px = IntArray(pw * ph)+ soft.getPixels(px, 0, pw, 0, 0, pw, ph)+ soft.recycle()+ val found = findSnapTarget(px, pw, ph, tx - x0, ty - y0, x0, y0, mask, radius, minSide, maxSide)+ ?: return@captureFrame unmoved+ val sx = ((found[0] + x0) * NORM_MAX + w / 2) / w+ val sy = ((found[1] + y0) * NORM_MAX + h / 2) / h+ Snap(sx.coerceIn(0, NORM_MAX), sy.coerceIn(0, NORM_MAX), true, found[2], sig)+ }+ }++ /**+ * Connected components of "not the background" in the patch, joined across+ * gaps of up to one block so the two dots of a "··" count as one shape.+ * Returns {centreX, centreY, distance} in patch pixels, or null.+ */+ private fun findSnapTarget(+ px: IntArray, pw: Int, ph: Int, tx: Int, ty: Int, ox: Int, oy: Int,+ mask: Rect?, radius: Int, minSide: Int, maxSide: Int,+ ): IntArray? {+ if (tx !in 0 until pw || ty !in 0 until ph) return null+ if (mask != null && mask.contains(tx + ox, ty + oy)) return null+ val bg = px[ty * pw + tx]+ fun fg(c: Int): Boolean {+ val d = maxOf(+ kotlin.math.abs(Color.red(c) - Color.red(bg)),+ kotlin.math.abs(Color.green(c) - Color.green(bg)),+ kotlin.math.abs(Color.blue(c) - Color.blue(bg)),+ )+ return d >= SNAP_FG_DELTA+ }+ // Only the pixel itself: the miss this exists for was 2 px off the plate's edge.+ if (fg(px[ty * pw + tx])) return null++ val b = SNAP_BLOCK+ val bw = (pw + b - 1) / b; val bh = (ph + b - 1) / b+ val blockFg = BooleanArray(bw * bh)+ val blockMasked = BooleanArray(bw * bh)+ for (yy in 0 until ph) for (xx in 0 until pw) {+ val bi = (yy / b) * bw + xx / b+ if (mask != null && mask.contains(xx + ox, yy + oy)) blockMasked[bi] = true+ else if (!blockFg[bi] && fg(px[yy * pw + xx])) blockFg[bi] = true+ }++ val label = IntArray(bw * bh) { -1 }+ val queue = IntArray(bw * bh)+ var near = 0+ var pick: IntArray? = null+ var comp = 0+ for (start in 0 until bw * bh) {+ if (!blockFg[start] || label[start] >= 0) continue+ var head = 0; var tail = 0+ queue[tail++] = start; label[start] = comp+ var bx0 = Int.MAX_VALUE; var by0 = Int.MAX_VALUE; var bx1 = -1; var by1 = -1+ var spoiled = false+ while (head < tail) {+ val i = queue[head++]+ val cx = i % bw; val cy = i / bw+ bx0 = minOf(bx0, cx); by0 = minOf(by0, cy); bx1 = maxOf(bx1, cx); by1 = maxOf(by1, cy)+ if (cx == 0 || cy == 0 || cx == bw - 1 || cy == bh - 1) spoiled = true+ for (dy in -SNAP_GAP_BLOCKS..SNAP_GAP_BLOCKS) for (dx in -SNAP_GAP_BLOCKS..SNAP_GAP_BLOCKS) {+ val nxb = cx + dx; val nyb = cy + dy+ if (nxb !in 0 until bw || nyb !in 0 until bh) continue+ val j = nyb * bw + nxb+ if (blockMasked[j]) spoiled = true+ if (blockFg[j] && label[j] < 0) { label[j] = comp; queue[tail++] = j }+ }+ }+ // Pixel-exact bounds of the component, not block-rounded.+ var l = Int.MAX_VALUE; var t = Int.MAX_VALUE; var r = -1; var btm = -1+ for (yy in by0 * b until minOf((by1 + 1) * b, ph)) for (xx in bx0 * b until minOf((bx1 + 1) * b, pw)) {+ if (label[(yy / b) * bw + xx / b] == comp && fg(px[yy * pw + xx])) {+ if (xx < l) l = xx; if (xx > r) r = xx+ if (yy < t) t = yy; if (yy > btm) btm = yy+ }+ }+ comp+++ if (r < 0) continue+ val dx = maxOf(l - tx, 0, tx - r); val dy = maxOf(t - ty, 0, ty - btm)+ val dist = kotlin.math.sqrt((dx * dx + dy * dy).toDouble()).toInt()+ if (dist > radius) continue+ near+++ if (near > 1) return null+ val sw = r - l + 1; val sh = btm - t + 1+ pick = if (spoiled || sw < minSide || sh < minSide || sw > maxSide || sh > maxSide) null+ else intArrayOf((l + r) / 2, (t + btm) / 2, dist)+ }+ return if (near == 1) pick else null}fun longPressAim(@@ -670,7 +844,7 @@ class ScreenController(return null}- fun tap(x: Int, y: Int): JSONObject {+ fun tap(x: Int, y: Int, snapFrame: FloatArray? = null): JSONObject {val path = Path().apply { moveTo(x.toFloat(), y.toFloat()) }// 150ms, not the classic 100ms: WeChat's Moments like/comment buttons// (and other touch-filtering surfaces) ignore injections shorter than@@ -682,7 +856,7 @@ class ScreenController(// a long-press.val stroke = GestureDescription.StrokeDescription(path, 0L, 150L)val before = screenFingerprint()- val beforePx = if (before.optBoolean("blind")) pixelSignature() else null+ val beforePx = if (before.optBoolean("blind")) snapFrame ?: pixelSignature() else nullval result = runGesture(GestureDescription.Builder().addStroke(stroke).build(), 2000L)marker?.mark(x, y, net.kuafuai.andee.ui.TapMarkerUi.Kind.TAP)result.put("x", x).put("y", y)@@ -772,7 +946,11 @@ class ScreenController(*/private fun pixelSignature(): FloatArray? {val mask = ownArea()- return captureFrame { hw ->+ return captureFrame { hw -> signatureOf(hw, mask) }+ }++ private fun signatureOf(hw: Bitmap, mask: Rect?): FloatArray {+ return run {val sw = PX_COLS * PX_SAMPLEval sh = PX_ROWS * PX_SAMPLEval scaled = Bitmap.createScaledBitmap(hw, sw, sh, true)@@ -2732,6 +2910,12 @@ class ScreenController(}companion object {+ /** Full display size in the current rotation, system bars included — the space gestures and screenshots use. */+ fun displaySize(ctx: android.content.Context): android.graphics.Point {+ val b = ctx.getSystemService(android.view.WindowManager::class.java).maximumWindowMetrics.bounds+ return android.graphics.Point(b.width(), b.height())+ }+/*** ADBKeyboard 的广播接口。action 是裸字符串,没有包名前缀 ——* 加了前缀就匹配不上它注册的 filter,广播静默消失。@@ -2798,6 +2982,17 @@ class ScreenController(private const val STATUS_BAR_FRACTION = 0.035fprivate const val CAPTURE_GAP_MS = 350L+ /** See [snapToShape]. ~50 px on this tablet; a miss further than that was not a near miss. */+ private const val SNAP_RADIUS_DP = 24+ private const val SNAP_MIN_DP = 2+ /** Bigger than an icon is content, not a button the tap narrowly missed. */+ private const val SNAP_MAX_DP = 56+ /** Max per-channel difference from the background; WeChat's f7 button plate on ff is 8. */+ private const val SNAP_FG_DELTA = 6+ private const val SNAP_BLOCK = 4+ /** Blocks of gap bridged, so the two dots of "··" are one shape. */+ private const val SNAP_GAP_BLOCKS = 2+/** See [awaitHelper]. Measured: the helper binds in about a second. */private const val HELPER_WAIT_MS = 3_000Lprivate const val HELPER_POLL_MS = 300Ldiff --git a/app/src/main/java/net/kuafuai/andee/ui/TapMarkerUi.kt b/app/src/main/java/net/kuafuai/andee/ui/TapMarkerUi.ktindex 532794b..3eb26fc 100644--- a/app/src/main/java/net/kuafuai/andee/ui/TapMarkerUi.kt+++ b/app/src/main/java/net/kuafuai/andee/ui/TapMarkerUi.kt@@ -148,9 +148,9 @@ class TapMarkerUi(private val context: Context) {// The label is the 0-1000 coordinate rather than the pixel one: that// is the only space the brain writes in, and the one that survives a// downscaled screenshot.- val dm = context.resources.displayMetrics- val nx = x * NORM_MAX / max(1, dm.widthPixels)- val ny = y * NORM_MAX / max(1, dm.heightPixels)+ val d = net.kuafuai.andee.screen.ScreenController.displaySize(context)+ val nx = x * NORM_MAX / max(1, d.x)+ val ny = y * NORM_MAX / max(1, d.y)v.mark(kind.label, "$nx,$ny")showing = trueapplyWindow()