Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

fix(screen): stop the ball from eating taps and misleading the brain

booboosuiefb454f6ebb7
diff --git a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
index 32f8160..a07a4aa 100644
--- a/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
+++ b/app/src/main/java/net/kuafuai/andee/net/CommandDispatcher.kt
@@ -117,6 +117,26 @@ class CommandDispatcher(
) {
private val ui = Handler(Looper.getMainLooper())
+ init {
+ // The effect snapshot after a tap is taken inside the gesture passthrough,
+ // where the ball is untouchable but still drawn. Hand the screen the same
+ // blanking every other capture gets, so there is one copy of how long to
+ // wait and which windows to hide.
+ screen.shotGuard = { shot -> blankedForCapture(shot) }
+ // Taps that would land under the ball window: park it off-screen for the
+ // duration. `passthroughForGesture` already made the window untouchable,
+ // but an untouchable overlay is still obscuring, and payment apps drop
+ // obscured taps — Alipay's bottom tabs reproduced this on 2026-10-09.
+ screen.gestureGuard = { target, body ->
+ val wraps = window.windowBounds()?.let { android.graphics.Rect.intersects(it, target) } ?: false
+ if (!wraps) body()
+ else {
+ window.parkForGesture()
+ try { body() } finally { window.unparkForGesture() }
+ }
+ }
+ }
+
/** Leak guard, not a guess about thinking time — see [setBusy]. */
private val endDriving = Runnable { setBusy(false) }
diff --git a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt
index 637def1..3557a83 100644
--- a/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt
+++ b/app/src/main/java/net/kuafuai/andee/screen/ScreenController.kt
@@ -47,6 +47,45 @@ class ScreenController(
private val ownArea: () -> Rect? = { null },
) {
+ /**
+ * Runs a capture with our own windows blanked. Set by the dispatcher, which
+ * owns the overlays; null where there are none, and then the capture just runs.
+ *
+ * Needed because the effect snapshot after a tap is taken from inside [tap],
+ * i.e. inside the dispatcher's gesture passthrough, which only makes the ball
+ * untouchable and leaves it drawn. Without this the brain's `after_shot` shows
+ * the ball over whatever sits in the bottom-right corner — a field report had
+ * it read the ball as covering Alipay's 我的 tab and abandon the route.
+ */
+ @Volatile
+ var shotGuard: ((() -> JSONObject) -> JSONObject)? = null
+
+ private fun guardedShot(shot: () -> JSONObject): JSONObject =
+ shotGuard?.invoke(shot) ?: shot()
+
+ /**
+ * Wraps a single injected gesture — [tap], [longPress], [swipe] — so the
+ * dispatcher can get the ball window out of the way when the stroke would
+ * land under it. The target rect is the stroke's bounding box in pixels;
+ * the wrapper decides whether moving the overlay is worth it.
+ *
+ * Measured on this hardware (2026-10-09, Alipay bottom tabs): without this,
+ * taps that landed inside the ball window's 168 dp frame returned
+ * `completed:true, screen_changed:true` from the accessibility layer but the
+ * target app never saw them — Android marks a stroke "obscured" when any
+ * untrusted overlay sits on it, and payment apps drop obscured taps. Making
+ * the overlay non-touchable (what `passthroughForGesture` already does) is
+ * not enough: an invisible non-touchable overlay still contributes to the
+ * obscuration ceiling.
+ *
+ * Null in a caller with no overlay to park.
+ */
+ @Volatile
+ var gestureGuard: ((android.graphics.Rect, () -> JSONObject) -> JSONObject)? = null
+
+ private fun guardedGesture(target: android.graphics.Rect, body: () -> JSONObject): JSONObject =
+ gestureGuard?.invoke(target, body) ?: body()
+
/** Last meaningful accessibility event from a non-overlay app. */
private val lastContentEventMs = AtomicLong(0L)
@@ -857,7 +896,9 @@ class ScreenController(
val stroke = GestureDescription.StrokeDescription(path, 0L, 150L)
val before = screenFingerprint()
val beforePx = if (before.optBoolean("blind")) snapFrame ?: pixelSignature() else null
- val result = runGesture(GestureDescription.Builder().addStroke(stroke).build(), 2000L)
+ val result = guardedGesture(android.graphics.Rect(x - 4, y - 4, x + 4, y + 4)) {
+ runGesture(GestureDescription.Builder().addStroke(stroke).build(), 2000L)
+ }
marker?.mark(x, y, net.kuafuai.andee.ui.TapMarkerUi.Kind.TAP)
result.put("x", x).put("y", y)
attachEffectSnapshot(result, before, beforePx)
@@ -1111,7 +1152,7 @@ class ScreenController(
// reveal, then give the post a beat to land.
marker?.revealNow()
Thread.sleep(80)
- result.put("after_shot", screenshot("tap_after"))
+ result.put("after_shot", guardedShot { screenshot("tap_after") })
}
}
// The NEW screen's element list, so the brain can chain its next
@@ -1159,7 +1200,7 @@ class ScreenController(
val ry = (tapY * 1000f / screenH).toInt()
val size = 300
val hUnits = (size * screenH.toFloat() / screenW).toInt().coerceAtLeast(1)
- result.put("miss_shot", lookRegion(rx, ry, size, hUnits))
+ result.put("miss_shot", guardedShot { lookRegion(rx, ry, size, hUnits) })
}
}
}
@@ -1187,13 +1228,22 @@ class ScreenController(
lineTo(x2.toFloat(), y2.toFloat())
}
val stroke = GestureDescription.StrokeDescription(path, 0L, durationMs)
- return runGesture(GestureDescription.Builder().addStroke(stroke).build(), durationMs + 2000).put("from", "$x1,$y1").put("to", "$x2,$y2").put("duration_ms", durationMs)
+ val target = android.graphics.Rect(
+ minOf(x1, x2) - 4, minOf(y1, y2) - 4,
+ maxOf(x1, x2) + 4, maxOf(y1, y2) + 4,
+ )
+ val result = guardedGesture(target) {
+ runGesture(GestureDescription.Builder().addStroke(stroke).build(), durationMs + 2000)
+ }
+ return result.put("from", "$x1,$y1").put("to", "$x2,$y2").put("duration_ms", durationMs)
}
fun longPress(x: Int, y: Int, durationMs: Long): JSONObject {
val path = Path().apply { moveTo(x.toFloat(), y.toFloat()) }
val stroke = GestureDescription.StrokeDescription(path, 0L, durationMs)
- val result = runGesture(GestureDescription.Builder().addStroke(stroke).build(), durationMs + 2000)
+ val result = guardedGesture(android.graphics.Rect(x - 4, y - 4, x + 4, y + 4)) {
+ runGesture(GestureDescription.Builder().addStroke(stroke).build(), durationMs + 2000)
+ }
marker?.mark(x, y, net.kuafuai.andee.ui.TapMarkerUi.Kind.HOLD)
return result.put("x", x).put("y", y).put("duration_ms", durationMs)
}
diff --git a/app/src/main/java/net/kuafuai/andee/ui/FloatingWindowUi.kt b/app/src/main/java/net/kuafuai/andee/ui/FloatingWindowUi.kt
index 619f7bb..f6c5766 100644
--- a/app/src/main/java/net/kuafuai/andee/ui/FloatingWindowUi.kt
+++ b/app/src/main/java/net/kuafuai/andee/ui/FloatingWindowUi.kt
@@ -734,6 +734,7 @@ class FloatingWindowUi(
return try {
ui.removeCallbacks(strollRunnable)
slide?.cancel()
+ parkedFrom = null
wm.removeView(v)
root = null
params = null
@@ -967,6 +968,74 @@ class FloatingWindowUi(
}
}
+ /** Where the window sat before [parkForGesture] moved it; null when not parked. Main thread only. */
+ private var parkedFrom: IntArray? = null
+
+ /**
+ * Move the folded ball's window off the left edge of the screen so an
+ * injected gesture can pass through where it sat. Blocks the caller (never
+ * the main thread) until the move has been applied and had a frame or two to
+ * reach the compositor, or [PARK_WAIT_MS] has passed.
+ *
+ * Why not just `FLAG_NOT_TOUCHABLE`, which [setTouchable] already sets: a
+ * non-touchable overlay still *obscures*. Measured on a Huawei with Alipay
+ * (2026-10-09), `dispatchGesture` on the bottom tabs reported
+ * `completed:true` for every tap, and the tabs inside the ball window's frame
+ * never switched while the ones outside it did — including 消息, which sits in
+ * the transparent part of the frame, clear of the drawn sphere. The platform
+ * flags a stroke that passes under an untrusted overlay as obscured, and a
+ * payment app is entitled to drop such a stroke without saying so.
+ *
+ * No-op unless folded: the unfolded card is a full-screen surface and moving
+ * it would take the whole card off the display. The dispatcher folds before
+ * every `screen.*` call, so that case is a passive observer, not a driver.
+ */
+ fun parkForGesture() {
+ val latch = java.util.concurrent.CountDownLatch(1)
+ var parked = false
+ ui.post {
+ try {
+ val v = root
+ val p = params
+ if (v != null && p != null && compact && parkedFrom == null) {
+ // A slide in flight would keep writing p.x/p.y over the parked
+ // position, and restoring a mid-flight number would strand the
+ // ball half-way between two resting places.
+ slide?.end()
+ parkedFrom = intArrayOf(p.x, p.y)
+ p.x = -p.width - dp(32)
+ runCatching { wm.updateViewLayout(v, p) }
+ parked = true
+ }
+ } finally {
+ latch.countDown()
+ }
+ }
+ if (!latch.await(PARK_WAIT_MS, java.util.concurrent.TimeUnit.MILLISECONDS)) return
+ // `updateViewLayout` is posted to the compositor; the same settle the
+ // flag flip in [setTouchable] needs, and for the same reason.
+ if (parked) Thread.sleep(PARK_SETTLE_MS)
+ }
+
+ /**
+ * Put the window back where [parkForGesture] found it. Not blocking: nothing
+ * the caller does next depends on the ball being visible again. Idempotent,
+ * and a no-op when something reshaped the window in the meantime (a fold or
+ * unfold writes its own geometry, and that is the newer fact).
+ */
+ fun unparkForGesture() {
+ ui.post {
+ val from = parkedFrom ?: return@post
+ parkedFrom = null
+ val v = root ?: return@post
+ val p = params ?: return@post
+ if (!compact || p.width != dp(COMPACT_DP)) return@post
+ p.x = from[0]
+ p.y = from[1]
+ runCatching { wm.updateViewLayout(v, p) }
+ }
+ }
+
/**
* Vanish while the typing field has the user in another app's camera or
* photo picker. The card still holds its typing height, which puts it over
@@ -2712,6 +2781,12 @@ class FloatingWindowUi(
private const val COMPACT_DP = 168
private const val COMPACT_MARGIN_DP = 16
+ /** How long [parkForGesture] waits for the main-thread move to run. */
+ private const val PARK_WAIT_MS = 200L
+
+ /** Compositor settle after [parkForGesture]'s window move. */
+ private const val PARK_SETTLE_MS = 50L
+
/**
* How much of the ball area's padding a held signboard's tail reaches
* into, as a fraction of the compact side. The sphere occupies ~64% of