Skip to content

armgit / andee

Public Imported from GitHub
0

Andee lives on your phone or Mac and uses any app like you do—no integrations, no root, any model; memory and passwords never leave your device | Andee 住进你的手机或 Mac,像你一样操作任何 App——无需集成、无需 Root、支持任意模型,记忆和密码绝不离开设备。

fix(wake): start the segment on the voice, not on the microphone

booboosui4f4ccb5e8242
diff --git a/app/src/main/java/net/kuafuai/andee/ui/WakeEnrollUi.kt b/app/src/main/java/net/kuafuai/andee/ui/WakeEnrollUi.kt
index c6d37e2..2b14a7b 100644
--- a/app/src/main/java/net/kuafuai/andee/ui/WakeEnrollUi.kt
+++ b/app/src/main/java/net/kuafuai/andee/ui/WakeEnrollUi.kt
@@ -6,6 +6,7 @@ import android.graphics.Color
import android.graphics.PixelFormat
import android.os.Handler
import android.os.Looper
+import android.util.Log
import android.view.Gravity
import android.view.View
import android.view.WindowManager
@@ -188,7 +189,11 @@ class WakeEnrollUi(
if (busy) return
busy = true
recordBtn.text = lctx.getString(R.string.wake_listening)
- status.text = lctx.getString(R.string.wake_speak_now)
+ // Deliberately *not* 现在说吧 here. Saying it on the tapper's thread, before
+ // a recorder exists, is what made "tap and talk" fail: the user was
+ // invited to speak during the 100–300 ms the microphone spends opening,
+ // and the onset went into a device that was not yet listening. The
+ // prompt moves to [captureOne], after `startRecording` has returned.
exec.execute {
// The listener has to let go first: it owns a recorder whenever the
// gate allows, and two captures in one app is not a thing to rely on.
@@ -206,10 +211,41 @@ class WakeEnrollUi(
if (got.compareAndSet(null, pcm)) done.countDown()
}
val mic = WakeMic { buf, n -> seg.feed(buf, n) }
- if (!mic.start()) return null
- runCatching { done.await(LISTEN_SECONDS, TimeUnit.SECONDS) }
+ if (!mic.start()) {
+ Log.w(TAG, "enroll take: the device refused the microphone")
+ return null
+ }
+ // The recorder is up, so *now* the user is invited to speak. Waiting for
+ // this rather than for the tap is the honest half of the trade: the card
+ // reads 正在听… for the 100–300 ms the microphone takes to come up, and
+ // 现在说吧 only once something is listening. Nothing is taken from the
+ // user to pay for it — the window is unchanged and the 300 ms pre-roll
+ // still catches an onset that beats the prompt.
+ ui.post {
+ if (root != null) status.text = lctx.getString(R.string.wake_speak_now)
+ }
+ val inTime = runCatching { done.await(LISTEN_SECONDS, TimeUnit.SECONDS) }
+ .getOrDefault(false)
mic.stop()
- return got.get()
+ val pcm = got.get()
+ // One line per attempt, whether or not it worked.
+ //
+ // Four different things answer "nothing" here — a segment dropped for
+ // being too short, one dropped for being too long, a capture thread that
+ // gave up on a bad read, and a noise floor that sat above the speaker
+ // for the entire window — and until this line, not one of them left a
+ // trace anywhere. A user who had to say the phrase twice therefore
+ // looked exactly like a user who was never heard at all, from every log
+ // we had. [VoiceSegmenter.stats] says which of the four it was, and
+ // this is the only place it is read.
+ Log.i(
+ TAG,
+ "enroll take: " +
+ (if (pcm == null) "no segment" else "${pcm.size * 1000 / Mfcc.SAMPLE_RATE} ms") +
+ (if (inTime) "" else " after waiting out the full window") +
+ " — ${seg.stats()}",
+ )
+ return pcm
}
@SuppressLint("SetTextI18n")
@@ -254,5 +290,11 @@ class WakeEnrollUi(
private companion object {
/** Long enough to fumble the first attempt, short enough not to feel hung. */
const val LISTEN_SECONDS = 8L
+
+ /**
+ * The wake pipeline's tag, not this card's own name, so the whole thing
+ * answers to one filter: `adb logcat -s Wake:V`. See [captureOne].
+ */
+ const val TAG = "Wake"
}
}
diff --git a/app/src/main/java/net/kuafuai/andee/wake/VoiceSegmenter.kt b/app/src/main/java/net/kuafuai/andee/wake/VoiceSegmenter.kt
index 2978358..45e16e4 100644
--- a/app/src/main/java/net/kuafuai/andee/wake/VoiceSegmenter.kt
+++ b/app/src/main/java/net/kuafuai/andee/wake/VoiceSegmenter.kt
@@ -19,7 +19,10 @@ import kotlin.math.sqrt
* Pre-roll ([PREROLL_FRAMES]) matters more than it looks: a voice crosses the
* threshold a few frames *after* it starts, and the missing onset is the
* consonant — the 'h' of "嘿" — which is most of what distinguishes the wake
- * phrase from every other short utterance.
+ * phrase from every other short utterance. It is also what buys the start
+ * *decision* its slowness without costing the user a syllable: the detector may
+ * take 80 ms to make up its mind ([START_FRAMES]) because those 80 ms are still
+ * in the ring. Delaying the decision is free; discarding the audio is not.
*
* Not thread-safe; it belongs to whichever thread is reading the microphone.
*/
@@ -33,6 +36,17 @@ class VoiceSegmenter(private val onSegment: (ShortArray) -> Unit) {
private var loudRun = 0
private var quietRun = 0
+ // Counters, for [stats] and nothing else. Deliberately not logging from
+ // here: this class is the DSP half and carries no Android dependency — see
+ // [Mfcc] and [Dtw]. The caller owns the log.
+ private var seenFrames = 0
+ private var loudFrames = 0
+ private var firstLoud = -1
+ private var peakRms = 0f
+ private var emitted = 0
+ private var droppedShort = 0
+ private var droppedLong = 0
+
/** Pre-roll ring, and the utterance once one starts. */
private val preroll = ArrayDeque<ShortArray>()
private var collected = ArrayList<ShortArray>()
@@ -60,6 +74,72 @@ class VoiceSegmenter(private val onSegment: (ShortArray) -> Unit) {
preroll.clear()
collected = ArrayList()
floor = -1f
+ // The counters describe a capture window, and this ends one — see [stats].
+ seenFrames = 0
+ loudFrames = 0
+ firstLoud = -1
+ peakRms = 0f
+ emitted = 0
+ droppedShort = 0
+ droppedLong = 0
+ }
+
+ /**
+ * What this segmenter has seen since construction, or since the last [reset].
+ *
+ * It exists for one question, and it is the question that turned out to be
+ * unanswerable from the outside: *why did a capture window produce no
+ * segment?* Four paths answer "nothing", and every one of them used to do it
+ * without a word anywhere — the two guards below, [WakeMic]'s capture thread
+ * giving up on a bad read, and the floor sitting above the speaker for the
+ * whole window. From a log that mentions none of them, "the user said it and
+ * nothing happened" is indistinguishable from "the user never spoke", which
+ * is how this stayed invisible.
+ *
+ * `loudFrames == 0` on a window with real [frames] is the last of those four:
+ * audio arrived and the detector never saw a syllable, which is a floor
+ * problem and not a drop. Anything else shows up in the three counts.
+ */
+ fun stats(): Stats = Stats(
+ frames = seenFrames,
+ loudFrames = loudFrames,
+ firstLoud = firstLoud,
+ peakRms = peakRms,
+ floorRms = if (floor < 0f) 0f else floor,
+ segments = emitted,
+ droppedShort = droppedShort,
+ droppedLong = droppedLong,
+ )
+
+ /**
+ * @param frames how many 10 ms frames arrived at all.
+ * @param loudFrames how many of them crossed the start line — the one that
+ * separates "a drop ate it" from "the floor was above the speaker".
+ * @param firstLoud which frame crossed first, 0-based, or -1 for never. This
+ * is the number that settles the case the rest of them only describe: a
+ * window whose first loud frame is at index 2 did not start on the user,
+ * it started on the microphone, and the segment that follows is long,
+ * dirty and — past [MAX_FRAMES] — thrown away. Measured values are 0–5 for
+ * that and 30-plus for a window that waited for the voice.
+ * @param peakRms / @param floorRms the two numbers that decide whether
+ * anything ever could have, in 16-bit units.
+ * @param segments / @param droppedShort / @param droppedLong what happened
+ * to the utterances that formed.
+ */
+ class Stats(
+ val frames: Int,
+ val loudFrames: Int,
+ val firstLoud: Int,
+ val peakRms: Float,
+ val floorRms: Float,
+ val segments: Int,
+ val droppedShort: Int,
+ val droppedLong: Int,
+ ) {
+ override fun toString(): String =
+ "frames=$frames loud=$loudFrames firstLoud=$firstLoud peak=${peakRms.toInt()} " +
+ "floor=${floorRms.toInt()} segments=$segments droppedShort=$droppedShort " +
+ "droppedLong=$droppedLong"
}
private fun onFrame() {
@@ -71,6 +151,13 @@ class VoiceSegmenter(private val onSegment: (ShortArray) -> Unit) {
val startAt = maxOf(floor * START_MULT, ABS_MIN)
val endAt = maxOf(floor * END_MULT, ABS_MIN * 0.7f)
+ seenFrames++
+ if (rms > peakRms) peakRms = rms
+ if (rms > startAt) {
+ loudFrames++
+ if (firstLoud < 0) firstLoud = seenFrames - 1
+ }
+
if (!speech) {
// Only while idle, and only downward-biased: rising slowly means a
// fan that starts up is eventually accepted as silence, while a
@@ -109,6 +196,7 @@ class VoiceSegmenter(private val onSegment: (ShortArray) -> Unit) {
// is a coin toss, and the user talking to somebody else in the room is
// by far the most common way to get here.
if (collected.size > MAX_FRAMES) {
+ droppedLong++
speech = false
loudRun = 0
quietRun = 0
@@ -123,7 +211,11 @@ class VoiceSegmenter(private val onSegment: (ShortArray) -> Unit) {
quietRun = 0
collected = ArrayList()
preroll.clear()
- if (frames.size < MIN_FRAMES) return
+ if (frames.size < MIN_FRAMES) {
+ droppedShort++
+ return
+ }
+ emitted++
val out = ShortArray(frames.size * FRAME)
var i = 0
for (f in frames) {
@@ -137,11 +229,58 @@ class VoiceSegmenter(private val onSegment: (ShortArray) -> Unit) {
/** 10 ms at 16 kHz — same grid as [Mfcc.HOP], so segment bounds land on frames. */
const val FRAME = Mfcc.HOP
- const val PREROLL_FRAMES = 15 // 150 ms of run-up
- const val START_FRAMES = 3 // 30 ms above the line before it counts
- const val END_FRAMES = 30 // 300 ms of quiet ends the utterance
- const val MIN_FRAMES = 35 // 350 ms — a word shorter than this is dropped, so it cannot be enrolled
- const val MAX_FRAMES = 180 // 1.8 s — longer than it can be
+ /**
+ * 300 ms of run-up, and it is load-bearing twice over.
+ *
+ * The original reason: a voice crosses the line a few frames *after* it
+ * starts, and the missing onset is the consonant — the 'h' of "嘿" —
+ * which is most of what tells the wake phrase from every other short
+ * utterance. 150 ms was judged enough for that.
+ *
+ * The second reason is the one that turned out to matter: this ring is
+ * what lets the start decision be *slow* without throwing audio away.
+ * [START_FRAMES] has to be long enough that a transient cannot open a
+ * segment, and whatever that decision costs in delay is paid out of
+ * this ring rather than out of the user's first syllable. Measured on
+ * the tablet: a user who taps and speaks immediately was losing the
+ * onset to a segment that had started on the microphone's own
+ * warm-up — see [MAX_FRAMES].
+ */
+ const val PREROLL_FRAMES = 30
+
+ /**
+ * 80 ms above the line before an utterance counts as started. Was 30 ms.
+ *
+ * 30 ms is three frames, which is a click, a tap on the tablet's edge,
+ * or the microphone's first read after `startRecording`. Every one of
+ * those opened a segment, and a segment opened early runs long — the
+ * first take of every enrolment measured on this device came out
+ * *longer* than the two after it, and the extra was not speech.
+ */
+ const val START_FRAMES = 8
+
+ /** 300 ms of quiet ends the utterance. */
+ const val END_FRAMES = 30
+
+ /**
+ * 500 ms. Raised with [PREROLL_FRAMES] so the floor still means what it
+ * meant: this is `preroll + 20 frames`, the same allowance as the old
+ * 35 against a 15-frame ring, so the guard keeps its old reach rather
+ * than quietly becoming unreachable.
+ */
+ const val MIN_FRAMES = 50
+
+ /**
+ * 1.95 s — longer than the phrase can be.
+ *
+ * Raised by exactly the 15 frames [PREROLL_FRAMES] gained. The headroom
+ * that matters is `MAX_FRAMES - PREROLL_FRAMES - END_FRAMES` — the
+ * speech itself — and this keeps it at its old 135 frames. Getting this
+ * wrong in the other direction would have re-created, in a new place,
+ * the very bug the diagnostics were added to find: a segment dropped for
+ * being too long, in silence, with the user convinced they had said it.
+ */
+ const val MAX_FRAMES = 195
const val START_MULT = 3.5f
const val END_MULT = 2.0f
diff --git a/app/src/main/java/net/kuafuai/andee/wake/WakeMic.kt b/app/src/main/java/net/kuafuai/andee/wake/WakeMic.kt
index 5bf2611..b792e75 100644
--- a/app/src/main/java/net/kuafuai/andee/wake/WakeMic.kt
+++ b/app/src/main/java/net/kuafuai/andee/wake/WakeMic.kt
@@ -64,7 +64,18 @@ class WakeMic(private val onPcm: (ShortArray, Int) -> Unit) {
try {
while (running.get()) {
val n = rec.read(buf, 0, buf.size)
- if (n <= 0) break
+ if (n <= 0) {
+ // Used to be a bare `break`, which made this the
+ // quietest way for a capture to fail: the caller's
+ // latch simply never counts down, so a microphone
+ // that gave up after one bad read looks exactly
+ // like a user who never spoke. `ERROR_DEAD_OBJECT`
+ // (-6) here is the mic being taken away by someone
+ // else; `ERROR_INVALID_OPERATION` (-3) is reading a
+ // record that never really started.
+ Log.w(TAG, "read returned $n — capture ending early")
+ break
+ }
onPcm(buf, n)
}
} catch (t: Throwable) {
diff --git a/app/src/main/java/net/kuafuai/andee/wake/WakeTemplates.kt b/app/src/main/java/net/kuafuai/andee/wake/WakeTemplates.kt
index 6fb030c..8c5664d 100644
--- a/app/src/main/java/net/kuafuai/andee/wake/WakeTemplates.kt
+++ b/app/src/main/java/net/kuafuai/andee/wake/WakeTemplates.kt
@@ -30,9 +30,12 @@ object WakeTemplates {
const val ENROLL_COUNT = 3
/**
- * Slack over the user's own worst pair. Below ~1.1 the phrase has to be
- * said more consistently than anyone says anything; above ~1.4 the door is
- * open wide enough for unrelated speech to walk in.
+ * Slack over the user's own *closest* pair. See [thresholdFor] for why it is
+ * the closest and not the worst, and for the measurements that decided it.
+ *
+ * Below ~1.1 the phrase has to be said more consistently than anyone says
+ * anything; above ~1.4 the door is open wide enough for unrelated speech to
+ * walk in.
*/
private const val SLACK = 1.25f
@@ -49,14 +52,21 @@ object WakeTemplates {
* opens the microphone and ships audio to 火山 without anyone asking, which
* is both a surprise and a cost. So when in doubt, be deaf.
*
- * These numbers are a starting point, not a measurement — a second take of
- * the same phrase lands well under 1.0 on synthetic audio, but real voices
- * spread wider and no real recording has been through this yet. The
+ * **The upper bound is no longer a guess.** It was 2.8, which was picked as
+ * "the widest thing that could still be called a threshold" and never
+ * measured — and the first sixty judgements this device ever logged put
+ * every non-wake utterance it heard at **2.21–2.59**, i.e. entirely *below*
+ * it. 2.8 was not a bound at all, it was a door left open. 1.8 comes from
+ * the same sixty: the widest value that would still have rejected all
+ * fifty-seven of them while accepting all three real attempts (1.16–1.46).
+ *
+ * One voice, one room, one phrase — thin evidence, and the honest reading of
+ * it is "better founded than the number it replaces", not "correct". The
* `no wake: d=… > …` line in [WakeWord] is what a near miss looks like in
- * logcat, and is the evidence to move them on.
+ * logcat, and is still the evidence to move these on.
*/
private const val MIN_THRESHOLD = 1.0f
- private const val MAX_THRESHOLD = 2.8f
+ private const val MAX_THRESHOLD = 1.8f
class Store(val threshold: Float, val templates: List<Array<FloatArray>>)
@@ -131,23 +141,61 @@ object WakeTemplates {
loaded = false
}
- /** Worst distance between any two takes, plus [SLACK], clamped. */
+ /**
+ * The distance between the two most alike takes, times [SLACK], clamped.
+ *
+ * **The closest pair, not the worst** — a correction the device paid for.
+ * The worst pair was the obvious choice: it is the honest answer to "how
+ * much does this user vary?". It is also the wrong question, because it is
+ * not what matching does. [bestDistance] returns the *nearest* template, so
+ * an utterance is accepted when it lands close to some take. The width the
+ * door needs is therefore "how far can a real attempt be from the nearest
+ * take", and the only evidence in the file for that is the takes that did
+ * land close.
+ *
+ * Two enrolments measured on the tablet — same voice, same room, twenty
+ * minutes apart, and the same logs:
+ *
+ * ```
+ * worst pair 1.82 / 1.69 → thresholds 2.28 / 2.11 (what shipped)
+ * best pair 1.40 / 1.30 → thresholds 1.75 / 1.62 (what this computes)
+ * ```
+ *
+ * The same voice's *actual* attempts, from logcat, landed at **1.16, 1.38,
+ * 1.46** — nearer even than the tightest enrolment pair. A wake attempt is
+ * made deliberately; an enrolment take is made while reading a card and
+ * reaching for a button. Anchoring on the worst take was anchoring on the
+ * one sample nobody wants to recognise.
+ *
+ * Where the impostors were is the other half: **2.21–2.59**, every one of
+ * the fifty-seven. So 2.28 was not a threshold standing between two
+ * distributions, it was a line through the middle of one of them — sixteen
+ * of the sixty judgements sat within 0.06 of it.
+ *
+ * The residual risk is the documented one, and it is why the enrolment card
+ * tells the user to say the phrase the way they will actually say it: three
+ * takes that come out *unusually* alike give a tight door and a phrase that
+ * has to be repeated carefully forever. [MIN_THRESHOLD] bounds that; the
+ * alternative it replaces — a door so wide the room walks in — costs the
+ * user an upload to 火山 they never asked for, which is the worse side of
+ * the asymmetry the bounds are built on.
+ */
private fun thresholdFor(templates: List<Array<FloatArray>>): Float {
- var worst = 0f
+ var best = Float.MAX_VALUE
var pairs = 0
for (i in templates.indices) {
for (j in i + 1 until templates.size) {
val d = Dtw.distance(templates[i], templates[j])
if (d == Dtw.NO_MATCH) continue
pairs++
- if (d > worst) worst = d
+ if (d < best) best = d
}
}
// No comparable pair at all — the takes differed so much in length that
// DTW refused them. Nothing was learned, so fall back to the middle of
// the useful range rather than to a number computed from nothing.
if (pairs == 0) return (MIN_THRESHOLD + MAX_THRESHOLD) / 2f
- return (worst * SLACK).coerceIn(MIN_THRESHOLD, MAX_THRESHOLD)
+ return (best * SLACK).coerceIn(MIN_THRESHOLD, MAX_THRESHOLD)
}
/**