Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(voice): exclude rejected speech from short phrase context

DavidShen912bfcdcdf60
diff --git a/packages/voice-input-core/src/SpeakerGate.ts b/packages/voice-input-core/src/SpeakerGate.ts
index 666f71c28e..665c711b29 100644
--- a/packages/voice-input-core/src/SpeakerGate.ts
+++ b/packages/voice-input-core/src/SpeakerGate.ts
@@ -156,7 +156,7 @@ export class SpeakerGate implements AudioGate {
private inSpeech = false;
private pendingStart = 0;
private lastAcceptedEnd = -1;
- // Most recent judged speech, used as embedding context across short pauses.
+ // Most recent accepted speech, used as context until a window is rejected.
private speechTail: Float32Array;
private speechTailLength = 0;
@@ -291,12 +291,12 @@ export class SpeakerGate implements AudioGate {
private async decide(token: number, start: number, end: number): Promise<void> {
const speech = this.readHeld(start, end);
- this.appendSpeechTail(speech);
// VAD frame boundaries can make this window longer than speechTail's
- // capacity. Preserve the whole window; history only pads short windows.
+ // capacity. Preserve the whole window; only accepted history may pad a
+ // short one. Do not commit the current speech until it has been judged.
const context = speech.length >= this.minVerifySamples
? speech
- : this.speechTailSlice(this.minVerifySamples);
+ : concatFloat(this.speechTailSlice(this.minVerifySamples - speech.length), speech);
const startedAt = now();
const embedding = await this.withTimeout(this.engine.embed(context));
if (token !== this.runToken || this.passthrough) return;
@@ -304,8 +304,15 @@ export class SpeakerGate implements AudioGate {
const score = embedding ? cosineSimilarity(embedding, this.enrolled) : -1;
const continuing = this.lastAcceptedEnd >= 0 && start <= this.lastAcceptedEnd;
const accepted = score >= (continuing ? this.continueThreshold : this.threshold);
- if (accepted) this.lastAcceptedEnd = end;
- else this.muteHeld(start, end);
+ if (accepted) {
+ this.lastAcceptedEnd = end;
+ this.appendSpeechTail(speech);
+ } else {
+ // A rejected speaker breaks continuity, including older accepted audio.
+ this.lastAcceptedEnd = -1;
+ this.speechTailLength = 0;
+ this.muteHeld(start, end);
+ }
this.onDecision?.({ startSample: start, endSample: end, score, accepted, computeMs });
this.pendingStart = end;
this.releaseUpTo(end);
diff --git a/packages/voice-input-core/src/__tests__/SpeakerGate.test.ts b/packages/voice-input-core/src/__tests__/SpeakerGate.test.ts
index 785afd727d..10fb26ecc9 100644
--- a/packages/voice-input-core/src/__tests__/SpeakerGate.test.ts
+++ b/packages/voice-input-core/src/__tests__/SpeakerGate.test.ts
@@ -147,6 +147,52 @@ describe('SpeakerGate', () => {
);
});
+ it.each(['different speaker', 'unusable embedding'] as const)(
+ 'does not pad the owner\'s next short phrase with a rejected %s', async rejection => {
+ const engine = new FakeEngine();
+ if (rejection === 'unusable embedding') {
+ const embed = engine.embed.bind(engine);
+ engine.embed = async samples => {
+ const result = await embed(samples);
+ return result?.[1] === 1 ? null : result;
+ };
+ }
+ const { gate, emitted, decisions, passthrough } = startGate(engine);
+ // Exact VAD frames: a rejected full window and remainder, a brief pause,
+ // then an owner's phrase too short to fill minVerifySec on its own.
+ const audio = makeAudio([[BYSTANDER, 1.6], [0, 0.064], [OWNER, 0.256]]);
+ pushAll(gate, audio);
+ await gate.finish();
+
+ expect(decisions[0].accepted).toBe(false);
+ const last = decisions[decisions.length - 1];
+ expect(last.accepted).toBe(true);
+ const current = Float32Array.from(audio.slice(last.startSample, last.endSample), sample => sample / 32768);
+ expect(engine.embedInputs[engine.embedInputs.length - 1]).toEqual(current);
+ const out = joined(emitted);
+ expect(out.length).toBe(audio.length);
+ expect(countLevel(out, 0, 1.6, BYSTANDER)).toBe(0);
+ expect(countLevel(out, 1.664, 1.92, OWNER)).toBe(Math.round(0.256 * SPEAKER_GATE_SAMPLE_RATE));
+ expect(passthrough).toEqual([]);
+ },
+ );
+
+ it('clears older accepted context when a different speaker interrupts', async () => {
+ const engine = new FakeEngine();
+ const { gate, emitted, decisions } = startGate(engine);
+ const audio = makeAudio([[OWNER, 1.6], [0, 0.064], [BYSTANDER, 1.6], [0, 0.064], [BYSTANDER, 0.256]]);
+ pushAll(gate, audio);
+ await gate.finish();
+
+ expect(decisions[0].accepted).toBe(true);
+ expect(decisions[decisions.length - 1].accepted).toBe(false);
+ expect(engine.embedInputs[engine.embedInputs.length - 1].some(sample => sample > 0)).toBe(false);
+ const out = joined(emitted);
+ expect(out.length).toBe(audio.length);
+ expect(countLevel(out, 0, 1.6, OWNER)).toBe(Math.round(1.6 * SPEAKER_GATE_SAMPLE_RATE));
+ expect(out.some(sample => sample === BYSTANDER)).toBe(false);
+ });
+
it('does not release new audio when an aborted finish resumes', async () => {
let release!: (value: Float32Array) => void;
const engine = new FakeEngine();