Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(mobile): 修复语音输入启动时按钮闪烁 (#4899)

Dash8b1d01dbede1
diff --git a/apps/mobile/app/sessions/[sessionId].tsx b/apps/mobile/app/sessions/[sessionId].tsx
index f7e53799fd..8aa48fc34d 100644
--- a/apps/mobile/app/sessions/[sessionId].tsx
+++ b/apps/mobile/app/sessions/[sessionId].tsx
@@ -1438,6 +1438,8 @@ export default function SessionScreen() {
}
voiceStateTransitionRef.current = next;
setVoiceStateInternal(next);
+ // 首段音频到达时与 listening 同批交接;停止、取消和错误也在这里收回 pending。
+ setVoiceStartPending(false);
}, []);
const [searchOpen, setSearchOpen] = useState(false);
const [settingsOpen, setSettingsOpen] = useState(false);
@@ -5120,8 +5122,13 @@ export default function SessionScreen() {
if (selection) input?.rememberSelection(text, selection);
writeVoiceDraft({ draft: text, initialDocument, initialSelection, insertionEnd: selection?.end, replacement });
},
- onStateChanged: setVoiceState,
+ onStateChanged: (next) => {
+ // 旧 controller 的取消可晚于新启动完成,只允许本次启动交接 UI 状态。
+ if (voiceStartupSeqRef.current !== startupSeq) return;
+ setVoiceState(next);
+ },
onError: (message) => {
+ if (voiceStartupSeqRef.current !== startupSeq) return;
setVoiceState('error');
setVoiceError(message);
},
@@ -5281,7 +5288,8 @@ export default function SessionScreen() {
voiceDictionaryLearningTrackerRef.current?.dispose();
// 语音结束 hold 属于上一个会话的输入现场;切会话时连同迁移基点一并复位,
// 被 cancel 的 run 迟到的状态回调不会在新会话上误布防。
- voiceStateTransitionRef.current = 'idle';
+ // 不依赖旧 controller 的迟到 done 回调来复位新页面。
+ setVoiceState('idle');
setComposerVoiceHoldArmed(false);
if (controller) void controller.cancel().catch(() => undefined);
discardPendingPrewarm();
@@ -5387,6 +5395,9 @@ export default function SessionScreen() {
void startVoiceRecording()
.catch(() => undefined)
.finally(() => {
+ // start() 可早于首段 PCM 返回。已有录音时由 setVoiceState 接续胶囊,
+ // 只有未起录的取消/失败才在这里收回,避免 pending → idle → listening 闪烁。
+ if (voiceRecordingActiveRef.current) return;
// 只收自己世代的 pending:切会话后旧启动的收尾不能塌掉新录音的胶囊。
if (voiceStartPendingSeqRef.current === pendingSeq) setVoiceStartPending(false);
});
@@ -10683,6 +10694,7 @@ function SessionComposerInput({
const composerCardActive = (canUseComposer && composerFocused)
|| modelSheetOpen
|| permissionSheetOpen
+ || voiceStartPending
|| voiceIsBusy
|| composerVoiceHoldActive;
const recommendationPresentation = usePromptRecommendationVisibility(sessionId, draft, composerCardActive);
diff --git a/apps/mobile/app/sessions/new.tsx b/apps/mobile/app/sessions/new.tsx
index cfe912bc42..a75f29a8e7 100644
--- a/apps/mobile/app/sessions/new.tsx
+++ b/apps/mobile/app/sessions/new.tsx
@@ -749,6 +749,7 @@ export default function NewRemoteSessionScreen() {
const [atPaletteLoading, setAtPaletteLoading] = useState(false);
const [atPaletteError, setAtPaletteError] = useState<string | null>(null);
const [atResourcesTruncated, setAtResourcesTruncated] = useState(false);
+ const [voiceStartPending, setVoiceStartPending] = useState(false);
const [voiceState, setVoiceStateInternal] = useState<MobileVoiceState>('idle');
const [voiceError, setVoiceError] = useState<string | null>(null);
// 「语音结束保持展开」hold:语音真实收尾(busy → done/error)时布防,草稿仍有
@@ -764,6 +765,8 @@ export default function NewRemoteSessionScreen() {
}
voiceStateTransitionRef.current = next;
setVoiceStateInternal(next);
+ // 首段音频到达时与 listening 同批交接;停止、取消和错误也在这里收回 pending。
+ setVoiceStartPending(false);
}, []);
const browseSeqRef = useRef(0);
const capabilitiesSeqRef = useRef(0);
@@ -1478,8 +1481,6 @@ export default function NewRemoteSessionScreen() {
);
const composerHasMessage = draft.firstMessage.trim().length > 0;
// 「按下即录」的乐观反馈(与会话页/桌面同款,详见 [sessionId].tsx 同名状态注释)。
- // 声明在 composerShowCreateButton 之前:pending 期就要占住创建槽。
- const [voiceStartPending, setVoiceStartPending] = useState(false);
const voiceStartPendingSeqRef = useRef(0);
const voiceStartedOnPressInRef = useRef(false);
// 语音生命周期内创建按钮常驻(与会话页发送槽同理,对齐桌面):录音中点创建
@@ -1559,6 +1560,7 @@ export default function NewRemoteSessionScreen() {
const composerCardActive = firstMessageInputFocused
|| modelSheetOpen
|| permissionSheetOpen
+ || voiceStartPending
|| voiceIsBusy
|| composerVoiceHoldActive;
useComposerCardTransition(composerCardActive, keyboardState);
@@ -3292,8 +3294,13 @@ export default function NewRemoteSessionScreen() {
}
setFirstMessageDraft(text);
},
- onStateChanged: setVoiceState,
+ onStateChanged: (next) => {
+ // 旧 controller 的取消可晚于新启动完成,只允许本次启动交接 UI 状态。
+ if (voiceStartupSeqRef.current !== startupSeq) return;
+ setVoiceState(next);
+ },
onError: (message) => {
+ if (voiceStartupSeqRef.current !== startupSeq) return;
setVoiceState('error');
setVoiceError(message);
},
@@ -3433,6 +3440,9 @@ export default function NewRemoteSessionScreen() {
void startVoiceRecording()
.catch(() => undefined)
.finally(() => {
+ // start() 可早于首段 PCM 返回。已有录音时由 setVoiceState 接续胶囊,
+ // 只有未起录的取消/失败才在这里收回,避免 pending → idle → listening 闪烁。
+ if (voiceRecordingActiveRef.current) return;
// 只收自己世代的 pending(与会话页同款守卫)。
if (voiceStartPendingSeqRef.current === pendingSeq) setVoiceStartPending(false);
});
diff --git a/apps/mobile/src/__tests__/composerAudioCleanup.test.ts b/apps/mobile/src/__tests__/composerAudioCleanup.test.ts
index 6d45864d0c..75bca1c026 100644
--- a/apps/mobile/src/__tests__/composerAudioCleanup.test.ts
+++ b/apps/mobile/src/__tests__/composerAudioCleanup.test.ts
@@ -60,6 +60,7 @@ function pageCleanup(page: string, recording: boolean, starting: boolean) {
voiceStopAfterStartRef: { current: false },
voiceDictionaryLearningTrackerRef: { current: null },
voiceStateTransitionRef: { current: "idle" },
+ setVoiceState: vi.fn(),
setComposerVoiceHoldArmed: vi.fn(),
discardPendingPrewarm,
setAudioModeAsync,
@@ -100,6 +101,9 @@ describe.each(["new.tsx", "[sessionId].tsx"])("%s audio cleanup", (page) => {
expect(run.discardPendingPrewarm).toHaveBeenCalledOnce();
expect(run.bindings.voiceControllerSessionRef).toEqual({ current: null });
expect(run.bindings.voiceStartupSeqRef).toEqual({ current: 2 });
+ if (page === "[sessionId].tsx") {
+ expect(run.bindings.setVoiceState).toHaveBeenCalledWith("idle");
+ }
},
);
});
diff --git a/apps/mobile/src/__tests__/composerVoiceStartupFeedback.test.ts b/apps/mobile/src/__tests__/composerVoiceStartupFeedback.test.ts
new file mode 100644
index 0000000000..a748717497
--- /dev/null
+++ b/apps/mobile/src/__tests__/composerVoiceStartupFeedback.test.ts
@@ -0,0 +1,202 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import ts from 'typescript';
+import { describe, expect, it, vi } from 'vitest';
+import { shouldArmComposerVoiceHold } from '@/session/composerVoiceHold';
+import type { MobileVoiceState } from '@/session/mobileVoiceInput';
+
+// Execute the production page callbacks, as in composerAudioCleanup.test.ts,
+// without mounting the pages' unrelated remote services and native controls.
+function readCallbacks(page: string) {
+ const source = ts.createSourceFile(page,
+ readFileSync(resolve(process.cwd(), 'app/sessions', page), 'utf8'),
+ ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX);
+ const callbacks = new Map<string, string>();
+ let cardActive = '';
+ function visit(node: ts.Node) {
+ if (ts.isVariableDeclaration(node) && node.name.getText(source) === 'composerCardActive' && node.initializer) {
+ cardActive = node.initializer.getText(source);
+ }
+ if (ts.isCallExpression(node) && node.expression.getText(source) === 'createMobileVoiceControllerSession') {
+ const options = node.arguments[0];
+ if (ts.isObjectLiteralExpression(options)) {
+ for (const prop of options.properties) {
+ if (ts.isPropertyAssignment(prop) && ['onStateChanged', 'onError'].includes(prop.name.getText(source))) {
+ callbacks.set(prop.name.getText(source), prop.initializer.getText(source));
+ }
+ }
+ }
+ }
+ if (ts.isVariableDeclaration(node) && node.initializer
+ && ts.isCallExpression(node.initializer)
+ && node.initializer.expression.getText(source) === 'useCallback') {
+ callbacks.set(node.name.getText(source), node.initializer.arguments[0].getText(source));
+ }
+ ts.forEachChild(node, visit);
+ }
+ visit(source);
+ return (bindings: Record<string, unknown>) => {
+ const names = ['handleVoiceButtonPressIn', 'setVoiceState', 'onStateChanged', 'onError'];
+ const compiled = ts.transpileModule(
+ `return { ${names.map((name) => {
+ if (!callbacks.has(name)) throw new Error(`Missing ${page} callback: ${name}`);
+ return `${name}: ${callbacks.get(name)}`;
+ }).join(',')}, cardActive: (voiceStartPending, voiceIsBusy) => {
+ const canUseComposer = true;
+ const composerFocused = false, firstMessageInputFocused = false;
+ const modelSheetOpen = false, permissionSheetOpen = false, composerVoiceHoldActive = false;
+ return ${cardActive};
+ } };`,
+ { compilerOptions: { target: ts.ScriptTarget.ES2022 } },
+ ).outputText;
+ return new Function(...Object.keys(bindings), compiled)(...Object.values(bindings)) as {
+ handleVoiceButtonPressIn(): void;
+ setVoiceState(state: MobileVoiceState): void;
+ onStateChanged(state: MobileVoiceState): void;
+ onError(message: string): void;
+ cardActive(pending: boolean, busy: boolean): boolean;
+ };
+ };
+}
+
+function deferred() {
+ let resolve!: () => void;
+ let reject!: (error: Error) => void;
+ const promise = new Promise<void>((ok, fail) => { resolve = ok; reject = fail; });
+ return { promise, resolve, reject };
+}
+
+async function settleCallbacks() {
+ // startVoiceRecording -> catch -> finally, without introducing a timer.
+ for (let i = 0; i < 5; i += 1) await Promise.resolve();
+}
+
+describe.each(['new.tsx', '[sessionId].tsx'])('%s voice startup feedback', (page) => {
+ const callbacks = readCallbacks(page);
+ function setup() {
+ let pending = false;
+ let state: MobileVoiceState = 'idle';
+ let startup = deferred();
+ const recording = { current: false };
+ const pendingSeq = { current: 0 };
+ const startupSeq = { current: 1 };
+ const setVoiceError = vi.fn();
+ const run = callbacks({
+ startupSeq: 1, voiceStartupSeqRef: startupSeq, setVoiceError,
+ setVoiceState: (value: MobileVoiceState) => run.setVoiceState(value),
+ creating: false, voiceIsProcessing: false, voiceState: 'idle',
+ selectedDeviceId: 'host', deviceId: 'host',
+ isMobileRealtimeAudioAvailable: () => true,
+ prewarmMobileRealtimeAudio: vi.fn(), prewarmMobileVoiceStart: vi.fn(),
+ auth: { apiFetch: vi.fn() },
+ voiceStartedOnPressInRef: { current: false },
+ voiceRecordingActiveRef: recording,
+ voiceStartupInFlightRef: { current: false }, voiceStopInFlightRef: { current: false },
+ voiceStartPendingSeqRef: pendingSeq,
+ setVoiceStartPending: (value: boolean) => { pending = value; },
+ startVoiceRecording: () => startup.promise,
+ voiceStateTransitionRef: { current: 'idle' },
+ shouldArmComposerVoiceHold, setComposerVoiceHoldArmed: vi.fn(),
+ setVoiceStateInternal: (value: MobileVoiceState) => { state = value; },
+ });
+ return {
+ press: run.handleVoiceButtonPressIn, state: run.setVoiceState, recording,
+ controllerState: run.onStateChanged, controllerError: run.onError, startupSeq, setVoiceError,
+ startup: () => startup,
+ nextStartup: () => { startup = deferred(); },
+ cardExpanded: () => run.cardActive(pending, state === 'listening' || state === 'submitting' || state === 'refining'),
+ view: () => ({ expanded: pending || state === 'listening', counting: state === 'listening', pending }),
+ };
+ }
+
+ it.each([false, true])('stays expanded across startup and first PCM (PCM first=%s)', async (pcmFirst) => {
+ const run = setup();
+ run.press();
+ expect(run.view()).toEqual({ expanded: true, counting: false, pending: true });
+ run.recording.current = true;
+ if (pcmFirst) run.state('listening');
+ run.startup().resolve();
+ await settleCallbacks();
+ expect(run.view().expanded).toBe(true);
+ expect(run.view().counting).toBe(pcmFirst);
+ run.state('listening');
+ expect(run.view()).toEqual({ expanded: true, counting: true, pending: false });
+ });
+
+ it('expands a collapsed composer with the capsule on press, before the first PCM', async () => {
+ const run = setup();
+ expect(run.cardExpanded()).toBe(false);
+ run.press();
+ expect(run.cardExpanded()).toBe(true);
+ expect(run.view()).toEqual({ expanded: true, counting: false, pending: true });
+ run.recording.current = true;
+ run.startup().resolve();
+ await settleCallbacks();
+ expect(run.cardExpanded()).toBe(true);
+ expect(run.view().expanded).toBe(true);
+ run.state('listening');
+ expect(run.cardExpanded()).toBe(true);
+ expect(run.view()).toEqual({ expanded: true, counting: true, pending: false });
+ run.state('idle');
+ expect(run.cardExpanded()).toBe(false);
+ });
+
+ it.each(['idle', 'error', 'submitting', 'done'] as const)(
+ 'clears startup feedback on %s before the first PCM', async (state) => {
+ const run = setup();
+ run.press();
+ run.recording.current = true;
+ run.startup().resolve();
+ await settleCallbacks();
+ run.state(state);
+ expect(run.view()).toEqual({ expanded: false, counting: false, pending: false });
+ },
+ );
+
+ it.each([false, true])('clears feedback if startup creates no recording (reject=%s)', async (reject) => {
+ const run = setup();
+ run.press();
+ if (reject) run.startup().reject(new Error('permission/startup failed'));
+ else run.startup().resolve(); // permission cancelled or unavailable
+ await settleCallbacks();
+ expect(run.view()).toEqual({ expanded: false, counting: false, pending: false });
+ });
+
+ it('does not let an old startup completion collapse a new pending press', async () => {
+ const run = setup();
+ run.press();
+ const oldStartup = run.startup();
+ run.state('idle');
+ run.nextStartup();
+ run.press();
+ oldStartup.resolve();
+ await settleCallbacks();
+ expect(run.view()).toEqual({ expanded: true, counting: false, pending: true });
+ run.startup().resolve();
+ await settleCallbacks();
+ expect(run.view().expanded).toBe(false);
+ });
+
+ it('ignores late controller states and errors after a new startup claims the page', () => {
+ const run = setup();
+ run.press();
+ run.controllerState('listening');
+ expect(run.view()).toEqual({ expanded: true, counting: true, pending: false });
+ run.state('idle');
+ run.startupSeq.current += 1; // task/device switch invalidates the old controller
+ run.nextStartup();
+ run.press();
+ for (const state of ['done', 'error', 'listening'] as const) run.controllerState(state);
+ run.controllerError('old capture failed during cancellation');
+ expect(run.view()).toEqual({ expanded: true, counting: false, pending: true });
+ expect(run.setVoiceError).not.toHaveBeenCalled();
+ });
+
+ it('still surfaces an error from the current controller', () => {
+ const run = setup();
+ run.press();
+ run.controllerError('capture failed');
+ expect(run.view()).toEqual({ expanded: false, counting: false, pending: false });
+ expect(run.setVoiceError).toHaveBeenCalledWith('capture failed');
+ });
+});