Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix: 对齐提问卡片的语音输入交互

Dasheb4b3483430b
diff --git a/apps/desktop/scripts/fixtures/ask-user-question/ask-user-question-fixture.tsx b/apps/desktop/scripts/fixtures/ask-user-question/ask-user-question-fixture.tsx
index 1e7120739c..65b209c8ae 100644
--- a/apps/desktop/scripts/fixtures/ask-user-question/ask-user-question-fixture.tsx
+++ b/apps/desktop/scripts/fixtures/ask-user-question/ask-user-question-fixture.tsx
@@ -1,5 +1,6 @@
import { useEffect, useRef, useState } from 'react';
-import { mountPage, forbid } from '../../ui-page/harness';
+import { mountPage, forbid, subscription } from '../../ui-page/harness';
+import { getDefaultVoiceInputSettings } from '../../../src/shared/voiceInputData';
import { AskUserQuestionPrompt } from '../../../src/renderer/components/new-chat/AskUserQuestionPrompt';
import type { AskUserDraft, AskUserViewerState } from '../../../src/renderer/lib/makerChatStore';
import type { VoiceQuestionTarget } from '../../../src/renderer/components/new-chat/voiceQuestionSelection';
@@ -34,6 +35,18 @@ function Fixture({ question, voice, multi }: { question: string; voice?: boolean
void mountPage({
host: {
+ voiceInput: {
+ getDataSnapshot: () => ({ settings: { ...getDefaultVoiceInputSettings('darwin'), playInteractionSound: false, muteSystemAudio: false } }),
+ getMicrophonePermissionCached: () => ({ ok: true, status: 'granted' }),
+ getSystemPermissionsCached: () => ({ microphone: { ok: true }, accessibility: { ok: true } }),
+ getReadinessCached: () => ({ ok: true }),
+ getHistoryForRefinement: () => [],
+ onEvent: subscription('voice'),
+ start: async () => ({ ok: true, runId: 'fixture-voice' }),
+ stop: async () => ({ ok: true }),
+ cancel: async () => ({ ok: true }),
+ drainAudioQueue: async () => ({ ok: true }),
+ },
resolvePathBatch: async ({ hrefs }: { hrefs: string[] }) => Object.fromEntries(hrefs.map((href) => [href,
href === 'missing.html' ? { status: 'none', candidates: [] }
: href === 'ambiguous.html' ? { status: 'multiple', candidates: ['/project/a/ambiguous.html', '/project/b/ambiguous.html'] }
diff --git a/apps/desktop/scripts/fixtures/ask-user-question/ui-page.mjs b/apps/desktop/scripts/fixtures/ask-user-question/ui-page.mjs
index 8ad4bcf7d7..7a683ee4da 100644
--- a/apps/desktop/scripts/fixtures/ask-user-question/ui-page.mjs
+++ b/apps/desktop/scripts/fixtures/ask-user-question/ui-page.mjs
@@ -7,13 +7,21 @@ export default {
real: 'AskUserQuestionPrompt, voice choice draft/confirmation, CompanionVoiceToolbar, question references, prompt shell and themes',
notCovered: 'Actual microphone/ASR/TTS, task execution, host file contents, browser navigation and remote connectivity',
// The visual runner uses esbuild, which does not implement Vite ?url assets.
- // Keep the real dictation hook; this fixture must never start audio capture.
+ // Keep the real dictation hook/state UI; only the microphone and host are fake.
stubs: {
- 'src/renderer/voice-input/workletUrl': 'export function getVoiceInputWorkletUrl() { throw new Error("Audio capture is outside this visual fixture"); }',
+ 'src/renderer/voice-input/workletUrl': 'export function getVoiceInputWorkletUrl() { return "/fixture-worklet.js"; }',
+ 'src/renderer/voice-input/captureSession': `export async function startVoiceInputCaptureSession(options) {
+ options.setEngine({ drainBufferedAudio: async () => {}, stop: async () => {} });
+ if (!window.__uiPage.ctx.case.endsWith('starting')) options.onCaptureReady?.();
+ return { ok: true, drainPendingChunks: () => {} };
+ }`,
},
cases: {
default: { about: 'Question with a bare HTML filename and URL', data: { question: '请查看 sidebar-layout-v1.html 后确认这版交互形态是否可以进入正式实现。也可以查看 https://example.com/prototype。' } },
narrow: { about: 'Long references at 320 px', viewport: { width: 320, height: 600 }, data: { question: '请查看 sidebar-layout-v1.html 或 https://example.com/prototypes/a-very-long-prototype-name-before-approval 后确认。' } },
+ 'dictation-starting': { about: 'Answer microphone starting', data: { question: '请补充你的意见。' } },
+ 'dictation-listening': { about: 'Answer recording with stop action', data: { question: '请补充你的意见。' } },
+ 'dictation-processing': { about: 'Answer transcription pending at 320 px', viewport: { width: 320, height: 600 }, data: { question: '请补充你的意见。' } },
unresolved: { about: 'Missing and ambiguous references remain plain text', data: { question: '请查看 missing.html 和 ambiguous.html,文件尚未找到或有重名时保持普通文字。' } },
'voice-single': { about: 'Voice choice instructions with an active listening toolbar', data: { question: '你希望采用哪种方案?', voice: true } },
'voice-multi': { about: 'Spoken multi-selection awaits confirmation in the existing card', viewport: { width: 800, height: 700 }, data: { question: '你希望保留哪些方案?', voice: true, multi: true } },
@@ -23,6 +31,21 @@ export default {
await page.getByText('确认原型').waitFor();
},
async check(page, ui) {
+ if (ui.case.startsWith('dictation-')) {
+ const mic = page.getByTestId('ask-user-voice');
+ await mic.click();
+ await page.waitForFunction(() => document.querySelector('[data-testid="ask-user-voice"]')?.getAttribute('aria-pressed') === 'true');
+ if (ui.case !== 'dictation-starting') {
+ await page.waitForFunction(() => document.querySelector('[data-testid="ask-user-voice"]')?.getAttribute('aria-busy') === 'false');
+ await page.evaluate(() => window.__uiPage.emit('voice', {
+ type: 'draft', runId: 'fixture-voice', text: '建议保留原来的内容,并把语音录入的状态提示做得更清楚。', source: 'partial',
+ }));
+ }
+ if (ui.case === 'dictation-processing') await mic.click();
+ assert.equal(await mic.getAttribute('aria-busy'), ui.case === 'dictation-listening' ? 'false' : 'true');
+ assert.equal(await mic.isDisabled(), ui.case === 'dictation-processing');
+ return;
+ }
const readVoiceFrame = () => page.getByTestId('ask-user-voice').evaluate((node) => ({
radius: parseFloat(getComputedStyle(node).borderTopLeftRadius),
width: node.getBoundingClientRect().width, height: node.getBoundingClientRect().height,
diff --git a/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx b/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
index fb219998bf..a82940f312 100644
--- a/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
+++ b/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
@@ -12,6 +12,7 @@ import type { VoiceQuestionBridge } from '../components/new-chat/voiceQuestionSe
const start = vi.fn(() => true);
const stop = vi.fn(async () => true);
+const cancel = vi.fn(async () => undefined);
let finalTranscript = '完整回答';
let idleBeforeStop = false;
@@ -22,6 +23,7 @@ vi.mock('../components/new-chat/useAskUserDictation', () => ({
const [state, setState] = useState<'idle' | 'listening'>('idle');
return {
state,
+ captureReady: true,
isBusy: state === 'listening',
liveText: null,
lastError: null,
@@ -37,12 +39,14 @@ vi.mock('../components/new-chat/useAskUserDictation', () => ({
setState('idle');
return ok;
},
+ cancel: async () => { await cancel(); setState('idle'); },
};
},
}));
beforeEach(async () => {
start.mockClear();
+ cancel.mockClear();
stop.mockReset().mockResolvedValue(true);
finalTranscript = '完整回答';
idleBeforeStop = false;
@@ -69,6 +73,36 @@ function renderAsk(questions: PendingAskUser['questions']) {
}
describe('AskUserQuestion voice entry', () => {
+ // Unit: real answer field/gesture; previous card tests covered only mic
+ // clicks. Holding must share composer semantics without hijacking selection.
+ it('records on hold, stops on release, and leaves ordinary drag selection alone', async () => {
+ vi.useFakeTimers();
+ const { view, onAnswer } = renderAsk([{ question: '回答?' }]);
+ const input = view.getByRole('textbox');
+ fireEvent.pointerDown(input, { button: 0, pointerId: 1 });
+ await act(async () => { await vi.advanceTimersByTimeAsync(650); });
+ expect(start).toHaveBeenCalledOnce();
+ fireEvent.pointerUp(window, { pointerId: 1 });
+ await act(async () => {});
+ expect(stop).toHaveBeenCalledOnce();
+ expect(onAnswer).not.toHaveBeenCalled();
+ fireEvent.pointerDown(input, { button: 0, pointerId: 2, clientX: 0, clientY: 0 });
+ fireEvent.pointerMove(window, { pointerId: 2, clientX: 20, clientY: 0 });
+ await act(async () => { await vi.advanceTimersByTimeAsync(650); });
+ expect(start).toHaveBeenCalledOnce();
+ });
+
+ it('Escape cancels dictation without skipping the question or discarding the typed answer', async () => {
+ const { view, onAnswer } = renderAsk([{ question: '回答?' }]);
+ const input = view.getByRole('textbox');
+ fireEvent.change(input, { target: { value: '已有回答' } });
+ fireEvent.click(view.getByTestId('ask-user-voice'));
+ await act(async () => { fireEvent.keyDown(input, { key: 'Escape' }); });
+ expect(cancel).toHaveBeenCalledOnce();
+ expect(stop).not.toHaveBeenCalled();
+ expect(onAnswer).not.toHaveBeenCalled();
+ expect((input as HTMLTextAreaElement).value).toBe('已有回答');
+ });
it('shows a voice button on the pinned free-text row and starts dictation from it', async () => {
const { view } = renderAsk([
{ question: '选哪个?', options: [{ label: '甲' }, { label: '乙' }] },
diff --git a/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx b/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx
index f00a279d9e..5918234ac1 100644
--- a/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx
+++ b/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx
@@ -5,6 +5,8 @@ import { afterEach, describe, expect, it, vi } from 'vitest';
import type { VoiceInputRendererEvent } from '@cindy/voice-input-core';
import { useAskUserDictation } from '../components/new-chat/useAskUserDictation';
+import { resolveVoiceInputStartGuards } from '@/voice-input/startGuards';
+import { startVoiceInputCaptureSession } from '@/voice-input/captureSession';
vi.mock('@/voice-input/startGuards', () => ({
resolveVoiceInputStartGuards: vi.fn(async () => ({
@@ -36,9 +38,55 @@ vi.mock('@/voice-input/workletUrl', () => ({
describe('useAskUserDictation', () => {
afterEach(() => {
+ vi.clearAllMocks();
vi.unstubAllGlobals();
});
+ // Unit: real hook, delayed readiness/IPC. Existing coverage only delivered
+ // transcripts after startup; it missed lost opening words and late captures
+ // continuing after Escape/unmount in the question answer field.
+ it.each(['cancel', 'unmount'] as const)('captures before readiness and ignores a late service start after %s', async (action) => {
+ let onEvent: ((event: VoiceInputRendererEvent) => void) | undefined;
+ let check!: (value: Awaited<ReturnType<typeof resolveVoiceInputStartGuards>>) => void;
+ vi.mocked(resolveVoiceInputStartGuards).mockImplementationOnce(() => new Promise((resolve) => { check = resolve; }));
+ let connected!: (value: { ok: true; runId: string }) => void;
+ const cancel = vi.fn(async () => ({ ok: true }));
+ const start = vi.fn(() => new Promise((resolve) => { connected = resolve; }));
+ const setText = vi.fn();
+ window.electronAPI = {
+ platform: 'darwin',
+ voiceInput: {
+ onEvent: (callback: typeof onEvent) => { onEvent = callback; return () => { onEvent = undefined; }; },
+ getDataSnapshot: () => ({ settings: { playInteractionSound: false, muteSystemAudio: false, refinementEnabled: false, dictionaryEntries: [], language: 'auto', refinementInstructions: '' } }),
+ getHistoryForRefinement: () => [],
+ start, cancel, drainAudioQueue: async () => ({ ok: true }),
+ },
+ } as unknown as typeof window.electronAPI;
+ const hook = renderHook(() => useAskUserDictation({
+ getText: () => '已有回答', setText,
+ getSelection: () => ({ start: 2, end: 2 }), focus: () => {},
+ }));
+ act(() => { hook.result.current.start(); });
+ expect(startVoiceInputCaptureSession).toHaveBeenCalledOnce();
+ expect(start).not.toHaveBeenCalled();
+ await act(async () => { check({ ok: true, permission: { ok: true, status: 'granted' }, readiness: { ok: true } } as Awaited<ReturnType<typeof resolveVoiceInputStartGuards>>); });
+ expect(start).toHaveBeenCalledOnce();
+ act(() => onEvent?.({ type: 'draft', runId: 'late-run', text: '未提交语音', source: 'partial' } as VoiceInputRendererEvent));
+ expect(hook.result.current.liveText).toBe('已有未提交语音回答');
+ if (action === 'cancel') {
+ await act(async () => { await hook.result.current.cancel(); });
+ expect(hook.result.current.isBusy).toBe(false);
+ expect(hook.result.current.liveText).toBeNull();
+ } else {
+ hook.unmount();
+ }
+ await act(async () => { connected({ ok: true, runId: 'late-run' }); });
+ act(() => onEvent?.({ type: 'submitted', runId: 'late-run', text: '迟到的结果' } as VoiceInputRendererEvent));
+ expect(cancel).toHaveBeenCalledWith({ runId: 'late-run' });
+ expect(setText).not.toHaveBeenCalled();
+ hook.unmount();
+ });
+
it.each(['done', 'error'] as const)('keeps the final text and reports terminal %s to the submitter', async (terminal) => {
let onEvent: ((event: VoiceInputRendererEvent) => void) | undefined;
let text = '已有';
@@ -98,6 +146,9 @@ describe('useAskUserDictation', () => {
stopping = hook.result.current.stop().then((ok) => { completed = true; return ok; });
});
expect(completed).toBe(false);
+ expect(hook.result.current.state).toBe('submitting');
+ act(() => onEvent?.({ type: 'state', runId: 'run-1', state: 'listening' } as VoiceInputRendererEvent));
+ expect(hook.result.current.state).toBe('submitting');
act(() => onEvent?.({
type: 'refined',
diff --git a/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx b/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
index c2a538a69a..bbf0f7524f 100644
--- a/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
+++ b/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
@@ -25,7 +25,7 @@ import {
type TextareaHTMLAttributes,
} from 'react';
import { useTranslation } from 'react-i18next';
-import { ArrowRight, Mic } from 'lucide-react';
+import { ArrowRight, Mic, Square } from 'lucide-react';
import { visibleAskOptions } from '@cindy/maker-shared';
import { InteractionPromptCardShell } from '@/components/interaction-portal';
@@ -41,6 +41,7 @@ import { cn } from '@/lib/utils';
import type { AskUserDraft, AskUserViewerState, PendingAskUser } from '@/lib/makerChatStore';
import { requestRendererMicrophonePermission } from '@/voice-input/startGuards';
import { formatVoiceInputShortcut, isVoiceInputShortcutMatch } from '@/voice-input/shortcut';
+import { createComposerLongPressVoiceGesture } from '@/voice-input/composerLongPressGesture';
import { nextAskUserVoiceShortcutOrder, registerAskUserVoiceShortcutCard } from './askUserVoiceShortcutOwner';
import { useAskUserDictation } from './useAskUserDictation';
import { parseVoiceQuestionSelection, type VoiceQuestionBridge, type VoiceQuestionTarget } from './voiceQuestionSelection';
@@ -53,6 +54,7 @@ interface AutoGrowingAnswerTextareaProps extends Omit<
> {
textareaRef: RefObject<HTMLTextAreaElement | null>;
value: string;
+ voice?: ReturnType<typeof useAskUserDictation>;
}
/**
@@ -64,9 +66,55 @@ function AutoGrowingAnswerTextarea({
textareaRef,
value,
className,
+ voice,
...props
}: AutoGrowingAnswerTextareaProps) {
useAutoResize(textareaRef, value, ASK_USER_ANSWER_MAX_HEIGHT_PX);
+ const voiceRef = useRef(voice);
+ voiceRef.current = voice;
+ useEffect(() => {
+ const field = textareaRef.current;
+ if (!field) return;
+ let pointerId: number | null = null;
+ const gesture = createComposerLongPressVoiceGesture({
+ getState: () => voiceRef.current?.state ?? 'idle',
+ start: () => voiceRef.current?.start() ?? false,
+ stop: async () => { await voiceRef.current?.stop(); },
+ onHoldStart: () => {
+ if (pointerId !== null) {
+ try { field.setPointerCapture(pointerId); } catch { /* The pointer may have left the window. */ }
+ }
+ },
+ onHoldEnd: () => {
+ if (pointerId !== null && field.hasPointerCapture?.(pointerId)) field.releasePointerCapture(pointerId);
+ pointerId = null;
+ },
+ });
+ const press = (event: PointerEvent) => {
+ if (!voiceRef.current || field.readOnly || event.button !== 0 || event.ctrlKey || event.metaKey || event.altKey || event.shiftKey) return;
+ if (event.pointerType && event.pointerType !== 'mouse') return;
+ if (getVoiceInputSettings().composerLongPressEnabled === false) return;
+ pointerId = event.pointerId;
+ gesture.press({ x: event.clientX, y: event.clientY });
+ };
+ const move = (event: PointerEvent) => {
+ if (event.pointerId === pointerId) gesture.move({ x: event.clientX, y: event.clientY });
+ };
+ const release = () => { gesture.release(); pointerId = null; };
+ field.addEventListener('pointerdown', press);
+ window.addEventListener('pointermove', move);
+ window.addEventListener('pointerup', release);
+ window.addEventListener('pointercancel', release);
+ window.addEventListener('blur', release);
+ return () => {
+ field.removeEventListener('pointerdown', press);
+ window.removeEventListener('pointermove', move);
+ window.removeEventListener('pointerup', release);
+ window.removeEventListener('pointercancel', release);
+ window.removeEventListener('blur', release);
+ gesture.dispose();
+ };
+ }, [textareaRef]);
return (
<textarea
@@ -277,7 +325,8 @@ function AskUserQuestionForm({
const [voiceShortcutOrder] = useState(nextAskUserVoiceShortcutOrder);
const answerValue = dictation.liveText ?? customInput;
const voiceShortcutLabel = formatVoiceInputShortcut(getVoiceInputSettings().shortcut);
- const voiceButtonLabel = dictation.isBusy
+ const voiceProcessing = dictation.state === 'submitting' || dictation.state === 'refining';
+ const voiceButtonLabel = voiceProcessing ? voiceSubmitLabel : dictation.isBusy
? t('newChat.chatInput.voiceInput.stop')
: voiceShortcutLabel
? `${t('newChat.chatInput.voiceInput.start')} · ${voiceShortcutLabel}`
@@ -596,6 +645,13 @@ function AskUserQuestionForm({
};
const onKeyDown = (event: KeyboardEvent) => {
if (event.repeat || event.isComposing) return;
+ if (event.key === 'Escape' && dictationRef.current.isBusy && shortcutCard.owns(event.target)) {
+ event.preventDefault();
+ event.stopPropagation();
+ pendingVoiceStartRef.current = false;
+ void dictationRef.current.cancel();
+ return;
+ }
const shortcut = getVoiceInputSettings().shortcut;
if (!isVoiceInputShortcutMatch(event, shortcut)) return;
// No defaultPrevented check: an unfocused sibling composer swallows every
@@ -900,6 +956,7 @@ function AskUserQuestionForm({
)}
<AutoGrowingAnswerTextarea
textareaRef={inputRef}
+ voice={voiceBridge ? undefined : dictation}
value={answerValue}
readOnly={voiceLocked}
onCompositionStart={markStarted}
@@ -934,7 +991,8 @@ function AskUserQuestionForm({
size="md"
label={voiceButtonLabel}
busy={dictation.isBusy}
- disabled={voiceSubmitting}
+ processing={voiceProcessing || (dictation.isBusy && !dictation.captureReady)}
+ disabled={voiceSubmitting || voiceProcessing}
onClick={beginVoiceInput}
/>)}
{!isMultiSelect && (
@@ -997,7 +1055,7 @@ function AskUserQuestionForm({
button for screen readers, as the option rows do. */}
<span className="sr-only">{options.length + 1}</span>
</button>
- {!voiceBridge && (<AskUserVoiceButton label={voiceButtonLabel} busy={dictation.isBusy} disabled={voiceSubmitting} onClick={beginVoiceInput} />)}
+ {!voiceBridge && (<AskUserVoiceButton label={voiceButtonLabel} busy={dictation.isBusy} processing={voiceProcessing || (dictation.isBusy && !dictation.captureReady)} disabled={voiceSubmitting || voiceProcessing} onClick={beginVoiceInput} />)}
{/* Pointer shortcut only: the row button above already carries the
action and its number for keyboard and screen-reader users. */}
<div
@@ -1134,6 +1192,7 @@ function AskUserQuestionForm({
<div className="flex items-end gap-2">
<AutoGrowingAnswerTextarea
textareaRef={inputRef}
+ voice={voiceBridge ? undefined : dictation}
value={answerValue}
readOnly={voiceLocked}
onCompositionStart={markStarted}
@@ -1162,7 +1221,8 @@ function AskUserQuestionForm({
size="md"
label={voiceButtonLabel}
busy={dictation.isBusy}
- disabled={voiceSubmitting}
+ processing={voiceProcessing || (dictation.isBusy && !dictation.captureReady)}
+ disabled={voiceSubmitting || voiceProcessing}
onClick={beginVoiceInput}
/>)}
<Tip text={voiceSubmitting ? voiceSubmitLabel : isLastQuestion ? null : t('chat.askUserQuestion.next')}>
@@ -1206,12 +1266,14 @@ function AskUserQuestionForm({
function AskUserVoiceButton({
label,
busy,
+ processing = false,
onClick,
size = 'sm',
disabled = false,
}: {
label: string;
busy: boolean;
+ processing?: boolean;
disabled?: boolean;
onClick: () => void;
/**
@@ -1226,6 +1288,7 @@ function AskUserVoiceButton({
data-testid="ask-user-voice"
aria-label={label}
aria-pressed={busy}
+ aria-busy={processing}
disabled={disabled}
onMouseDown={(event) => {
// Keep the caret in the answer field so dictation replaces the selection
@@ -1245,7 +1308,7 @@ function AskUserVoiceButton({
: 'bg-[var(--ask-badge-bg)] text-[var(--ask-badge-text)]',
)}
>
- <Mic size={15} strokeWidth={2} aria-hidden="true" />
+ {processing ? <Spinner size={15} /> : busy ? <Square size={13} fill="currentColor" aria-hidden="true" /> : <Mic size={15} strokeWidth={2} aria-hidden="true" />}
</button>
);
}
diff --git a/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts b/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
index a8ae2cdd6a..bb0283d503 100644
--- a/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
+++ b/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
@@ -63,14 +63,17 @@ export function useAskUserDictation(
options?: UseAskUserDictationOptions,
): {
state: VoiceInputState;
+ captureReady: boolean;
isBusy: boolean;
liveText: string | null;
lastError: string | null;
start: () => boolean;
stop: () => Promise<boolean>;
+ cancel: () => Promise<void>;
} {
const { t } = useTranslation();
const [state, setState] = useState<VoiceInputState>('idle');
+ const [captureReady, setCaptureReady] = useState(false);
const [liveText, setLiveText] = useState<string | null>(null);
const [lastError, setLastError] = useState<string | null>(null);
const stateRef = useRef<VoiceInputState>('idle');
@@ -132,9 +135,15 @@ export function useAskUserDictation(
if (!engine) return;
try {
await engine.drainBufferedAudio();
- await engine.stop();
+ } catch (error) {
+ log.warn('drain microphone failed', error instanceof Error ? error.message : String(error));
} finally {
+ // Retain ownership while draining the final PCM, then revoke it before
+ // stop waits on getUserMedia so a late capture cannot reopen this take.
if (engineRef.current === engine) engineRef.current = null;
+ await engine.stop().catch((error) => {
+ log.warn('stop microphone failed', error instanceof Error ? error.message : String(error));
+ });
}
try {
await window.electronAPI.voiceInput.drainAudioQueue?.();
@@ -145,6 +154,7 @@ export function useAskUserDictation(
const fail = useCallback(async (message: string) => {
attemptRef.current += 1;
+ const runId = runIdRef.current;
runIdRef.current = null;
anchorRef.current = null;
setLiveText(null);
@@ -153,7 +163,7 @@ export function useAskUserDictation(
await stopEngine();
await restoreMute();
try {
- await window.electronAPI.voiceInput.cancel();
+ if (runId) await window.electronAPI.voiceInput.cancel({ runId });
} catch {
/* The run may never have started. */
}
@@ -166,7 +176,7 @@ export function useAskUserDictation(
if (!shouldHandleVoiceInputEvent(
runIdRef.current,
event.runId,
- isVoiceInputEventScopeActive(stateRef.current),
+ anchorRef.current !== null && isVoiceInputEventScopeActive(stateRef.current),
)) return;
if (event.type === 'draft') {
const anchor = anchorRef.current;
@@ -215,6 +225,7 @@ export function useAskUserDictation(
return;
}
if (event.type === 'state') {
+ if (event.state === 'listening' && stopInFlightRef.current) return;
setVoiceState(event.state);
if (event.state === 'done' || event.state === 'error') {
anchorRef.current = null;
@@ -266,6 +277,7 @@ export function useAskUserDictation(
completionRef.current = { promise: completion, resolve: resolveCompletion };
runIdRef.current = null;
setLastError(null);
+ setCaptureReady(false);
setLiveText(null);
setVoiceState('listening');
let resolveStartReady: () => void = () => undefined;
@@ -276,17 +288,41 @@ export function useAskUserDictation(
void (async () => {
try {
const settings = getVoiceInputSettings();
+ // Like the composer, retain opening words while readiness is checked.
+ const capturePromise = startVoiceInputCaptureSession({
+ label: 'ask-user',
+ workletUrl: getVoiceInputWorkletUrl(),
+ deviceId: settings.microphoneDeviceId ?? undefined,
+ fastActivationEnabled: settings.fastActivationEnabled,
+ getRunId: () => runIdRef.current,
+ setEngine: (engine) => { engineRef.current = engine; },
+ isCurrentEngine: (engine) => attemptRef.current === attempt && engineRef.current === engine,
+ onCaptureReady: () => {
+ if (attemptRef.current === attempt && stateRef.current === 'listening') setCaptureReady(true);
+ },
+ appendAudioChunk: (chunk) => { api.appendAudio(chunk); },
+ onInterrupted: (message) => { if (attemptRef.current === attempt) void fail(message); },
+ onStateChange: () => undefined,
+ getFallbackMessage: () => t('settings.voiceInput.microphone.errors.fallbackToAuto'),
+ onFallback: (message) => toast.warning(message),
+ formatStartError: (error) => error instanceof Error ? error.message : String(error),
+ });
+ let feedback: Promise<unknown> = Promise.resolve();
if (settings.playInteractionSound) {
prepareVoiceInputCues();
- void playVoiceInputStartCue();
+ feedback = playVoiceInputStartCue();
}
if (settings.muteSystemAudio && supportsSystemAudioMute()) {
- const muteRequest = api.muteSystemAudio().then((result) => {
+ const muteRequest = feedback.catch(() => undefined).then(async () => {
+ if (attemptRef.current !== attempt || stateRef.current !== 'listening') return;
+ const result = await api.muteSystemAudio();
if (result.ok) mutedRef.current = true;
}).catch(() => undefined).finally(() => {
if (pendingMuteRef.current === muteRequest) pendingMuteRef.current = null;
});
pendingMuteRef.current = muteRequest;
+ } else {
+ void feedback.catch(() => undefined);
}
let guards: Awaited<ReturnType<typeof resolveVoiceInputStartGuards>>;
@@ -304,6 +340,7 @@ export function useAskUserDictation(
: guards.permission.error;
setVoiceState('error');
anchorRef.current = null;
+ await stopEngine();
if (!guards.permission.ok) {
await optionsRef.current?.onPermissionRequired?.(guards.permission.error);
} else {
@@ -313,27 +350,7 @@ export function useAskUserDictation(
return;
}
- const capture = await startVoiceInputCaptureSession({
- label: 'ask-user',
- workletUrl: getVoiceInputWorkletUrl(),
- deviceId: settings.microphoneDeviceId ?? undefined,
- fastActivationEnabled: settings.fastActivationEnabled,
- getRunId: () => runIdRef.current,
- setEngine: (engine) => {
- engineRef.current = engine;
- },
- isCurrentEngine: (engine) => engineRef.current === engine,
- appendAudioChunk: (chunk) => {
- window.electronAPI.voiceInput.appendAudio(chunk);
- },
- onInterrupted: (message) => {
- if (attemptRef.current === attempt) void fail(message);
- },
- onStateChange: () => undefined,
- getFallbackMessage: () => t('settings.voiceInput.microphone.errors.fallbackToAuto'),
- onFallback: (message) => toast.warning(message),
- formatStartError: (error) => (error instanceof Error ? error.message : String(error)),
- });
+ const capture = await capturePromise;
if (attemptRef.current !== attempt) return;
if (!capture.ok) {
if (capture.cancelled) {
@@ -385,21 +402,27 @@ export function useAskUserDictation(
}
runIdRef.current = started.runId;
capture.drainPendingChunks();
+ } catch (error) {
+ if (attemptRef.current === attempt) await fail(formatError(error instanceof Error ? error.message : String(error)));
} finally {
resolveStartReady();
}
})();
return true;
- }, [fail, formatError, reportError, restoreMute, setVoiceState, t]);
+ }, [fail, formatError, reportError, restoreMute, setVoiceState, stopEngine, t]);
const stop = useCallback(async () => {
if (stopInFlightRef.current) return stopInFlightRef.current;
+ const attempt = attemptRef.current;
+ if (!isVoiceInputEventScopeActive(stateRef.current)) return stateRef.current === 'done';
+ setVoiceState('submitting');
const stopping = (async () => {
try {
// Capture may already be live while the service is connecting. Stop it
// before awaiting startup so post-press speech is not included.
await stopEngine();
await startReadyRef.current;
+ if (attemptRef.current !== attempt) return false;
const completion = completionRef.current;
const settings = getVoiceInputSettings();
if (settings.playInteractionSound) playVoiceInputEndCue();
@@ -417,21 +440,49 @@ export function useAskUserDictation(
// the terminal event proves that setText has received the final text.
return completion ? await completion.promise : false;
} catch (error) {
- await fail(formatError(error instanceof Error ? error.message : String(error)));
+ if (attemptRef.current === attempt) await fail(formatError(error instanceof Error ? error.message : String(error)));
return false;
} finally {
- await restoreMute();
- targetRef.current.focus();
+ if (attemptRef.current === attempt) {
+ await restoreMute();
+ targetRef.current.focus();
+ }
}
})();
- stopInFlightRef.current = stopping.finally(() => {
- stopInFlightRef.current = null;
+ const finishing = stopping.finally(() => {
+ if (stopInFlightRef.current === finishing) stopInFlightRef.current = null;
});
- return stopInFlightRef.current;
+ stopInFlightRef.current = finishing;
+ return finishing;
}, [fail, formatError, restoreMute, setVoiceState, stopEngine]);
+ const cancel = useCallback(async () => {
+ const attempt = ++attemptRef.current;
+ const runId = runIdRef.current;
+ runIdRef.current = null;
+ anchorRef.current = null;
+ draftTextRef.current = null;
+ completionRef.current?.resolve(false);
+ setLiveText(null);
+ // Retain committed text, discard only the unsubmitted live preview.
+ setVoiceState('submitting');
+ try {
+ await stopEngine();
+ } finally {
+ await restoreMute();
+ if (runId) await window.electronAPI.voiceInput.cancel({ runId }).catch(() => undefined);
+ if (attemptRef.current === attempt) {
+ stopInFlightRef.current = null;
+ setVoiceState('idle');
+ targetRef.current.focus();
+ }
+ }
+ }, [restoreMute, setVoiceState, stopEngine]);
+
useEffect(() => () => {
attemptRef.current += 1;
+ stateRef.current = 'idle';
+ anchorRef.current = null;
completionRef.current?.resolve(false);
const runId = runIdRef.current;
runIdRef.current = null;
@@ -442,10 +493,12 @@ export function useAskUserDictation(
return {
state,
+ captureReady,
isBusy: state === 'listening' || state === 'submitting' || state === 'refining',
liveText,
lastError,
start,
stop,
+ cancel,
};
}
diff --git a/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx b/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
index 200eccadde..a7c45318eb 100644
--- a/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
+++ b/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
@@ -79,7 +79,7 @@ vi.mock('@/components/AppText', async () => {
vi.mock('lucide-react-native', async () => {
const { createElement } = await import('react');
return { ShieldCheck: () => null, MessageCircle: () => null, ChevronDown: () => null, ChevronUp: () => null, Check: () => null, CornerDownLeft: () => null,
- Maximize2: () => null, Minimize2: () => null, Minus: () => null, Mic: ({ color }: { color: string }) => createElement('span', { 'data-mic-color': color }), Pencil: () => null, Plus: () => null };
+ Maximize2: () => null, Minimize2: () => null, Minus: () => null, Mic: ({ color }: { color: string }) => createElement('span', { 'data-mic-color': color }), Square: ({ color }: { color: string }) => createElement('span', { 'data-stop-color': color }), Pencil: () => null, Plus: () => null };
});
vi.mock('react-i18next', async (importOriginal) => ({
...await importOriginal<typeof import('react-i18next')>(),
@@ -136,7 +136,7 @@ async function type(id: string, value: string) {
// #6778, unit: render the real card/hook and start/stop dictation. Earlier tests
// only checked labels and clicks, missing the invisible inverse-colour mic.
-// Preserve the existing unfilled control; only startup replaces the mic.
+// Preserve the unfilled control; distinguish startup, recording and processing.
// Minimal input: one question in each answer layout, in either theme.
it.each(['light', 'dark'] as const)('keeps the recording mic visible in %s answer layouts', async (mode) => {
theme.mode = mode;
@@ -170,7 +170,7 @@ it.each(['light', 'dark'] as const)('keeps the recording mic visible in %s answe
expect(button().getAttribute('aria-label')).toBe('session.common.voiceStopRecording');
expect(button().getAttribute('aria-busy')).toBe('false');
expect(style()).toEqual(idleStyle);
- expect(micColor()).toBe(colors.textSecondary);
+ expect(button().querySelector('[data-stop-color]')!.getAttribute('data-stop-color')).toBe(colors.textSecondary);
await click('interaction.ask.voice');
expect(button().getAttribute('aria-label')).toBe('session.screen.voiceStartInput');
expect(style()).toEqual(idleStyle);
@@ -244,6 +244,38 @@ it('queues a submit before the voice handle exists without pretending to record'
expect(JSON.stringify(resolveInteraction.mock.calls[0])).toContain('最终回答');
});
+// Unit: real answer card/hook, fake microphone. Earlier fixtures never returned
+// the controller's caret with a transcript, so a second take inserted at the
+// stale selection instead of following the previous sentence.
+it('continues a second dictation after the first sentence and shows processing while stopping', async () => {
+ const selections: unknown[] = [];
+ let finish!: () => void;
+ voiceStarter.start.mockImplementation(async (input) => {
+ selections.push(input.getSelection());
+ input.onListening();
+ return {
+ stop: () => new Promise<void>((resolve) => {
+ finish = () => { input.setText('第一句', { start: 3, end: 3 }); resolve(); };
+ }),
+ cancel: async () => undefined,
+ };
+ });
+ remoteSessionStore.setPendingInteractions('s1', [{ request: {
+ kind: 'ask_user_question', requestId, questions: [{ question: 'Answer?' }],
+ } }]);
+ await act(async () => root.render(<AskCardAuthContext.Provider value={{
+ getAccessToken: async () => null, refreshAccessToken: async () => null, apiFetch: async () => null,
+ }}><Harness /></AskCardAuthContext.Provider>));
+ await click('interaction.ask.voice');
+ await click('interaction.ask.voice');
+ const button = host.querySelector('[data-testid="interaction.ask.voice"]')!;
+ expect(button.getAttribute('aria-busy')).toBe('true');
+ await act(async () => finish());
+ await click('interaction.ask.voice');
+ expect(selections).toEqual([{ start: 0, end: 0 }, { start: 3, end: 3 }]);
+ expect(resolveInteraction).not.toHaveBeenCalled();
+});
+
it.each(['stop', 'live'] as const)('retains text and permits manual correction after a %s failure', async (failure) => {
let fail!: () => void;
voiceStarter.start.mockImplementation(async (input) => {
diff --git a/apps/mobile/src/session/InteractionPanel.tsx b/apps/mobile/src/session/InteractionPanel.tsx
index 30ac00b6de..c0c740cc99 100644
--- a/apps/mobile/src/session/InteractionPanel.tsx
+++ b/apps/mobile/src/session/InteractionPanel.tsx
@@ -21,6 +21,7 @@ import {
Maximize2,
Minimize2,
Mic,
+ Square,
Minus,
Pencil,
Plus,
@@ -1000,21 +1001,21 @@ type AskUserQuestionCardProps = {
touchLayout: InteractionTouchLayout;
};
-function AskCardVoiceButton({ listening, starting, disabled, onPress }: { listening: boolean; starting: boolean; disabled?: boolean; onPress: () => void }) {
+function AskCardVoiceButton({ listening, starting, stopping, disabled, onPress }: { listening: boolean; starting: boolean; stopping: boolean; disabled?: boolean; onPress: () => void }) {
const { t } = useTranslation();
const { colors } = useTheme();
return (
<Pressable
- accessibilityLabel={listening || starting ? t('session.common.voiceStopRecording') : t('session.screen.voiceStartInput')}
+ accessibilityLabel={stopping ? t('session.presentation.composer.stop.processing') : listening || starting ? t('session.common.voiceStopRecording') : t('session.screen.voiceStartInput')}
accessibilityRole="button"
- accessibilityState={{ busy: starting, disabled: Boolean(disabled) }}
+ accessibilityState={{ busy: starting || stopping, disabled: Boolean(disabled) }}
disabled={disabled}
hitSlop={8}
onPress={onPress}
style={{ alignItems: 'center', height: 44, justifyContent: 'center', width: 44 }}
testID="interaction.ask.voice"
>
- {starting ? <ActivityIndicator color={colors.textSecondary} size="small" /> : <Mic color={colors.textSecondary} size={iconSize.md} />}
+ {starting || stopping ? <ActivityIndicator color={colors.textSecondary} size="small" /> : listening ? <Square color={colors.textSecondary} fill={colors.textSecondary} size={iconSize.sm} /> : <Mic color={colors.textSecondary} size={iconSize.md} />}
</Pressable>
);
}
@@ -1059,8 +1060,9 @@ function AskUserQuestionForm({
const voice = useAskCardVoice({
deviceId,
getText: () => answerTextRef.current,
- setText: (value) => {
+ setText: (value, selection) => {
answerTextRef.current = value;
+ if (selection) selectionRef.current = selection;
onStartAnswer();
setCustomInput(value);
},
@@ -1105,6 +1107,7 @@ function AskUserQuestionForm({
);
setSelectedLabels(next.selectedLabels);
setCustomInput(next.customInput);
+ selectionRef.current = { start: next.customInput.length, end: next.customInput.length };
setShowCustomInput(next.showCustomInput);
}, [answers, current, currentIndex]);
@@ -1313,14 +1316,14 @@ function AskUserQuestionForm({
onStartAnswer();
setCustomInput(text);
}}
- onSelectionChange={(event) => { selectionRef.current = event.nativeEvent.selection; }}
+ onSelectionChange={(event) => { if (!voice.busy) selectionRef.current = event.nativeEvent.selection; }}
placeholder={t('interaction.panel.customAnswerPlaceholder')}
placeholderTextColor={colors.textPlaceholder}
style={styles.inlineInput}
testID="interaction.ask.customInput"
value={customInput}
/>
- <AskCardVoiceButton listening={voice.listening} starting={voice.starting} disabled={voice.stopping || voiceSubmitting} onPress={beginVoice} />
+ <AskCardVoiceButton listening={voice.listening} starting={voice.starting} stopping={voice.stopping} disabled={busy || voice.stopping || voiceSubmitting} onPress={beginVoice} />
</View>
) : (
<View style={[styles.customInputRow, { gap: touchLayout.actionGap }]}>
@@ -1347,7 +1350,7 @@ function AskUserQuestionForm({
>
<Text style={styles.optionCustom}>{t('interaction.panel.customAnswerButtonText')}</Text>
</InteractionTouchButton>
- <AskCardVoiceButton listening={voice.listening} starting={voice.starting} disabled={voice.stopping || voiceSubmitting} onPress={beginVoice} />
+ <AskCardVoiceButton listening={voice.listening} starting={voice.starting} stopping={voice.stopping} disabled={busy || voice.stopping || voiceSubmitting} onPress={beginVoice} />
</View>
)}
</View>
@@ -1367,14 +1370,14 @@ function AskUserQuestionForm({
onStartAnswer();
setCustomInput(text);
}}
- onSelectionChange={(event) => { selectionRef.current = event.nativeEvent.selection; }}
+ onSelectionChange={(event) => { if (!voice.busy) selectionRef.current = event.nativeEvent.selection; }}
placeholder={t('interaction.panel.answerInput')}
placeholderTextColor={colors.textPlaceholder}
style={styles.inlineInput}
testID="interaction.ask.textInput"
value={customInput}
/>
- <AskCardVoiceButton listening={voice.listening} starting={voice.starting} disabled={voice.stopping || voiceSubmitting} onPress={beginVoice} />
+ <AskCardVoiceButton listening={voice.listening} starting={voice.starting} stopping={voice.stopping} disabled={busy || voice.stopping || voiceSubmitting} onPress={beginVoice} />
</View>
)}
diff --git a/apps/mobile/src/session/askCardVoiceStart.ts b/apps/mobile/src/session/askCardVoiceStart.ts
index eb14168139..c00689153a 100644
--- a/apps/mobile/src/session/askCardVoiceStart.ts
+++ b/apps/mobile/src/session/askCardVoiceStart.ts
@@ -45,7 +45,7 @@ export async function startAskCardDictation(input: {
isAppActive?: () => boolean;
subscribeToAppState?: (listener: AppStateListener) => () => void;
getText: () => string;
- setText: (value: string) => void;
+ setText: (value: string, selection?: MobileVoiceSelection) => void;
getSelection: () => MobileVoiceSelection;
onListening: () => void;
onRunCreated?: (run: { stop: () => Promise<void>; cancel: () => Promise<void> }) => void;
@@ -105,7 +105,7 @@ export async function startAskCardDictation(input: {
initialDraft,
initialSelection: input.getSelection(),
readCurrentDraft: input.getText,
- onDraftChanged: (text) => input.setText(text),
+ onDraftChanged: (text, selection) => input.setText(text, selection),
onStateChanged: (state) => {
if (state === 'listening') input.onListening();
},
diff --git a/apps/mobile/src/session/useAskCardVoice.ts b/apps/mobile/src/session/useAskCardVoice.ts
index 6bdd182740..933fea5709 100644
--- a/apps/mobile/src/session/useAskCardVoice.ts
+++ b/apps/mobile/src/session/useAskCardVoice.ts
@@ -19,7 +19,7 @@ type VoiceRun = {
export function useAskCardVoice(input: {
deviceId?: string;
getText: () => string;
- setText: (value: string) => void;
+ setText: (value: string, selection?: MobileVoiceSelection) => void;
getSelection: () => MobileVoiceSelection;
}): {
listening: boolean;
@@ -124,6 +124,9 @@ export function useAskCardVoice(input: {
runReadyRef.current = ready;
const abort = new AbortController();
startAbortRef.current = abort;
+ // Opening permissions/dismissing the keyboard can move native selection.
+ // Dictation belongs to the caret the user had when they pressed the mic.
+ const selection = { ...inputRef.current.getSelection() };
void (async () => {
try {
const { startAskCardDictation } = await import('@/session/askCardVoiceStart');
@@ -135,8 +138,8 @@ export function useAskCardVoice(input: {
refreshAccessToken: () => currentAuth.refreshAccessToken(),
apiFetch: currentAuth.apiFetch,
getText: () => inputRef.current.getText(),
- setText: (value) => { if (!abort.signal.aborted) inputRef.current.setText(value); },
- getSelection: () => inputRef.current.getSelection(),
+ setText: (value, nextSelection) => { if (!abort.signal.aborted) inputRef.current.setText(value, nextSelection); },
+ getSelection: () => selection,
onListening: () => {
if (abort.signal.aborted) return;
startingRef.current = false;