Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

feat: 提问卡的自由回答支持语音输入

zcinta05144b13cd91c43f
diff --git a/apps/desktop/src/renderer/__tests__/askUserDictation.test.ts b/apps/desktop/src/renderer/__tests__/askUserDictation.test.ts
new file mode 100644
index 0000000000..fa63f62f17
--- /dev/null
+++ b/apps/desktop/src/renderer/__tests__/askUserDictation.test.ts
@@ -0,0 +1,32 @@
+import { describe, expect, it } from 'vitest';
+
+import { replaceUneditedSpan, spliceText } from '../components/new-chat/askUserDictation';
+
+describe('ask user dictation text', () => {
+ it('inserts at the caret and replaces a selection', () => {
+ expect(spliceText('hello ', { start: 6, end: 6 }, 'world')).toEqual({
+ value: 'hello world',
+ range: { start: 6, end: 11 },
+ });
+ expect(spliceText('keep this', { start: 5, end: 9 }, 'that')).toEqual({
+ value: 'keep that',
+ range: { start: 5, end: 9 },
+ });
+ });
+
+ it('clamps a stale caret instead of throwing', () => {
+ expect(spliceText('ab', { start: 20, end: 40 }, '!')).toEqual({
+ value: 'ab!',
+ range: { start: 2, end: 3 },
+ });
+ });
+
+ it('replaces a submitted span only while the user has not edited it', () => {
+ const committed = spliceText('', { start: 0, end: 0 }, '原始');
+ expect(replaceUneditedSpan(committed.value, committed.range, ['原始'], '润色后')).toEqual({
+ value: '润色后',
+ range: { start: 0, end: 3 },
+ });
+ expect(replaceUneditedSpan('我改过了', committed.range, ['原始'], '润色后')).toBeNull();
+ });
+});
diff --git a/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx b/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
new file mode 100644
index 0000000000..53d1046150
--- /dev/null
+++ b/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
@@ -0,0 +1,120 @@
+// @vitest-environment jsdom
+
+import { createElement } from 'react';
+import { cleanup, fireEvent, render, waitFor, within } from '@testing-library/react';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import i18n from '@/i18n';
+import type { PendingAskUser } from '@/lib/makerChatStore';
+import { AskUserQuestionPrompt } from '../components/new-chat/AskUserQuestionPrompt';
+
+const start = vi.fn(() => true);
+const stop = vi.fn(async () => undefined);
+
+vi.mock('../components/new-chat/useAskUserDictation', () => ({
+ useAskUserDictation: () => ({
+ state: 'idle',
+ isBusy: false,
+ liveText: null,
+ lastError: null,
+ start,
+ stop,
+ }),
+}));
+
+beforeEach(async () => {
+ start.mockClear();
+ stop.mockClear();
+ await i18n.changeLanguage('zh-CN');
+});
+
+afterEach(() => {
+ cleanup();
+});
+
+function renderAsk(questions: PendingAskUser['questions']) {
+ const onAnswer = vi.fn();
+ const view = render(createElement(AskUserQuestionPrompt, {
+ sessionId: 'ask-voice',
+ pending: { requestId: 'req-voice', questions },
+ onAnswer,
+ viewerState: 'expanded',
+ onViewerStateChange: () => {},
+ draft: null,
+ onDraftChange: () => {},
+ }));
+ return { view, onAnswer };
+}
+
+describe('AskUserQuestion voice entry', () => {
+ it('shows a voice button on the pinned free-text row and starts dictation from it', async () => {
+ const { view } = renderAsk([
+ { question: '选哪个?', options: [{ label: '甲' }, { label: '乙' }] },
+ ]);
+
+ const voice = view.getByTestId('ask-user-voice');
+ expect(voice.getAttribute('aria-label')).toContain('语音');
+ fireEvent.click(voice);
+ await waitFor(() => expect(start).toHaveBeenCalledOnce());
+ expect(view.getByPlaceholderText('请输入回答…')).toBeTruthy();
+ });
+
+ it('keeps the voice button in the same bottom-pinned cluster as the send button', () => {
+ const { view } = renderAsk([
+ { question: '选哪个?', options: [{ label: '甲' }, { label: '乙' }] },
+ ]);
+ fireEvent.click(view.getByText('输入其他回答…'));
+
+ const voice = view.getByTestId('ask-user-voice');
+ const cluster = voice.parentElement!;
+ // Sharing one container is what keeps the two on the same center line as the
+ // answer grows; self-end pins the pair to the row's bottom edge.
+ expect(within(cluster).getByText('提交').tagName).toBe('BUTTON');
+ expect(cluster.className).toContain('items-center');
+ expect(cluster.className).toContain('self-end');
+ });
+
+ it('keeps a voice button when the question has no options', () => {
+ const { view } = renderAsk([{ question: '补充一句' }]);
+ const voice = view.getByTestId('ask-user-voice');
+ expect(voice).toBeTruthy();
+ expect(view.getByPlaceholderText('请输入回答…')).toBeTruthy();
+ expect(within(voice.parentElement!).getByText('提交').tagName).toBe('BUTTON');
+ });
+
+ it('bottom-pins the voice button in multi-select where the row has no send button', () => {
+ const { view } = renderAsk([
+ { question: '选哪些?', multiSelect: true, options: [{ label: '甲' }, { label: '乙' }] },
+ ]);
+ fireEvent.click(view.getByText('输入其他回答…'));
+
+ const voice = view.getByTestId('ask-user-voice');
+ const cluster = voice.parentElement!;
+ expect(within(cluster).queryByText('提交')).toBeNull();
+ expect(cluster.className).toContain('self-end');
+ });
+
+ it('starts dictation from the global voice shortcut while the card is open', async () => {
+ const previous = window.electronAPI;
+ let trigger: ((payload?: { id?: string; phase?: 'start' | 'tap' | 'end' }) => void) | undefined;
+ window.electronAPI = {
+ voiceInput: {
+ onGlobalShortcutTrigger: (callback: typeof trigger) => {
+ trigger = callback;
+ return () => { trigger = undefined; };
+ },
+ claimGlobalShortcutTrigger: () => {},
+ getDataSnapshot: () => ({ settings: { shortcut: null } }),
+ },
+ } as unknown as typeof window.electronAPI;
+ try {
+ renderAsk([
+ { question: '选哪个?', options: [{ label: '甲' }, { label: '乙' }] },
+ ]);
+ trigger?.({ id: 'shortcut-1', phase: 'start' });
+ await waitFor(() => expect(start).toHaveBeenCalledOnce());
+ } finally {
+ window.electronAPI = previous;
+ }
+ });
+});
diff --git a/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx b/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx
new file mode 100644
index 0000000000..a9b9d9899e
--- /dev/null
+++ b/apps/desktop/src/renderer/__tests__/useAskUserDictation.test.tsx
@@ -0,0 +1,102 @@
+// @vitest-environment jsdom
+
+import { renderHook, waitFor } from '@testing-library/react';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import type { VoiceInputRendererEvent } from '@cindy/voice-input-core';
+
+import { useAskUserDictation } from '../components/new-chat/useAskUserDictation';
+
+vi.mock('@/voice-input/startGuards', () => ({
+ resolveVoiceInputStartGuards: vi.fn(async () => ({
+ ok: true,
+ permission: { ok: true, status: 'granted' },
+ readiness: { ok: true },
+ })),
+}));
+
+vi.mock('@/voice-input/captureSession', () => ({
+ startVoiceInputCaptureSession: vi.fn(async (options: { setEngine: (engine: unknown) => void }) => {
+ options.setEngine({
+ drainBufferedAudio: async () => undefined,
+ stop: async () => undefined,
+ });
+ return { ok: true, drainPendingChunks: () => undefined };
+ }),
+}));
+
+vi.mock('@/voice-input/startCue', () => ({
+ prepareVoiceInputCues: vi.fn(),
+ playVoiceInputStartCue: vi.fn(async () => undefined),
+ playVoiceInputEndCue: vi.fn(),
+}));
+
+vi.mock('@/voice-input/workletUrl', () => ({
+ getVoiceInputWorkletUrl: () => '/worklet.js',
+}));
+
+describe('useAskUserDictation', () => {
+ afterEach(() => {
+ vi.unstubAllGlobals();
+ });
+
+ it('writes the submitted transcript into the answer and applies an unedited refinement', async () => {
+ let onEvent: ((event: VoiceInputRendererEvent) => void) | undefined;
+ let text = '已有';
+ const setText = vi.fn((value: string) => { text = value; });
+ window.electronAPI = {
+ platform: 'darwin',
+ voiceInput: {
+ onEvent: (callback: typeof onEvent) => {
+ onEvent = callback;
+ return () => { onEvent = undefined; };
+ },
+ getDataSnapshot: () => ({ settings: { shortcut: null, playInteractionSound: false, muteSystemAudio: false, refinementEnabled: false, language: 'auto', fastActivationEnabled: false, microphoneDeviceId: null, dictionaryEntries: [], refinementInstructions: '' } }),
+ start: async () => ({ ok: true, runId: 'run-1' }),
+ stop: async () => ({ ok: true }),
+ cancel: async () => ({ ok: true }),
+ appendAudio: () => undefined,
+ drainAudioQueue: async () => ({ ok: true }),
+ muteSystemAudio: async () => ({ ok: true }),
+ restoreSystemAudio: async () => ({ ok: true }),
+ recordHistory: () => 'hist-1',
+ updateHistoryEntry: () => undefined,
+ getHistoryForRefinement: () => [],
+ },
+ } as unknown as typeof window.electronAPI;
+
+ const hook = renderHook(() => useAskUserDictation({
+ getText: () => text,
+ setText,
+ getSelection: () => ({ start: text.length, end: text.length }),
+ focus: () => undefined,
+ }));
+
+ expect(hook.result.current.start()).toBe(true);
+ await waitFor(() => expect(onEvent).toBeTypeOf('function'));
+ onEvent?.({
+ type: 'draft',
+ runId: 'run-1',
+ text: '补充',
+ source: 'partial',
+ segment: { id: 'seg-1', text: '补充', source: 'partial', status: 'partial', updatedAt: 1 },
+ } as unknown as VoiceInputRendererEvent);
+ await waitFor(() => expect(hook.result.current.liveText).toBe('已有补充'));
+
+ onEvent?.({
+ type: 'submitted',
+ runId: 'run-1',
+ text: '补充',
+ segment: { id: 'seg-1', text: '补充', basedOnText: '补充', source: 'stable', status: 'final', updatedAt: 2 },
+ } as unknown as VoiceInputRendererEvent);
+ await waitFor(() => expect(setText).toHaveBeenCalledWith('已有补充'));
+
+ onEvent?.({
+ type: 'refined',
+ runId: 'run-1',
+ text: '补充说明',
+ segment: { id: 'seg-1', text: '补充说明', basedOnText: '补充', source: 'stable', status: 'final', updatedAt: 3 },
+ range: { segmentIds: ['seg-1'], start: 2, end: 4 },
+ } as unknown as VoiceInputRendererEvent);
+ await waitFor(() => expect(text).toBe('已有补充说明'));
+ });
+});
diff --git a/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx b/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
index 4a6c4d1bc7..c1a2b61771 100644
--- a/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
+++ b/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
@@ -22,17 +22,23 @@ import {
type TextareaHTMLAttributes,
} from 'react';
import { useTranslation } from 'react-i18next';
-import { ArrowRight } from 'lucide-react';
+import { ArrowRight, Mic } from 'lucide-react';
import { visibleAskOptions } from '@cindy/maker-shared';
import { InteractionPromptCardShell } from '@/components/interaction-portal';
import { ChatSessionFileProvider, useChatSessionFile, useChatSessionFileValue } from '@/components/chat/ChatSessionFileContext';
import { QuestionReferenceText } from '@/components/chat/MarkdownRenderer';
+import { useOptionalConfirmDialog } from '@/components/ui/confirm-dialog-provider';
import { Tip } from '@/components/ui/tooltip';
import { useAutoResize } from '@/hooks/useAutoResize';
+import { getVoiceInputSettings } from '@/hooks/useVoiceInputSettings';
import { shouldCardShortcutYield } from '@/lib/editableKeyboardTarget';
+import { toast } from '@/lib/toast';
import { cn } from '@/lib/utils';
import type { AskUserDraft, AskUserViewerState, PendingAskUser } from '@/lib/makerChatStore';
+import { requestRendererMicrophonePermission } from '@/voice-input/startGuards';
+import { formatVoiceInputShortcut, isVoiceInputShortcutMatch } from '@/voice-input/shortcut';
+import { useAskUserDictation } from './useAskUserDictation';
const ASK_USER_ANSWER_MAX_HEIGHT_PX = 148;
@@ -184,6 +190,59 @@ function AskUserQuestionForm({
cardRef.current = node;
}, []);
const inputRef = useRef<HTMLTextAreaElement>(null);
+ const answerTextRef = useRef(customInput);
+ answerTextRef.current = customInput;
+ const confirmDialog = useOptionalConfirmDialog();
+ const handleVoicePermissionRequired = useCallback(async () => {
+ if (!confirmDialog) {
+ toast.error(t('newChat.chatInput.voiceInput.permissionDialog.description'));
+ return;
+ }
+ const confirmed = await confirmDialog.confirm({
+ title: t('newChat.chatInput.voiceInput.permissionDialog.title'),
+ description: t('newChat.chatInput.voiceInput.permissionDialog.description'),
+ confirmText: t('newChat.chatInput.voiceInput.permissionDialog.confirm'),
+ cancelText: t('newChat.chatInput.voiceInput.permissionDialog.cancel'),
+ autoFocusConfirm: true,
+ });
+ if (!confirmed) return;
+ const result = await requestRendererMicrophonePermission();
+ if (result.ok) return;
+ const settingsResult = await window.electronAPI.voiceInput.openMicrophoneSettings();
+ if (!settingsResult.ok) toast.error(settingsResult.error);
+ }, [confirmDialog, t]);
+ const setAnswerText = useCallback((value: string) => {
+ answerTextRef.current = value;
+ markStarted();
+ setCustomInput(value);
+ }, [markStarted]);
+ const dictation = useAskUserDictation({
+ getText: () => answerTextRef.current,
+ setText: setAnswerText,
+ getSelection: () => {
+ const el = inputRef.current;
+ const length = answerTextRef.current.length;
+ if (!el) return { start: length, end: length };
+ return {
+ start: el.selectionStart ?? length,
+ end: el.selectionEnd ?? length,
+ };
+ },
+ focus: () => inputRef.current?.focus(),
+ }, {
+ onPermissionRequired: handleVoicePermissionRequired,
+ });
+ const dictationRef = useRef(dictation);
+ dictationRef.current = dictation;
+ const pendingVoiceStartRef = useRef(false);
+ const voiceShortcutAtRef = useRef(0);
+ const answerValue = dictation.liveText ?? customInput;
+ const voiceShortcutLabel = formatVoiceInputShortcut(getVoiceInputSettings().shortcut);
+ const voiceButtonLabel = dictation.isBusy
+ ? t('newChat.chatInput.voiceInput.stop')
+ : voiceShortcutLabel
+ ? `${t('newChat.chatInput.voiceInput.start')} · ${voiceShortcutLabel}`
+ : t('newChat.chatInput.voiceInput.start');
// ── Animation state ──
const [slideDirection, setSlideDirection] = useState<'left' | 'right' | null>(null);
@@ -401,6 +460,65 @@ function AskUserQuestionForm({
setTimeout(() => inputRef.current?.focus(), 0);
}, []);
+ const beginVoiceInput = useCallback(() => {
+ const session = dictationRef.current;
+ if (session.isBusy) {
+ void session.stop();
+ return;
+ }
+ if (options.length > 0 && !showCustomInput) {
+ pendingVoiceStartRef.current = true;
+ setShowCustomInput(true);
+ return;
+ }
+ inputRef.current?.focus();
+ session.start();
+ }, [options.length, showCustomInput]);
+
+ useEffect(() => {
+ if (!showCustomInput || !pendingVoiceStartRef.current) return;
+ pendingVoiceStartRef.current = false;
+ inputRef.current?.focus();
+ dictationRef.current.start();
+ }, [showCustomInput]);
+
+ useEffect(() => {
+ const claim = (phase?: 'start' | 'tap' | 'end') => {
+ if (viewerState === 'minimized') return;
+ const now = performance.now();
+ if (now - voiceShortcutAtRef.current < 250) return;
+ voiceShortcutAtRef.current = now;
+ const session = dictationRef.current;
+ if (session.state === 'submitting' || session.state === 'refining') return;
+ if (phase === 'end' || phase === 'tap') {
+ if (session.isBusy) void session.stop();
+ return;
+ }
+ beginVoiceInput();
+ };
+ const onKeyDown = (event: KeyboardEvent) => {
+ if (event.repeat || event.isComposing) return;
+ const shortcut = getVoiceInputSettings().shortcut;
+ if (!isVoiceInputShortcutMatch(event, shortcut)) return;
+ // The answer field is the dictation target. Yield only when some other
+ // editor (sidebar search, a dialog) owns the key.
+ if (event.target !== inputRef.current && shouldCardShortcutYield(event, 'character', cardRef.current)) return;
+ event.preventDefault();
+ event.stopPropagation();
+ claim();
+ };
+ window.addEventListener('keydown', onKeyDown, true);
+ const unsubscribe = window.electronAPI?.voiceInput?.onGlobalShortcutTrigger?.((payload) => {
+ if (viewerState === 'minimized') return;
+ if (payload?.id) window.electronAPI.voiceInput.claimGlobalShortcutTrigger(payload.id);
+ claim(payload?.phase);
+ });
+ return () => {
+ window.removeEventListener('keydown', onKeyDown, true);
+ unsubscribe?.();
+ };
+ }, [beginVoiceInput, viewerState]);
+
// ── Custom input submit (single-select mode) ──
const handleCustomSubmit = useCallback(() => {
const trimmed = customInput.trim();
@@ -609,19 +727,24 @@ function AskUserQuestionForm({
)}
<AutoGrowingAnswerTextarea
textareaRef={inputRef}
- value={customInput}
+ value={answerValue}
+ readOnly={dictation.isBusy}
onCompositionStart={markStarted}
- onChange={(e) => { markStarted(); setCustomInput(e.target.value); }}
+ onChange={(e) => { markStarted(); setAnswerText(e.target.value); }}
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing) {
e.preventDefault();
+ if (dictation.isBusy) {
+ void dictation.stop();
+ return;
+ }
if (isMultiSelect) handleNext();
else handleCustomSubmit();
}
if (e.key === 'Escape') {
e.preventDefault();
setShowCustomInput(false);
- setCustomInput('');
+ setAnswerText('');
}
}}
placeholder={t('chat.askUserQuestion.answerPlaceholder')}
@@ -631,54 +754,68 @@ function AskUserQuestionForm({
'select-text',
)}
/>
- {!isMultiSelect && (
- <Tip text={isLastQuestion ? null : t('chat.askUserQuestion.next')}>
- <button
- type="button"
- onClick={handleCustomSubmit}
- disabled={!customInput.trim()}
- data-testid={isLastQuestion ? undefined : 'ask-user-custom-next'}
- aria-label={isLastQuestion ? undefined : t('chat.askUserQuestion.next')}
- className={cn(
- 'self-end shrink-0 rounded-[9999px] text-13 font-medium',
- isLastQuestion
- ? 'px-[16px] py-[6px]'
- : 'flex h-8 w-8 items-center justify-center',
- customInput.trim()
- ? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
- : 'bg-[var(--ask-send-disabled-bg)] text-[var(--ask-send-disabled-text)]',
- )}
- >
- {isLastQuestion ? (
- t(pending.resumeOnSubmit ? 'chat.askUserQuestion.sendAndContinue' : 'chat.askUserQuestion.submit')
- ) : (
- <ArrowRight size={15} strokeWidth={2} aria-hidden="true" />
- )}
- </button>
- </Tip>
- )}
+ {/* Voice and submit stay in one bottom-pinned cluster: as the answer
+ grows, both keep the same center line instead of the mic drifting
+ up to the top of the box. */}
+ <div className="flex shrink-0 items-center gap-2 self-end">
+ <AskUserVoiceButton
+ size="md"
+ label={voiceButtonLabel}
+ busy={dictation.isBusy}
+ onClick={beginVoiceInput}
+ />
+ {!isMultiSelect && (
+ <Tip text={isLastQuestion ? null : t('chat.askUserQuestion.next')}>
+ <button
+ type="button"
+ onClick={handleCustomSubmit}
+ disabled={!customInput.trim()}
+ data-testid={isLastQuestion ? undefined : 'ask-user-custom-next'}
+ aria-label={isLastQuestion ? undefined : t('chat.askUserQuestion.next')}
+ className={cn(
+ 'shrink-0 rounded-[9999px] text-13 font-medium',
+ isLastQuestion
+ ? 'px-[16px] py-[6px]'
+ : 'flex h-8 w-8 items-center justify-center',
+ customInput.trim()
+ ? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
+ : 'bg-[var(--ask-send-disabled-bg)] text-[var(--ask-send-disabled-text)]',
+ )}
+ >
+ {isLastQuestion ? (
+ t(pending.resumeOnSubmit ? 'chat.askUserQuestion.sendAndContinue' : 'chat.askUserQuestion.submit')
+ ) : (
+ <ArrowRight size={15} strokeWidth={2} aria-hidden="true" />
+ )}
+ </button>
+ </Tip>
+ )}
+ </div>
</div>
) : (
- <button
- type="button"
- className={cn(
- 'flex w-full items-center justify-between px-[16px] py-[14px] text-left',
- 'transition-colors hover:bg-[var(--ask-option-hover)]',
- )}
- onClick={handleCustomOptionClick}
- >
- <div className="flex items-center gap-3">
- {isMultiSelect && (
- <div className="h-[18px] w-[18px] shrink-0 rounded-[4px] border-[1.5px] border-[var(--ask-checkbox-border)]" />
+ <div className="flex items-center gap-2 px-[16px] py-[8px]">
+ <button
+ type="button"
+ className={cn(
+ 'flex min-w-0 flex-1 items-center justify-between py-[6px] text-left',
+ 'transition-colors hover:bg-[var(--ask-option-hover)]',
)}
- <span className="text-14 italic text-[var(--ask-option-custom)]">
- {t('chat.askUserQuestion.customAnswer')}
- </span>
- </div>
- <div className="ml-3 flex h-[28px] w-[28px] shrink-0 items-center justify-center rounded-[8px] bg-[var(--ask-badge-bg)] text-13 font-medium text-[var(--ask-badge-text)]">
- {options.length + 1}
- </div>
- </button>
+ onClick={handleCustomOptionClick}
+ >
+ <div className="flex items-center gap-3">
+ {isMultiSelect && (
+ <div className="h-[18px] w-[18px] shrink-0 rounded-[4px] border-[1.5px] border-[var(--ask-checkbox-border)]" />
+ )}
+ <span className="text-14 italic text-[var(--ask-option-custom)]">
+ {t('chat.askUserQuestion.customAnswer')}
+ </span>
+ </div>
+ <div className="ml-3 flex h-[28px] w-[28px] shrink-0 items-center justify-center rounded-[8px] bg-[var(--ask-badge-bg)] text-13 font-medium text-[var(--ask-badge-text)]">
+ {options.length + 1}
+ </div>
+ </button>
+ <AskUserVoiceButton label={voiceButtonLabel} busy={dictation.isBusy} onClick={beginVoiceInput} />
+ </div>
)}
</div>
) : null;
@@ -802,12 +939,17 @@ function AskUserQuestionForm({
<div className="flex items-end gap-2">
<AutoGrowingAnswerTextarea
textareaRef={inputRef}
- value={customInput}
+ value={answerValue}
+ readOnly={dictation.isBusy}
onCompositionStart={markStarted}
- onChange={(e) => { markStarted(); setCustomInput(e.target.value); }}
+ onChange={(e) => { markStarted(); setAnswerText(e.target.value); }}
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing) {
e.preventDefault();
+ if (dictation.isBusy) {
+ void dictation.stop();
+ return;
+ }
submitDirectAnswer();
}
if (e.key === 'Escape') {
@@ -823,31 +965,83 @@ function AskUserQuestionForm({
'placeholder:text-[var(--ask-input-placeholder)]',
)}
/>
- <Tip text={isLastQuestion ? null : t('chat.askUserQuestion.next')}>
- <button
- type="button"
- onClick={submitDirectAnswer}
- disabled={!customInput.trim()}
- data-testid={isLastQuestion ? undefined : 'ask-user-custom-next'}
- aria-label={isLastQuestion ? undefined : t('chat.askUserQuestion.next')}
- className={cn(
- 'h-10 rounded-[9999px] text-14 font-medium transition-colors',
- isLastQuestion ? 'px-4' : 'flex w-10 items-center justify-center',
- customInput.trim()
- ? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
- : 'bg-[var(--ask-send-disabled-bg)] text-[var(--ask-send-disabled-text)]',
- )}
- >
- {isLastQuestion ? (
- t(pending.resumeOnSubmit ? 'chat.askUserQuestion.sendAndContinue' : 'chat.askUserQuestion.submit')
- ) : (
- <ArrowRight size={17} strokeWidth={2} aria-hidden="true" />
- )}
- </button>
- </Tip>
+ <div className="flex shrink-0 items-center gap-2">
+ <AskUserVoiceButton
+ size="md"
+ label={voiceButtonLabel}
+ busy={dictation.isBusy}
+ onClick={beginVoiceInput}
+ />
+ <Tip text={isLastQuestion ? null : t('chat.askUserQuestion.next')}>
+ <button
+ type="button"
+ onClick={submitDirectAnswer}
+ disabled={!customInput.trim()}
+ data-testid={isLastQuestion ? undefined : 'ask-user-custom-next'}
+ aria-label={isLastQuestion ? undefined : t('chat.askUserQuestion.next')}
+ className={cn(
+ 'h-10 rounded-[9999px] text-14 font-medium transition-colors',
+ isLastQuestion ? 'px-4' : 'flex w-10 items-center justify-center',
+ customInput.trim()
+ ? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
+ : 'bg-[var(--ask-send-disabled-bg)] text-[var(--ask-send-disabled-text)]',
+ )}
+ >
+ {isLastQuestion ? (
+ t(pending.resumeOnSubmit ? 'chat.askUserQuestion.sendAndContinue' : 'chat.askUserQuestion.submit')
+ ) : (
+ <ArrowRight size={17} strokeWidth={2} aria-hidden="true" />
+ )}
+ </button>
+ </Tip>
+ </div>
</div>
)}
</div>
</InteractionPromptCardShell>
);
}
+
+function AskUserVoiceButton({
+ label,
+ busy,
+ onClick,
+ size = 'sm',
+}: {
+ label: string;
+ busy: boolean;
+ onClick: () => void;
+ /**
+ * md matches the answer row's send button (32px) so the two share one center
+ * line; the collapsed free-text row keeps sm to match its 28px option badge.
+ */
+ size?: 'sm' | 'md';
+}) {
+ return (
+ <button
+ type="button"
+ data-testid="ask-user-voice"
+ aria-label={label}
+ aria-pressed={busy}
+ onMouseDown={(event) => {
+ // Keep the caret in the answer field so dictation replaces the selection
+ // the user was editing, instead of jumping after the button takes focus.
+ event.preventDefault();
+ }}
+ onClick={(event) => {
+ event.stopPropagation();
+ onClick();
+ }}
+ className={cn(
+ 'flex shrink-0 items-center justify-center rounded-[8px]',
+ size === 'md' ? 'h-8 w-8' : 'h-7 w-7',
+ 'focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-[var(--focus-ring)]',
+ busy
+ ? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
+ : 'bg-[var(--ask-badge-bg)] text-[var(--ask-badge-text)]',
+ )}
+ >
+ <Mic size={15} strokeWidth={2} aria-hidden="true" />
+ </button>
+ );
+}
diff --git a/apps/desktop/src/renderer/components/new-chat/askUserDictation.ts b/apps/desktop/src/renderer/components/new-chat/askUserDictation.ts
new file mode 100644
index 0000000000..f414da5991
--- /dev/null
+++ b/apps/desktop/src/renderer/components/new-chat/askUserDictation.ts
@@ -0,0 +1,39 @@
+/**
+ * Plain-text edits for dictation landing in the question card's answer box.
+ * The composer keeps its TipTap editor; this surface is a textarea, so ranges
+ * are JavaScript string offsets (not ProseMirror positions).
+ */
+
+export type TextRange = { start: number; end: number };
+
+export function clampTextRange(value: string, range: TextRange): TextRange {
+ const start = Math.max(0, Math.min(range.start, value.length));
+ const end = Math.max(start, Math.min(range.end, value.length));
+ return { start, end };
+}
+
+export function spliceText(value: string, range: TextRange, insert: string): { value: string; range: TextRange } {
+ const clamped = clampTextRange(value, range);
+ const next = value.slice(0, clamped.start) + insert + value.slice(clamped.end);
+ return {
+ value: next,
+ range: { start: clamped.start, end: clamped.start + insert.length },
+ };
+}
+
+/**
+ * Replace a previously submitted span only when the user has not edited it.
+ * Returns null when the span no longer matches the text dictation committed
+ * or the raw transcript refinement was based on.
+ */
+export function replaceUneditedSpan(
+ value: string,
+ range: TextRange,
+ expected: readonly string[],
+ insert: string,
+): { value: string; range: TextRange } | null {
+ const clamped = clampTextRange(value, range);
+ const current = value.slice(clamped.start, clamped.end);
+ if (!expected.some((candidate) => candidate === current)) return null;
+ return spliceText(value, clamped, insert);
+}
diff --git a/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts b/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
new file mode 100644
index 0000000000..2649fe99f6
--- /dev/null
+++ b/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
@@ -0,0 +1,411 @@
+/**
+ * Dictation for the question card's free-text answer.
+ *
+ * Reuses the same microphone session, start guards, and main-process voice
+ * controller as the composer. The composer hook inserts into a TipTap editor;
+ * this one inserts into the answer string, because the card's field is a
+ * textarea and the composer is unmounted while the card is showing.
+ */
+import { useCallback, useEffect, useRef, useState } from 'react';
+import type { VoiceInputErrorCode, VoiceInputRendererEvent, VoiceInputState } from '@cindy/voice-input-core';
+
+import { getVoiceInputSettings } from '@/hooks/useVoiceInputSettings';
+import {
+ recordVoiceInputHistory,
+ updateVoiceInputHistoryEntry,
+} from '@/hooks/useVoiceInputHistory';
+import { toast } from '@/lib/toast';
+import { createLogger } from '@/lib/logger';
+import { getVoiceInputErrorMessageKey, VOICE_INPUT_ERROR_CODE_KEYS } from '@/voice-input/overlayErrors';
+import { buildBaseVoiceInputRefinementContext } from '@/voice-input/refinementContextBuilder';
+import {
+ MAX_REFINEMENT_SIDE_CONTEXT_CHARS,
+ takeContextHead,
+ takeContextTail,
+ VOICE_INPUT_REFINEMENT_CACHE_SCOPE,
+} from '@/voice-input/refinementContext';
+import { resolveVoiceInputStartGuards } from '@/voice-input/startGuards';
+import { startVoiceInputCaptureSession } from '@/voice-input/captureSession';
+import { getVoiceInputWorkletUrl } from '@/voice-input/workletUrl';
+import { playVoiceInputEndCue, playVoiceInputStartCue, prepareVoiceInputCues } from '@/voice-input/startCue';
+import {
+ isVoiceInputEventScopeActive,
+ shouldHandleVoiceInputEvent,
+} from '@/voice-input/eventScope';
+import type { WebMicAudioEngine } from '@/voice-input/WebMicAudioEngine';
+import { useTranslation } from 'react-i18next';
+
+import { replaceUneditedSpan, spliceText, type TextRange } from './askUserDictation';
+
+const log = createLogger('ask-user-dictation');
+
+export type AskUserDictationTarget = {
+ getText: () => string;
+ setText: (value: string) => void;
+ getSelection: () => TextRange;
+ focus: () => void;
+};
+
+export type UseAskUserDictationOptions = {
+ onPermissionRequired?: (error: string) => void | Promise<void>;
+};
+
+type Anchor = { text: string; range: TextRange };
+type SubmittedSpan = { range: TextRange; text: string; historyId: string | null };
+
+function supportsSystemAudioMute(): boolean {
+ const platform = window.electronAPI?.platform;
+ return platform === 'darwin' || platform === 'win32';
+}
+
+export function useAskUserDictation(
+ target: AskUserDictationTarget,
+ options?: UseAskUserDictationOptions,
+): {
+ state: VoiceInputState;
+ isBusy: boolean;
+ liveText: string | null;
+ lastError: string | null;
+ start: () => boolean;
+ stop: () => Promise<void>;
+} {
+ const { t } = useTranslation();
+ const [state, setState] = useState<VoiceInputState>('idle');
+ const [liveText, setLiveText] = useState<string | null>(null);
+ const [lastError, setLastError] = useState<string | null>(null);
+ const stateRef = useRef<VoiceInputState>('idle');
+ const targetRef = useRef(target);
+ const optionsRef = useRef(options);
+ const attemptRef = useRef(0);
+ const runIdRef = useRef<string | null>(null);
+ const engineRef = useRef<WebMicAudioEngine | null>(null);
+ const anchorRef = useRef<Anchor | null>(null);
+ const submittedRef = useRef<SubmittedSpan | null>(null);
+ const stopInFlightRef = useRef<Promise<void> | null>(null);
+ const startReadyRef = useRef<Promise<void>>(Promise.resolve());
+ const mutedRef = useRef(false);
+ targetRef.current = target;
+ optionsRef.current = options;
+
+ const setVoiceState = useCallback((next: VoiceInputState) => {
+ stateRef.current = next;
+ setState(next);
+ }, []);
+
+ const formatError = useCallback((message: string, code?: VoiceInputErrorCode) => {
+ if (code) return t(VOICE_INPUT_ERROR_CODE_KEYS[code]);
+ const key = getVoiceInputErrorMessageKey(message);
+ return key ? t(key) : message;
+ }, [t]);
+
+ const reportError = useCallback((message: string) => {
+ setLastError(message);
+ toast.error(message);
+ }, []);
+
+ const restoreMute = useCallback(async () => {
+ if (!mutedRef.current || !supportsSystemAudioMute()) return;
+ mutedRef.current = false;
+ try {
+ await window.electronAPI.voiceInput.restoreSystemAudio();
+ } catch (error) {
+ log.warn('restore system audio failed', error instanceof Error ? error.message : String(error));
+ }
+ }, []);
+
+ const stopEngine = useCallback(async () => {
+ const engine = engineRef.current;
+ if (!engine) return;
+ try {
+ await engine.drainBufferedAudio();
+ await engine.stop();
+ } finally {
+ if (engineRef.current === engine) engineRef.current = null;
+ }
+ try {
+ await window.electronAPI.voiceInput.drainAudioQueue?.();
+ } catch (error) {
+ log.warn('drain audio queue failed', error instanceof Error ? error.message : String(error));
+ }
+ }, []);
+
+ const fail = useCallback(async (message: string) => {
+ attemptRef.current += 1;
+ runIdRef.current = null;
+ anchorRef.current = null;
+ setLiveText(null);
+ setVoiceState('error');
+ reportError(message);
+ await stopEngine();
+ await restoreMute();
+ try {
+ await window.electronAPI.voiceInput.cancel();
+ } catch {
+ /* The run may never have started. */
+ }
+ }, [reportError, restoreMute, setVoiceState, stopEngine]);
+
+ useEffect(() => {
+ const subscribe = window.electronAPI?.voiceInput?.onEvent;
+ if (!subscribe) return;
+ return subscribe((event: VoiceInputRendererEvent) => {
+ if (!shouldHandleVoiceInputEvent(
+ runIdRef.current,
+ event.runId,
+ isVoiceInputEventScopeActive(stateRef.current),
+ )) return;
+ if (event.type === 'draft') {
+ const anchor = anchorRef.current;
+ if (!anchor) return;
+ setLiveText(spliceText(anchor.text, anchor.range, event.text).value);
+ return;
+ }
+ if (event.type === 'submitted') {
+ const anchor = anchorRef.current;
+ if (!anchor) return;
+ const next = spliceText(anchor.text, anchor.range, event.text);
+ submittedRef.current = {
+ range: next.range,
+ text: event.text,
+ historyId: recordVoiceInputHistory(event.text),
+ };
+ anchorRef.current = { text: next.value, range: next.range };
+ targetRef.current.setText(next.value);
+ setLiveText(next.value);
+ return;
+ }
+ if (event.type === 'refinement-preview') {
+ const submitted = submittedRef.current;
+ if (!submitted) return;
+ const preview = spliceText(targetRef.current.getText(), submitted.range, event.text);
+ setLiveText(preview.value);
+ return;
+ }
+ if (event.type === 'refined') {
+ const submitted = submittedRef.current;
+ if (!submitted) return;
+ const basedOn = event.segment.basedOnText ?? submitted.text;
+ const next = replaceUneditedSpan(
+ targetRef.current.getText(),
+ submitted.range,
+ [submitted.text, basedOn],
+ event.text,
+ );
+ setLiveText(null);
+ if (!next) return;
+ submittedRef.current = { ...submitted, range: next.range, text: event.text };
+ targetRef.current.setText(next.value);
+ if (submitted.historyId) updateVoiceInputHistoryEntry(submitted.historyId, event.text);
+ return;
+ }
+ if (event.type === 'state') {
+ setVoiceState(event.state);
+ if (event.state === 'done' || event.state === 'error') {
+ runIdRef.current = null;
+ anchorRef.current = null;
+ setLiveText(null);
+ if (event.state === 'error') {
+ /* The error event carries the message. */
+ }
+ }
+ return;
+ }
+ if (event.type === 'error') {
+ const kept = event.transcriptKept === true && submittedRef.current !== null;
+ const message = formatError(event.message, event.code);
+ reportError(kept ? t('voiceInputOverlay.transcriptKept', { message }) : message);
+ setVoiceState('error');
+ setLiveText(null);
+ runIdRef.current = null;
+ void restoreMute();
+ }
+ });
+ }, [formatError, reportError, restoreMute, setVoiceState, t]);
+
+ const start = useCallback((): boolean => {
+ const api = window.electronAPI?.voiceInput;
+ if (!api?.start) return false;
+ const current = stateRef.current;
+ if (
+ stopInFlightRef.current
+ || current === 'listening'
+ || current === 'submitting'
+ || current === 'refining'
+ ) return false;
+
+ const attempt = attemptRef.current + 1;
+ attemptRef.current = attempt;
+ const selection = targetRef.current.getSelection();
+ anchorRef.current = { text: targetRef.current.getText(), range: selection };
+ submittedRef.current = null;
+ runIdRef.current = null;
+ setLastError(null);
+ setLiveText(null);
+ setVoiceState('listening');
+ let resolveStartReady: () => void = () => undefined;
+ startReadyRef.current = new Promise<void>((resolve) => {
+ resolveStartReady = resolve;
+ });
+
+ void (async () => {
+ try {
+ const settings = getVoiceInputSettings();
+ if (settings.playInteractionSound) {
+ prepareVoiceInputCues();
+ void playVoiceInputStartCue();
+ }
+ if (settings.muteSystemAudio && supportsSystemAudioMute()) {
+ void api.muteSystemAudio().then((result) => {
+ if (result.ok) mutedRef.current = true;
+ }).catch(() => undefined);
+ }
+
+ let guards: Awaited<ReturnType<typeof resolveVoiceInputStartGuards>>;
+ try {
+ guards = await resolveVoiceInputStartGuards();
+ } catch (error) {
+ if (attemptRef.current !== attempt) return;
+ await fail(error instanceof Error ? error.message : String(error));
+ return;
+ }
+ if (attemptRef.current !== attempt) return;
+ if (!guards.ok) {
+ const message = guards.permission.ok
+ ? (guards.readiness.error ?? t('settings.voiceInput.serviceSource.credentialError.customAsrConfigMissing'))
+ : guards.permission.error;
+ setVoiceState('error');
+ anchorRef.current = null;
+ if (!guards.permission.ok) {
+ await optionsRef.current?.onPermissionRequired?.(guards.permission.error);
+ } else {
+ reportError(message);
+ }
+ await restoreMute();
+ return;
+ }
+
+ const capture = await startVoiceInputCaptureSession({
+ label: 'ask-user',
+ workletUrl: getVoiceInputWorkletUrl(),
+ deviceId: settings.microphoneDeviceId ?? undefined,
+ fastActivationEnabled: settings.fastActivationEnabled,
+ getRunId: () => runIdRef.current,
+ setEngine: (engine) => {
+ engineRef.current = engine;
+ },
+ isCurrentEngine: (engine) => engineRef.current === engine,
+ appendAudioChunk: (chunk) => {
+ window.electronAPI.voiceInput.appendAudio(chunk);
+ },
+ onInterrupted: (message) => {
+ if (attemptRef.current === attempt) void fail(message);
+ },
+ onStateChange: () => undefined,
+ getFallbackMessage: () => t('settings.voiceInput.microphone.errors.fallbackToAuto'),
+ onFallback: (message) => toast.warning(message),
+ formatStartError: (error) => (error instanceof Error ? error.message : String(error)),
+ });
+ if (attemptRef.current !== attempt) return;
+ if (!capture.ok) {
+ if (capture.cancelled) {
+ setVoiceState('idle');
+ anchorRef.current = null;
+ await restoreMute();
+ return;
+ }
+ if (capture.permissionDenied) {
+ setVoiceState('error');
+ anchorRef.current = null;
+ await optionsRef.current?.onPermissionRequired?.(capture.error);
+ await restoreMute();
+ return;
+ }
+ await fail(capture.error);
+ return;
+ }
+
+ const anchor = anchorRef.current;
+ const text = anchor?.text ?? '';
+ const range = anchor?.range ?? { start: text.length, end: text.length };
+ const baseContext = buildBaseVoiceInputRefinementContext({ settings });
+ let started: { ok: true; runId: string } | { ok: false; error: string };
+ try {
+ started = await api.start({
+ sourceLanguage: settings.language,
+ refinementEnabled: settings.refinementEnabled,
+ refinementCacheScope: VOICE_INPUT_REFINEMENT_CACHE_SCOPE,
+ refinementContext: settings.refinementEnabled
+ ? {
+ ...baseContext,
+ selectionBefore: takeContextTail(text.slice(0, range.start), MAX_REFINEMENT_SIDE_CONTEXT_CHARS),
+ selectedText: takeContextHead(text.slice(range.start, range.end), MAX_REFINEMENT_SIDE_CONTEXT_CHARS),
+ selectionAfter: takeContextHead(text.slice(range.end), MAX_REFINEMENT_SIDE_CONTEXT_CHARS),
+ }
+ : undefined,
+ });
+ } catch (error) {
+ started = { ok: false, error: error instanceof Error ? error.message : String(error) };
+ }
+ if (attemptRef.current !== attempt) {
+ if (started.ok) void api.cancel({ runId: started.runId });
+ return;
+ }
+ if (!started.ok) {
+ await fail(formatError(started.error));
+ return;
+ }
+ runIdRef.current = started.runId;
+ capture.drainPendingChunks();
+ } finally {
+ resolveStartReady();
+ }
+ })();
+ return true;
+ }, [fail, formatError, reportError, restoreMute, setVoiceState, t]);
+
+ const stop = useCallback(async () => {
+ if (stopInFlightRef.current) return stopInFlightRef.current;
+ const stopping = (async () => {
+ await startReadyRef.current;
+ const settings = getVoiceInputSettings();
+ if (settings.playInteractionSound) playVoiceInputEndCue();
+ await stopEngine();
+ if (!runIdRef.current) {
+ await restoreMute();
+ targetRef.current.focus();
+ return;
+ }
+ try {
+ const result = await window.electronAPI.voiceInput.stop();
+ if (!result.ok) reportError(formatError(result.error));
+ } catch (error) {
+ reportError(formatError(error instanceof Error ? error.message : String(error)));
+ } finally {
+ await restoreMute();
+ }
+ targetRef.current.focus();
+ })();
+ stopInFlightRef.current = stopping.finally(() => {
+ stopInFlightRef.current = null;
+ });
+ return stopInFlightRef.current;
+ }, [formatError, reportError, restoreMute, stopEngine]);
+
+ useEffect(() => () => {
+ attemptRef.current += 1;
+ const runId = runIdRef.current;
+ runIdRef.current = null;
+ void stopEngine();
+ void restoreMute();
+ if (runId) void window.electronAPI?.voiceInput?.cancel?.({ runId });
+ }, [restoreMute, stopEngine]);
+
+ return {
+ state,
+ isBusy: state === 'listening' || state === 'submitting' || state === 'refining',
+ liveText,
+ lastError,
+ start,
+ stop,
+ };
+}
diff --git a/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx b/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
index 7f3193d09e..bc7a398084 100644
--- a/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
+++ b/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
@@ -67,7 +67,7 @@ vi.mock('@/components/AppText', async () => {
};
});
vi.mock('lucide-react-native', () => ({ ShieldCheck: () => null, MessageCircle: () => null, ChevronDown: () => null, ChevronUp: () => null, Check: () => null, CornerDownLeft: () => null,
- Maximize2: () => null, Minimize2: () => null, Minus: () => null, Pencil: () => null, Plus: () => null }));
+ Maximize2: () => null, Minimize2: () => null, Minus: () => null, Mic: () => null, Pencil: () => null, Plus: () => null }));
vi.mock('react-i18next', async (importOriginal) => ({
...await importOriginal<typeof import('react-i18next')>(),
useTranslation: () => ({ t: (key: string) => key, i18n: { language: 'en' } }),
diff --git a/apps/mobile/src/auth/AuthContext.tsx b/apps/mobile/src/auth/AuthContext.tsx
index bda65e0b1e..a2002a08a7 100644
--- a/apps/mobile/src/auth/AuthContext.tsx
+++ b/apps/mobile/src/auth/AuthContext.tsx
@@ -48,6 +48,7 @@ import {
// 运行时另有 __DEV__ guard 双保险。
import { resolveLoginScenarioFetch } from '@cindy/auth-client/fixtures';
+import { AskCardAuthContext, type AskCardAuth } from '@/session/askCardAuth';
import {
apiFetchRaw,
ApiError,
@@ -383,7 +384,9 @@ export function AuthProvider({ children }: { children: ReactNode }) {
apiFetch: visualMockApiFetch,
};
return (
- <AuthContext.Provider value={value}>{children}</AuthContext.Provider>
+ <AuthContext.Provider value={value}>
+ <AskCardAuthContext.Provider value={value as AskCardAuth}>{children}</AskCardAuthContext.Provider>
+ </AuthContext.Provider>
);
}
@@ -3415,7 +3418,11 @@ export function AuthProvider({ children }: { children: ReactNode }) {
],
);
- return <AuthContext.Provider value={value}>{children}</AuthContext.Provider>;
+ return (
+ <AuthContext.Provider value={value}>
+ <AskCardAuthContext.Provider value={value as AskCardAuth}>{children}</AskCardAuthContext.Provider>
+ </AuthContext.Provider>
+ );
}
export function useAuth(): AuthContextValue {
diff --git a/apps/mobile/src/session/InteractionPanel.tsx b/apps/mobile/src/session/InteractionPanel.tsx
index 4e58b108ae..f52ed9d902 100644
--- a/apps/mobile/src/session/InteractionPanel.tsx
+++ b/apps/mobile/src/session/InteractionPanel.tsx
@@ -19,6 +19,7 @@ import {
CornerDownLeft,
Maximize2,
Minimize2,
+ Mic,
Minus,
Pencil,
Plus,
@@ -40,6 +41,7 @@ import { QuestionReferenceText } from '@/session/QuestionReferenceText';
import { formatRemoteError } from '@/device-link/remoteStatus';
import { useMobileMakerTransport } from '@/device-link/useMobileMakerTransport';
import { asyncQuestionNeedsSupplement, formatAsyncQuestionSupplement } from '@cindy/maker-shared/interaction';
+import { useAskCardVoice } from '@/session/useAskCardVoice';
import { buildQueuedTextMessage } from '@/session/inputProjection';
import {
answerKey,
@@ -672,6 +674,7 @@ function InteractionItem({
return (
<AskUserQuestionCard
busy={busy}
+ deviceId={deviceId}
item={item}
onDecision={(decision) => void submitDecision(decision)}
onStartAnswer={() => remoteSessionStore.protectAsyncQuestion(sessionId, item)}
@@ -979,6 +982,7 @@ function PermissionEvidence({
type AskUserQuestionCardProps = {
busy: boolean;
+ deviceId?: string;
item: PendingInteraction;
onDecision(decision: Record<string, unknown>): void;
onStartAnswer(): void;
@@ -986,6 +990,23 @@ type AskUserQuestionCardProps = {
touchLayout: InteractionTouchLayout;
};
+function AskCardVoiceButton({ listening, onPress }: { listening: boolean; onPress: () => void }) {
+ const { t } = useTranslation();
+ const { colors } = useTheme();
+ return (
+ <Pressable
+ accessibilityLabel={listening ? t('session.common.voiceStopRecording') : t('session.screen.voiceStartInput')}
+ accessibilityRole="button"
+ hitSlop={8}
+ onPress={onPress}
+ style={{ alignItems: 'center', height: 44, justifyContent: 'center', width: 44 }}
+ testID="interaction.ask.voice"
+ >
+ <Mic color={listening ? colors.ctaText : colors.textSecondary} size={iconSize.md} />
+ </Pressable>
+ );
+}
+
function AskUserQuestionCard(props: AskUserQuestionCardProps) {
const questions = useMemo(() => normalizeAskQuestions(props.item.request.questions), [props.item.request.questions]);
// 与桌面 AskUserQuestionPrompt 同口径:内容相同的重新拉取保持挂载与输入;
@@ -996,6 +1017,7 @@ function AskUserQuestionCard(props: AskUserQuestionCardProps) {
function AskUserQuestionForm({
busy,
+ deviceId,
item,
onDecision,
onStartAnswer,
@@ -1019,6 +1041,31 @@ function AskUserQuestionForm({
const [selectedLabels, setSelectedLabels] = useState<Set<string>>(() => new Set(initialDraft?.selectedLabels ?? []));
const [customInput, setCustomInput] = useState(() => initialDraft?.customInput ?? '');
const [showCustomInput, setShowCustomInput] = useState(() => initialDraft?.showCustomInput ?? false);
+ const answerTextRef = useRef(customInput);
+ answerTextRef.current = customInput;
+ const selectionRef = useRef({ start: customInput.length, end: customInput.length });
+ const voice = useAskCardVoice({
+ deviceId,
+ getText: () => answerTextRef.current,
+ setText: (value) => {
+ answerTextRef.current = value;
+ onStartAnswer();
+ setCustomInput(value);
+ },
+ getSelection: () => {
+ const length = answerTextRef.current.length;
+ const selection = selectionRef.current;
+ if (selection.end > length) return { start: length, end: length };
+ return selection;
+ },
+ });
+ const beginVoice = () => {
+ if (optionsLengthRef.current > 0 && !showCustomInputRef.current) setShowCustomInput(true);
+ voice.toggle();
+ };
+ const showCustomInputRef = useRef(showCustomInput);
+ showCustomInputRef.current = showCustomInput;
+ const optionsLengthRef = useRef(0);
// 选中态/自定义输入已对应的题号。只在切题时从 answers 回填:请求被重新拉取
// (截图后回到前台等)会换掉 current/answers 的引用,不能因此清掉正在输入的内容。
const syncedIndexRef = useRef(currentIndex);
@@ -1080,6 +1127,7 @@ function AskUserQuestionForm({
const isLast = currentIndex === questions.length - 1;
const options = visibleAskOptions(current.options);
+ optionsLengthRef.current = options.length;
const isMulti = current.multiSelect === true;
const currentAnswerKey = answerKey(current);
const existingAnswer = answers[currentAnswerKey];
@@ -1226,40 +1274,51 @@ function AskUserQuestionForm({
<TextInput
accessibilityLabel={t('interaction.panel.customAnswerInputAccessibility')}
autoFocus
- editable={!busy}
+ editable={!busy && !voice.listening}
onKeyPress={({ nativeEvent }) => { if (nativeEvent.key.length === 1) onStartAnswer(); }}
- onChangeText={(text) => { onStartAnswer(); setCustomInput(text); }}
+ onChangeText={(text) => {
+ answerTextRef.current = text;
+ onStartAnswer();
+ setCustomInput(text);
+ }}
+ onSelectionChange={(event) => { selectionRef.current = event.nativeEvent.selection; }}
placeholder={t('interaction.panel.customAnswerPlaceholder')}
placeholderTextColor={colors.textPlaceholder}
style={styles.inlineInput}
testID="interaction.ask.customInput"
value={customInput}
/>
+ <AskCardVoiceButton listening={voice.listening} onPress={beginVoice} />
</View>
) : (
- <InteractionTouchButton
- accessibilityLabel={t('interaction.panel.customAnswerPlaceholder')}
- disabled={busy}
- onPress={() => {
- if (!isMulti) {
- clearCurrentAnswer();
- setSelectedLabels(new Set());
- }
- setShowCustomInput(true);
- }}
- style={[
- styles.optionRow,
- {
- gap: touchLayout.actionGap,
- minHeight: touchLayout.optionRowMinHeight,
- paddingHorizontal: touchLayout.cardPadding,
- },
- ]}
- testID="interaction.ask.showCustomButton"
- >
- <Text style={styles.optionCustom}>{t('interaction.panel.customAnswerButtonText')}</Text>
- </InteractionTouchButton>
+ <View style={[styles.customInputRow, { gap: touchLayout.actionGap }]}>
+ <InteractionTouchButton
+ accessibilityLabel={t('interaction.panel.customAnswerPlaceholder')}
+ disabled={busy}
+ onPress={() => {
+ if (!isMulti) {
+ clearCurrentAnswer();
+ setSelectedLabels(new Set());
+ }
+ setShowCustomInput(true);
+ }}
+ style={[
+ styles.optionRow,
+ {
+ flex: 1,
+ gap: touchLayout.actionGap,
+ minHeight: touchLayout.optionRowMinHeight,
+ paddingHorizontal: touchLayout.cardPadding,
+ },
+ ]}
+ testID="interaction.ask.showCustomButton"
+ >
+ <Text style={styles.optionCustom}>{t('interaction.panel.customAnswerButtonText')}</Text>
+ </InteractionTouchButton>
+ <AskCardVoiceButton listening={voice.listening} onPress={beginVoice} />
+ </View>
)}
+ {voice.error ? <Text testID="interaction.ask.voiceError">{voice.error}</Text> : null}
</View>
) : (
<View style={[
@@ -1270,15 +1329,21 @@ function AskUserQuestionForm({
<TextInput
accessibilityLabel={t('interaction.panel.answerInput')}
autoFocus
- editable={!busy}
+ editable={!busy && !voice.listening}
onKeyPress={({ nativeEvent }) => { if (nativeEvent.key.length === 1) onStartAnswer(); }}
- onChangeText={(text) => { onStartAnswer(); setCustomInput(text); }}
+ onChangeText={(text) => {
+ answerTextRef.current = text;
+ onStartAnswer();
+ setCustomInput(text);
+ }}
+ onSelectionChange={(event) => { selectionRef.current = event.nativeEvent.selection; }}
placeholder={t('interaction.panel.answerInput')}
placeholderTextColor={colors.textPlaceholder}
style={styles.inlineInput}
testID="interaction.ask.textInput"
value={customInput}
/>
+ <AskCardVoiceButton listening={voice.listening} onPress={beginVoice} />
</View>
)}
diff --git a/apps/mobile/src/session/askCardAuth.ts b/apps/mobile/src/session/askCardAuth.ts
new file mode 100644
index 0000000000..b5384c1663
--- /dev/null
+++ b/apps/mobile/src/session/askCardAuth.ts
@@ -0,0 +1,14 @@
+import { createContext, useContext } from 'react';
+
+/** The slice of auth the question-card microphone needs. Kept out of AuthContext so card tests do not load the auth native graph. */
+export type AskCardAuth = {
+ getAccessToken(): Promise<string | null>;
+ refreshAccessToken(): Promise<string | null>;
+ apiFetch(path: string, opts: object): Promise<unknown>;
+};
+
+export const AskCardAuthContext = createContext<AskCardAuth | null>(null);
+
+export function useAskCardAuth(): AskCardAuth | null {
+ return useContext(AskCardAuthContext);
+}
diff --git a/apps/mobile/src/session/askCardVoiceStart.ts b/apps/mobile/src/session/askCardVoiceStart.ts
new file mode 100644
index 0000000000..10b3a83713
--- /dev/null
+++ b/apps/mobile/src/session/askCardVoiceStart.ts
@@ -0,0 +1,116 @@
+/**
+ * Opens the same managed dictation session the composer uses, aimed at the
+ * question card's answer text instead of the chat draft.
+ */
+import { AppState } from 'react-native';
+
+import type { AskCardAuth } from '@/session/askCardAuth';
+import {
+ CINDY_MANAGED_REFINER_PROVIDER,
+ createMobileCindyVoiceCredential,
+ MobileCindyVoiceRunContext,
+} from '@/session/mobileCindyVoiceSession';
+import { createMobileVoiceControllerSession } from '@/session/mobileVoiceController';
+import { playMobileVoiceInputEndCue } from '@/session/mobileVoiceCue';
+import {
+ mobileVoiceMicPermissionError,
+ mobileVoiceRealtimeAudioUnavailableError,
+ type MobileVoiceSelection,
+} from '@/session/mobileVoiceInput';
+import { isMobileRealtimeAudioAvailable } from '@/session/mobileRealtimeAudio';
+import { createMobileSpeakerGate } from '@/session/mobileSpeakerFilter';
+import { resolveMobileVoiceRecordingPermission } from '@/session/mobileVoiceStartup';
+
+type AppStateListener = (state: string) => void;
+
+export async function startAskCardDictation(input: {
+ deviceId: string;
+ getAccessToken: () => Promise<string | null>;
+ refreshAccessToken: () => Promise<string | null>;
+ apiFetch: AskCardAuth['apiFetch'];
+ isAppActive?: () => boolean;
+ subscribeToAppState?: (listener: AppStateListener) => () => void;
+ getText: () => string;
+ setText: (value: string) => void;
+ getSelection: () => MobileVoiceSelection;
+ onListening: () => void;
+ onError: (message: string) => void;
+}): Promise<{ stop: () => Promise<void>; cancel: () => Promise<void> }> {
+ if (!isMobileRealtimeAudioAvailable()) {
+ throw new Error(mobileVoiceRealtimeAudioUnavailableError());
+ }
+ const { getRecordingPermissionsAsync, requestRecordingPermissionsAsync } = await import('expo-audio');
+ const isAppActive = input.isAppActive ?? (() => AppState.currentState === 'active');
+ const subscribeToAppState = input.subscribeToAppState ?? ((listener: AppStateListener) => {
+ const subscription = AppState.addEventListener('change', (state) => listener(state));
+ return () => subscription.remove();
+ });
+ const abort = new AbortController();
+ const permission = await resolveMobileVoiceRecordingPermission({
+ getPermission: getRecordingPermissionsAsync,
+ requestPermission: requestRecordingPermissionsAsync,
+ isRequestCurrent: () => !abort.signal.aborted,
+ isAppActive,
+ subscribeToAppState,
+ signal: abort.signal,
+ waitForAppActive: () => waitUntilActive(isAppActive, subscribeToAppState, abort.signal),
+ });
+ if (permission === 'denied') throw new Error(mobileVoiceMicPermissionError());
+ if (permission !== 'granted') throw new Error('语音输入已取消。');
+
+ const credential = createMobileCindyVoiceCredential(input.deviceId);
+ const voiceContext = new MobileCindyVoiceRunContext(
+ input.getAccessToken,
+ input.refreshAccessToken,
+ input.apiFetch as ConstructorParameters<typeof MobileCindyVoiceRunContext>[2],
+ credential.settings?.language,
+ CINDY_MANAGED_REFINER_PROVIDER,
+ );
+ let audioGate: Awaited<ReturnType<typeof createMobileSpeakerGate>> | undefined;
+ try {
+ audioGate = await createMobileSpeakerGate();
+ } catch {
+ audioGate = undefined;
+ }
+ const initialDraft = input.getText();
+ const session = createMobileVoiceControllerSession({
+ credential,
+ ...(audioGate ? { audioGate } : {}),
+ connectionProvider: (providerId) => voiceContext.createAsrConnection(providerId),
+ refinerTargetProvider: (providerId, options) => voiceContext.createRefinerTarget(providerId, options),
+ warmRefiner: (request) => voiceContext.warmRefiner(request),
+ initialDraft,
+ initialSelection: input.getSelection(),
+ readCurrentDraft: input.getText,
+ onDraftChanged: (text) => input.setText(text),
+ onStateChanged: (state) => {
+ if (state === 'listening') input.onListening();
+ },
+ onError: input.onError,
+ onReadyForEndCue: credential.settings?.playInteractionSound ? playMobileVoiceInputEndCue : undefined,
+ });
+ await session.start();
+ return {
+ stop: () => session.stop().then(() => undefined),
+ cancel: () => session.cancel(),
+ };
+}
+
+function waitUntilActive(
+ isAppActive: () => boolean,
+ subscribe: (listener: AppStateListener) => () => void,
+ signal: AbortSignal,
+): Promise<boolean> {
+ if (isAppActive()) return Promise.resolve(true);
+ return new Promise((resolve, reject) => {
+ const unsubscribe = subscribe((state) => {
+ if (state !== 'active' && !isAppActive()) return;
+ unsubscribe();
+ resolve(true);
+ });
+ signal.addEventListener('abort', () => {
+ unsubscribe();
+ reject(new Error('语音输入已取消。'));
+ }, { once: true });
+ });
+}
diff --git a/apps/mobile/src/session/useAskCardVoice.ts b/apps/mobile/src/session/useAskCardVoice.ts
new file mode 100644
index 0000000000..24976f2452
--- /dev/null
+++ b/apps/mobile/src/session/useAskCardVoice.ts
@@ -0,0 +1,98 @@
+/**
+ * Voice input for the mobile question card's free-text answer.
+ *
+ * The composer owns the full press-and-hold path. This hook starts the same
+ * dictation session against the answer string, and only loads the native audio
+ * modules when the user actually taps the microphone.
+ */
+import { useCallback, useRef, useState } from 'react';
+
+import { useAskCardAuth } from '@/session/askCardAuth';
+import type { MobileVoiceSelection } from '@/session/mobileVoiceInput';
+
+type VoiceRun = {
+ stop: () => Promise<void>;
+ cancel: () => Promise<void>;
+};
+
+export function useAskCardVoice(input: {
+ deviceId?: string;
+ getText: () => string;
+ setText: (value: string) => void;
+ getSelection: () => MobileVoiceSelection;
+}): {
+ listening: boolean;
+ error: string | null;
+ toggle: () => void;
+} {
+ const auth = useAskCardAuth();
+ const [listening, setListening] = useState(false);
+ const [error, setError] = useState<string | null>(null);
+ const runRef = useRef<VoiceRun | null>(null);
+ const startingRef = useRef(false);
+ const inputRef = useRef(input);
+ inputRef.current = input;
+ const authRef = useRef(auth);
+ authRef.current = auth;
+
+ const stop = useCallback(() => {
+ startingRef.current = false;
+ const run = runRef.current;
+ runRef.current = null;
+ setListening(false);
+ if (!run) return;
+ void run.stop().catch(() => undefined);
+ }, []);
+
+ const toggle = useCallback(() => {
+ if (runRef.current || startingRef.current) {
+ stop();
+ return;
+ }
+ const deviceId = inputRef.current.deviceId?.trim();
+ const currentAuth = authRef.current;
+ if (!deviceId || !currentAuth) {
+ setError('语音输入需要先登录手机端。');
+ return;
+ }
+ startingRef.current = true;
+ setError(null);
+ void (async () => {
+ try {
+ const { startAskCardDictation } = await import('@/session/askCardVoiceStart');
+ if (!startingRef.current) return;
+ const run = await startAskCardDictation({
+ deviceId,
+ getAccessToken: () => currentAuth.getAccessToken(),
+ refreshAccessToken: () => currentAuth.refreshAccessToken(),
+ apiFetch: currentAuth.apiFetch,
+ getText: () => inputRef.current.getText(),
+ setText: (value) => inputRef.current.setText(value),
+ getSelection: () => inputRef.current.getSelection(),
+ onListening: () => {
+ if (startingRef.current || runRef.current) setListening(true);
+ },
+ onError: (message) => {
+ startingRef.current = false;
+ runRef.current = null;
+ setListening(false);
+ setError(message);
+ },
+ });
+ if (!startingRef.current) {
+ await run.cancel();
+ return;
+ }
+ runRef.current = run;
+ startingRef.current = false;
+ setListening(true);
+ } catch (error) {
+ startingRef.current = false;
+ setListening(false);
+ setError(error instanceof Error ? error.message : String(error));
+ }
+ })();
+ }, [stop]);
+
+ return { listening, error, toggle };
+}