Skip to content

armgit / cindy

Public Imported from GitHub
0

Consider it done. The open-source AI agent that works out of the box · 想到,就能做到。开源、开箱即用的 AI Agent。

fix(voice): 允许提问卡片在语音输入期间提交回答

Dash2791aecd2a74
diff --git a/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx b/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
index 45988006bd..4d3150aae8 100644
--- a/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
+++ b/apps/desktop/src/renderer/__tests__/askUserQuestionVoice.test.tsx
@@ -11,16 +11,16 @@ import { AskUserQuestionPrompt } from '../components/new-chat/AskUserQuestionPro
import type { VoiceQuestionBridge } from '../components/new-chat/voiceQuestionSelection';
const start = vi.fn(() => true);
-const stop = vi.fn(async () => undefined);
+const stop = vi.fn(async (): Promise<string | undefined> => undefined);
// Stateful stand-in for the dictation hook: starting flips the card into the
// busy state, which is what disables question navigation.
vi.mock('../components/new-chat/useAskUserDictation', () => ({
- useAskUserDictation: () => {
- const [state, setState] = useState<'idle' | 'listening'>('idle');
+ useAskUserDictation: (target: { setText: (text: string) => void }) => {
+ const [state, setState] = useState<'idle' | 'listening' | 'done' | 'error'>('idle');
return {
state,
- isBusy: state !== 'idle',
+ isBusy: state === 'listening',
liveText: null,
lastError: null,
start: () => {
@@ -29,8 +29,13 @@ vi.mock('../components/new-chat/useAskUserDictation', () => ({
return true;
},
stop: async () => {
- await stop();
- setState('idle');
+ try {
+ const text = await stop();
+ if (text !== undefined) target.setText(text);
+ setState('done');
+ } catch {
+ setState('error');
+ }
},
};
},
@@ -38,7 +43,7 @@ vi.mock('../components/new-chat/useAskUserDictation', () => ({
beforeEach(async () => {
start.mockClear();
- stop.mockClear();
+ stop.mockReset();
await i18n.changeLanguage('zh-CN');
});
@@ -249,7 +254,7 @@ describe('AskUserQuestion voice entry', () => {
}
});
- it('blocks question navigation and submit while dictation owns the answer', async () => {
+ it('blocks skipping while dictation owns the answer', async () => {
const { view } = renderAsk([
{ question: '第一题?', options: [{ label: '甲' }, { label: '乙' }] },
{ question: '第二题?', options: [{ label: '丙' }, { label: '丁' }] },
@@ -259,14 +264,47 @@ describe('AskUserQuestion voice entry', () => {
await waitFor(() => expect(start).toHaveBeenCalledOnce());
// Switching questions mid-dictation would write the rest into another answer.
- for (const label of ['跳过', '下一题', '提交']) {
- const button = view.queryByText(label);
- if (button?.closest('button')) {
- expect((button.closest('button') as HTMLButtonElement).disabled).toBe(true);
- }
- }
expect((view.getByText('跳过').closest('button') as HTMLButtonElement).disabled).toBe(true);
});
+
+ // Regression: recording disabled Send, and Enter only stopped capture. Unit
+ // coverage previously asserted the broken disabled state; defer the final
+ // transcript to prove submission uses it exactly once across all answer forms.
+ it.each(['free', 'custom', 'multi', 'enter'])(
+ 'finishes dictation and submits the final %s answer once', async (mode) => {
+ let finish!: (text: string) => void;
+ stop.mockImplementationOnce(() => new Promise((resolve) => { finish = resolve; }));
+ const { view, onAnswer } = renderAsk([{
+ question: '问题',
+ ...(mode === 'custom' || mode === 'multi' ? { options: [{ label: '甲' }] } : {}),
+ multiSelect: mode === 'multi',
+ }]);
+ if (mode === 'multi') fireEvent.click(view.getByText('甲'));
+ fireEvent.click(view.getByTestId('ask-user-voice'));
+ const send = view.getByRole('button', { name: '提交' }) as HTMLButtonElement;
+ expect(send.disabled).toBe(false);
+ if (mode === 'enter') fireEvent.keyDown(view.getByPlaceholderText('请输入回答…'), { key: 'Enter' });
+ else fireEvent.click(send);
+ fireEvent.click(send);
+ expect(onAnswer).not.toHaveBeenCalled();
+ await act(async () => finish('最后一句'));
+ expect(stop).toHaveBeenCalledOnce();
+ expect(onAnswer).toHaveBeenCalledExactlyOnceWith('req-voice', {
+ '问题': mode === 'multi' ? '["甲","最后一句"]' : '最后一句',
+ });
+ },
+ );
+
+ it('keeps the existing answer when finishing dictation fails', async () => {
+ const { view, onAnswer } = renderAsk([{ question: '问题' }]);
+ fireEvent.change(view.getByPlaceholderText('请输入回答…'), { target: { value: '已有草稿' } });
+ fireEvent.click(view.getByTestId('ask-user-voice'));
+ stop.mockRejectedValueOnce(new Error('voice failed'));
+ await act(async () => fireEvent.click(view.getByRole('button', { name: '提交' })));
+ expect(stop).toHaveBeenCalledOnce();
+ expect(onAnswer).not.toHaveBeenCalled();
+ expect((view.getByPlaceholderText('请输入回答…') as HTMLTextAreaElement).value).toBe('已有草稿');
+ });
});
describe('voice shortcut ownership across cards', () => {
diff --git a/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx b/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
index 8d5fc0f9ae..a594fd6774 100644
--- a/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
+++ b/apps/desktop/src/renderer/components/new-chat/AskUserQuestionPrompt.tsx
@@ -254,6 +254,27 @@ function AskUserQuestionForm({
});
const dictationRef = useRef(dictation);
dictationRef.current = dictation;
+ const pendingVoiceSubmitRef = useRef<((text: string) => void) | null>(null);
+ const submitAfterDictation = useCallback((submit: (text: string) => void) => {
+ if (pendingVoiceSubmitRef.current) return;
+ if (!dictationRef.current.isBusy) {
+ submit(answerTextRef.current.trim());
+ return;
+ }
+ pendingVoiceSubmitRef.current = submit;
+ void dictationRef.current.stop().catch((error: unknown) => {
+ pendingVoiceSubmitRef.current = null;
+ toast.error(error instanceof Error ? error.message : String(error));
+ });
+ }, []);
+ useEffect(() => {
+ if (dictation.isBusy) return;
+ const submit = pendingVoiceSubmitRef.current;
+ pendingVoiceSubmitRef.current = null;
+ // stop() can return before refinement. Only the terminal event means the
+ // final text has landed; errors/cancellation keep the answer editable.
+ if (dictation.state === 'done') submit?.(answerTextRef.current.trim());
+ }, [dictation.isBusy, dictation.state]);
const pendingVoiceStartRef = useRef(false);
const voiceShortcutAtRef = useRef(0);
// A shortcut press emits `start` then, on release, `tap`. The flag lives in a
@@ -450,16 +471,16 @@ function AskUserQuestionForm({
}, [markStarted]);
// ── Multi-select: Next button → encode as JSON array string ──
- const handleNext = useCallback(() => {
- if (selectedLabels.size === 0 && !customInput.trim()) return;
+ const handleNext = useCallback(() => submitAfterDictation((text) => {
+ if (selectedLabels.size === 0 && !text) return;
// Maintain UI visual order: filter options by selected state, then append custom text
const parts: string[] = options.filter((o) => selectedLabels.has(o.label)).map((o) => o.label);
- if (customInput.trim()) {
+ if (text) {
// Custom text is an additional selected item
- parts.push(customInput.trim());
+ parts.push(text);
}
advance(JSON.stringify(parts));
- }, [selectedLabels, customInput, advance]);
+ }), [selectedLabels, options, advance, submitAfterDictation]);
// ── Back ──
const handleBack = useCallback(() => {
@@ -586,8 +607,7 @@ function AskUserQuestionForm({
}, [beginVoiceInput, hostVisible, viewerState, voiceBridge, voiceShortcutOrder, voiceShortcutPreferred]);
// ── Custom input submit (single-select mode) ──
- const handleCustomSubmit = useCallback(() => {
- const trimmed = customInput.trim();
+ const handleCustomSubmit = useCallback(() => submitAfterDictation((trimmed) => {
if (!trimmed) return;
if (isMultiSelect) {
// In multi-select, custom text is part of the selection — click Next to submit
@@ -596,16 +616,15 @@ function AskUserQuestionForm({
advance(trimmed);
// advance restores the next page when navigating. On the final page keep
// this field retryable until the host confirms and removes the card.
- }, [customInput, isMultiSelect, advance]);
+ }), [isMultiSelect, advance, submitAfterDictation]);
// ── Direct free-text answer when no option rows render ──
// Multi-select keeps the documented JSON-array encoding (`["text"]`), otherwise
// the answer cannot be parsed back when the user returns to edit this question.
- const submitDirectAnswer = useCallback(() => {
- const trimmed = customInput.trim();
+ const submitDirectAnswer = useCallback(() => submitAfterDictation((trimmed) => {
if (!trimmed) return;
advance(isMultiSelect ? JSON.stringify([trimmed]) : trimmed);
- }, [customInput, isMultiSelect, advance]);
+ }), [isMultiSelect, advance, submitAfterDictation]);
const voiceHandler = useRef<(text: string) => void>(() => {});
const voiceTarget = useMemo<VoiceQuestionTarget>(() => {
@@ -809,11 +828,11 @@ function AskUserQuestionForm({
if (isMultiSelect) {
handleNext();
} else {
- advance(existingAnswer ?? '');
+ submitAfterDictation((text) => advance(text || existingAnswer || ''));
}
}}
disabled={
- dictation.isBusy ||
+ !dictation.isBusy &&
(isMultiSelect
? selectedLabels.size === 0 && !customInput.trim()
: existingAnswer === undefined)
@@ -871,10 +890,6 @@ function AskUserQuestionForm({
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing) {
e.preventDefault();
- if (dictation.isBusy) {
- void dictation.stop();
- return;
- }
if (isMultiSelect) handleNext();
else handleCustomSubmit();
}
@@ -909,7 +924,7 @@ function AskUserQuestionForm({
<button
type="button"
onClick={handleCustomSubmit}
- disabled={!customInput.trim() || dictation.isBusy}
+ disabled={!customInput.trim() && !dictation.isBusy}
data-testid={isLastQuestion ? undefined : 'ask-user-custom-next'}
aria-label={isLastQuestion ? undefined : t('chat.askUserQuestion.next')}
className={cn(
@@ -917,7 +932,7 @@ function AskUserQuestionForm({
isLastQuestion
? 'px-[16px] py-[6px]'
: 'flex h-8 w-8 items-center justify-center',
- customInput.trim()
+ customInput.trim() || dictation.isBusy
? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
: 'bg-[var(--ask-send-disabled-bg)] text-[var(--ask-send-disabled-text)]',
)}
@@ -1098,10 +1113,6 @@ function AskUserQuestionForm({
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey && !e.nativeEvent.isComposing) {
e.preventDefault();
- if (dictation.isBusy) {
- void dictation.stop();
- return;
- }
submitDirectAnswer();
}
if (e.key === 'Escape') {
@@ -1129,13 +1140,13 @@ function AskUserQuestionForm({
<button
type="button"
onClick={submitDirectAnswer}
- disabled={!customInput.trim() || dictation.isBusy}
+ disabled={!customInput.trim() && !dictation.isBusy}
data-testid={isLastQuestion ? undefined : 'ask-user-custom-next'}
aria-label={isLastQuestion ? undefined : t('chat.askUserQuestion.next')}
className={cn(
'h-10 rounded-[9999px] text-14 font-medium transition-colors',
isLastQuestion ? 'px-4' : 'flex w-10 items-center justify-center',
- customInput.trim()
+ customInput.trim() || dictation.isBusy
? 'bg-[var(--ask-send-bg)] text-[var(--ask-send-text)]'
: 'bg-[var(--ask-send-disabled-bg)] text-[var(--ask-send-disabled-text)]',
)}
diff --git a/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts b/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
index 063989f930..2c8d23a69a 100644
--- a/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
+++ b/apps/desktop/src/renderer/components/new-chat/useAskUserDictation.ts
@@ -391,8 +391,12 @@ export function useAskUserDictation(
}
try {
const result = await window.electronAPI.voiceInput.stop();
- if (!result.ok) reportError(formatError(result.error));
+ if (!result.ok) {
+ setVoiceState('error');
+ reportError(formatError(result.error));
+ }
} catch (error) {
+ setVoiceState('error');
reportError(formatError(error instanceof Error ? error.message : String(error)));
} finally {
await restoreMute();
@@ -403,7 +407,7 @@ export function useAskUserDictation(
stopInFlightRef.current = null;
});
return stopInFlightRef.current;
- }, [formatError, reportError, restoreMute, stopEngine]);
+ }, [formatError, reportError, restoreMute, setVoiceState, stopEngine]);
useEffect(() => () => {
attemptRef.current += 1;
diff --git a/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx b/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
index fc2537bc51..b48ba0c153 100644
--- a/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
+++ b/apps/mobile/src/__tests__/InteractionPanelReceipt.test.tsx
@@ -13,6 +13,10 @@ import { ChatFilePathContext, type ChatFilePathContextValue } from '@/session/ch
import { QuestionReferenceText } from '@/session/QuestionReferenceText';
import { _clearRemotePathVerdictCache } from '@/session/remotePathVerdict';
import { i18n } from '@/i18n';
+import { AskCardAuthContext } from '@/session/askCardAuth';
+
+const { startDictation } = vi.hoisted(() => ({ startDictation: vi.fn() }));
+vi.mock('@/session/askCardVoiceStart', () => ({ startAskCardDictation: startDictation }));
const evidence = vi.hoisted(() => ({ openURL: vi.fn() }));
// As in LegendList, setters are inert outside a list item. The composer card
@@ -120,6 +124,55 @@ async function type(id: string, value: string) {
});
}
+// Unit regression: the real card/hook disabled submit during recording. Existing
+// voice coverage only exercised signed-out errors, never final text or failures.
+it.each(['free', 'custom', 'multi', 'failed', 'removed'])(
+ 'submits final voice text once and preserves failed/cancelled answers: %s', async (mode) => {
+ let finish!: () => void;
+ let fail!: (error: Error) => void;
+ const stop = vi.fn();
+ startDictation.mockImplementation(async (input: { onListening: () => void; setText: (text: string) => void }) => {
+ input.onListening();
+ stop.mockImplementation(() => new Promise<void>((resolve, reject) => {
+ finish = () => { input.setText('Final answer'); resolve(); };
+ fail = reject;
+ }));
+ return { stop, cancel: vi.fn(async () => undefined) };
+ });
+ remoteSessionStore.setPendingInteractions('s1', [{ request: {
+ kind: 'ask_user_question', requestId, questions: [{ question: 'Question?',
+ ...(mode === 'custom' || mode === 'multi' ? { options: [{ label: 'A' }] } : {}),
+ multiSelect: mode === 'multi',
+ }],
+ } }]);
+ resolveInteraction.mockResolvedValue({ accepted: true });
+ await act(async () => root.render(
+ <AskCardAuthContext.Provider value={{ getAccessToken: async () => 'test', refreshAccessToken: async () => 'test', apiFetch: async () => ({}) }}>
+ <Harness />
+ </AskCardAuthContext.Provider>,
+ ));
+ if (mode === 'multi') await click('interaction.ask.option.1');
+ if (mode === 'failed') await type('interaction.ask.textInput', 'Saved draft');
+ await click('interaction.ask.voice');
+ const send = host.querySelector<HTMLButtonElement>('[data-testid="interaction.ask.submitButton"]')!;
+ expect(send.disabled).toBe(false);
+ await click('interaction.ask.submitButton');
+ await click('interaction.ask.submitButton');
+ expect(stop).toHaveBeenCalledOnce();
+ expect(resolveInteraction).not.toHaveBeenCalled();
+ if (mode === 'removed') await act(async () => remoteSessionStore.setPendingInteractions('s1', []));
+ await act(async () => mode === 'failed' ? fail(new Error('Voice failed')) : finish());
+ if (mode === 'failed' || mode === 'removed') {
+ expect(resolveInteraction).not.toHaveBeenCalled();
+ if (mode === 'failed') expect(readAskUserDraft(requestId)?.customInput).toBe('Saved draft');
+ } else {
+ expect(resolveInteraction).toHaveBeenCalledExactlyOnceWith(requestId, {
+ kind: 'ask_user_question', answers: { 'Question?': mode === 'multi' ? '["A","Final answer"]' : 'Final answer' },
+ });
+ }
+ },
+);
+
// Unit regression for PR #6306 review: a signed-out microphone tap used to
// render hardcoded Chinese. Existing receipt tests never tapped this entry.
// Exercise the real card and hook in each supported locale, without auth.
diff --git a/apps/mobile/src/session/InteractionPanel.tsx b/apps/mobile/src/session/InteractionPanel.tsx
index f70157139e..58204fd66a 100644
--- a/apps/mobile/src/session/InteractionPanel.tsx
+++ b/apps/mobile/src/session/InteractionPanel.tsx
@@ -1043,6 +1043,7 @@ function AskUserQuestionForm({
const [showCustomInput, setShowCustomInput] = useState(() => initialDraft?.showCustomInput ?? false);
const answerTextRef = useRef(customInput);
answerTextRef.current = customInput;
+ const voiceSubmitPendingRef = useRef(false);
const selectionRef = useRef({ start: customInput.length, end: customInput.length });
const voice = useAskCardVoice({
deviceId,
@@ -1173,14 +1174,22 @@ function AskUserQuestionForm({
});
};
- const submitMulti = () => {
- if (!canSubmitMulti) return;
- advance(encodeMultiSelectAnswer(options, selectedLabels, customInput));
- };
-
- const submitSingle = () => {
- if (!canSubmitSingle || singleAnswer === undefined) return;
- advance(singleAnswer);
+ const submitAnswer = async () => {
+ if (voiceSubmitPendingRef.current) return;
+ voiceSubmitPendingRef.current = true;
+ try {
+ const text = await voice.stop();
+ if (text === null) return;
+ if (isMulti) {
+ if (selectedLabels.size === 0 && !text.trim()) return;
+ advance(encodeMultiSelectAnswer(options, selectedLabels, text));
+ } else {
+ const answer = customModeActive ? text.trim() : existingAnswer;
+ if (answer?.trim()) advance(answer);
+ }
+ } finally {
+ voiceSubmitPendingRef.current = false;
+ }
};
const toggleLabel = (label: string) => {
@@ -1392,10 +1401,9 @@ function AskUserQuestionForm({
<ResolveButton
accessibilityLabel={isLast ? t(resumeOnSubmit ? 'interaction.panel.sendAndContinue' : 'interaction.panel.submitAnswerAccessibility') : t('interaction.panel.next')}
busy={busy}
- disabled={voice.listening}
- invalidReason={!canSubmitMulti ? t('interaction.panel.answerRequired') : null}
+ invalidReason={!canSubmitMulti && !voice.listening ? t('interaction.panel.answerRequired') : null}
label={isLast ? t(resumeOnSubmit ? 'interaction.panel.sendAndContinue' : 'interaction.panel.submit') : t('interaction.panel.next')}
- onPress={submitMulti}
+ onPress={submitAnswer}
requestId={requestId}
touchStyle={resolveButtonLayoutStyle(touchLayout, 'primary', companion)}
testID="interaction.ask.submitButton"
@@ -1405,10 +1413,9 @@ function AskUserQuestionForm({
<ResolveButton
accessibilityLabel={isLast ? t(resumeOnSubmit ? 'interaction.panel.sendAndContinue' : 'interaction.panel.submitAnswerAccessibility') : t('interaction.panel.next')}
busy={busy}
- disabled={voice.listening}
- invalidReason={!canSubmitSingle ? t('interaction.panel.answerRequired') : null}
+ invalidReason={!canSubmitSingle && !voice.listening ? t('interaction.panel.answerRequired') : null}
label={isLast ? t(resumeOnSubmit ? 'interaction.panel.sendAndContinue' : 'interaction.panel.submit') : t('interaction.panel.next')}
- onPress={submitSingle}
+ onPress={submitAnswer}
requestId={requestId}
touchStyle={resolveButtonLayoutStyle(touchLayout, 'primary', companion)}
testID="interaction.ask.submitButton"
diff --git a/apps/mobile/src/session/useAskCardVoice.ts b/apps/mobile/src/session/useAskCardVoice.ts
index 4689081dae..c714f31489 100644
--- a/apps/mobile/src/session/useAskCardVoice.ts
+++ b/apps/mobile/src/session/useAskCardVoice.ts
@@ -25,13 +25,14 @@ export function useAskCardVoice(input: {
listening: boolean;
error: string | null;
toggle: () => void;
+ stop: () => Promise<string | null>;
} {
const auth = useAskCardAuth();
const [listening, setListening] = useState(false);
const [error, setError] = useState<string | null>(null);
const runRef = useRef<VoiceRun | null>(null);
const startingRef = useRef(false);
- const stoppingRef = useRef<Promise<void> | null>(null);
+ const stoppingRef = useRef<Promise<string | null> | null>(null);
const startAbortRef = useRef<AbortController | null>(null);
const inputRef = useRef(input);
inputRef.current = input;
@@ -39,19 +40,32 @@ export function useAskCardVoice(input: {
authRef.current = auth;
const stop = useCallback(() => {
+ if (stoppingRef.current) return stoppingRef.current;
+ const wasStarting = startingRef.current;
startingRef.current = false;
// Abort the startup so the microphone closes now instead of waiting for a
// slow voice-service handshake to finish.
startAbortRef.current?.abort();
startAbortRef.current = null;
const run = runRef.current;
- runRef.current = null;
- setListening(false);
- if (!run) return;
- const stopping = run.stop().catch(() => undefined).finally(() => {
+ if (!run) {
+ setListening(false);
+ return Promise.resolve(wasStarting ? null : inputRef.current.getText());
+ }
+ // Keep the run owned until final ASR/refinement settles. Unmount can then
+ // cancel it, and another Send joins this same stop instead of using a draft.
+ const stopping = run.stop().then(() => (
+ runRef.current === run ? inputRef.current.getText() : null
+ )).catch((error: unknown) => {
+ setError(error instanceof Error ? error.message : String(error));
+ return null;
+ }).finally(() => {
+ if (runRef.current === run) runRef.current = null;
+ setListening(false);
if (stoppingRef.current === stopping) stoppingRef.current = null;
});
stoppingRef.current = stopping;
+ return stopping;
}, []);
// The card unmounts when the answer is submitted, skipped or the request
@@ -126,5 +140,5 @@ export function useAskCardVoice(input: {
})();
}, [stop]);
- return { listening, error, toggle };
+ return { listening, error, toggle, stop };
}