Skip to content

armgit / avibe

Public Imported from GitHub
0

Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.

fix(desktop): give mini voice failures a text fallback and refresh ASR status

Maxab4973259a03
diff --git a/docs/plans/2026-10-12-desktop-mini-delivery.md b/docs/plans/2026-10-12-desktop-mini-delivery.md
index e7441c622..0fb1461e3 100644
--- a/docs/plans/2026-10-12-desktop-mini-delivery.md
+++ b/docs/plans/2026-10-12-desktop-mini-delivery.md
@@ -49,19 +49,25 @@ quick-reply, and pet lifecycle paths. Backend APIs are outside this change.
wait; its dictation implementation is not refactored. Pet uses the existing
`ShowPageVoiceDictation` wrapper and thus the same capture/transcription pipeline.
-## Voice: enabled, handoff verified on real hardware
+## Voice: enabled; real-hardware handoff pending
Pet voice was first shipped behind a `__AVIBE_PET_VOICE_ENABLED__` window marker
that no production shell installed, so the microphone controls stayed hidden
everywhere. The owner has since removed that gate so the captured-speech handoff
can be verified on a real desktop test prerelease. Pet voice now appears
whenever Web Locks, `getUserMedia` and ASR are all available; missing Web Locks
-still denies it, and hotkey/hold still falls back to text in that case.
+still denies it, and hotkey/hold still falls back to text in that case. ASR
+status is re-read whenever the pet window becomes active, so enabling or
+disabling transcription in settings takes effect without restarting the pet.
+A microphone or transcription failure reopens the text input with the pre-voice
+draft and a short note; the mini does not keep the failed recording for retry
+(the main Composer does).
Acceptance on the test prerelease: start dictation in the main Composer, then
claim the microphone from the mini (and the reverse). Each side must keep the
transcript of the speech it captured before the handoff. If either direction
-loses speech, fix the handoff before the next stable release.
+loses speech, fix the handoff before the next stable release. Until that
+transcript evidence is recorded here, the handoff remains unverified.
In the isolated macOS desktop build, `main` and `pet` both exposed Web Locks and
reported the same `avibe.voice-capture` owner/client ID. A pet claim changed that
diff --git a/ui/src/i18n/en.json b/ui/src/i18n/en.json
index f4d738515..0cf32fda2 100644
--- a/ui/src/i18n/en.json
+++ b/ui/src/i18n/en.json
@@ -5610,7 +5610,10 @@
"cloudRequired": "Voice input needs Avibe Cloud. You can type a message instead.",
"listening": "Listening…",
"open": "Voice input",
- "stop": "Stop voice input"
+ "stop": "Stop voice input",
+ "startFailed": "Couldn't use the microphone. Check microphone access in system settings, or type instead.",
+ "empty": "No speech was heard. Try again, or type instead.",
+ "failed": "Couldn't turn that into text. Try again, or type instead."
},
"keyboard": {
"open": "Type a message",
diff --git a/ui/src/i18n/zh.json b/ui/src/i18n/zh.json
index afbfa4aa2..42fc77f7c 100644
--- a/ui/src/i18n/zh.json
+++ b/ui/src/i18n/zh.json
@@ -5610,7 +5610,10 @@
"cloudRequired": "语音输入需要 Avibe Cloud,暂时可以打字发送。",
"listening": "正在听…",
"open": "语音输入",
- "stop": "停止语音输入"
+ "stop": "停止语音输入",
+ "startFailed": "无法使用麦克风。请在系统设置里检查麦克风权限,或直接打字。",
+ "empty": "没听到说话。可以再说一次,或直接打字。",
+ "failed": "没能转成文字。可以再说一次,或直接打字。"
},
"keyboard": {
"open": "输入消息",
diff --git a/ui/src/pet/PetPage.test.tsx b/ui/src/pet/PetPage.test.tsx
index a62d9095b..59b01fa50 100644
--- a/ui/src/pet/PetPage.test.tsx
+++ b/ui/src/pet/PetPage.test.tsx
@@ -1,5 +1,6 @@
/* @vitest-environment jsdom */
+import type { VoiceTranscriptionError as VoiceTranscriptionErrorType } from '@/lib/voiceTranscription';
import { fireEvent, act, cleanup, render, screen, waitFor, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
@@ -1633,6 +1634,71 @@ describe('desktop mini v2 interaction contract', () => {
expect(screen.queryByText('Session B')).toBeNull();
});
+ async function withVoiceEnv(asrAvailable: () => boolean, dictation: unknown, body: () => Promise<void>) {
+ vi.doMock('@/lib/showPageVoiceDictation', () => ({ ShowPageVoiceDictation: dictation }));
+ const locks = Object.getOwnPropertyDescriptor(navigator, 'locks');
+ const media = Object.getOwnPropertyDescriptor(navigator, 'mediaDevices');
+ try {
+ Object.defineProperty(navigator, 'locks', { configurable: true, value: {} });
+ Object.defineProperty(navigator, 'mediaDevices', { configurable: true, value: { getUserMedia: vi.fn() } });
+ vi.stubGlobal('fetch', vi.fn(async () => ({ ok: true, json: async () => ({ available: asrAvailable() }) })));
+ vi.resetModules();
+ ({ PetPage } = await import('./PetPage'));
+ await body();
+ } finally {
+ vi.doUnmock('@/lib/showPageVoiceDictation');
+ vi.unstubAllGlobals();
+ if (locks) Object.defineProperty(navigator, 'locks', locks);
+ else Reflect.deleteProperty(navigator, 'locks');
+ if (media) Object.defineProperty(navigator, 'mediaDevices', media);
+ else Reflect.deleteProperty(navigator, 'mediaDevices');
+ }
+ }
+
+ it.each([
+ ['microphone start', 'pet.voice.startFailed', () => ({ start: Promise.reject(new DOMException('denied', 'NotAllowedError')), done: new Promise<string>(() => {}) })],
+ ['transcription', 'pet.voice.failed', (TranscriptionError: typeof VoiceTranscriptionErrorType) => ({ start: Promise.resolve(), done: Promise.reject(new TranscriptionError('failed')) })],
+ ])('falls back to text with a note when %s fails, keeping the typed draft', async (_label, note, outcome) => {
+ // Resolved after the module reset so PetPage and the test share one class.
+ let TranscriptionError!: typeof VoiceTranscriptionErrorType;
+ class Dictation {
+ private readonly outcome = outcome(TranscriptionError);
+ done = this.outcome.done;
+ start = () => this.outcome.start;
+ finish = () => {};
+ abort = () => {};
+ }
+ await withVoiceEnv(() => true, Dictation, async () => {
+ ({ VoiceTranscriptionError: TranscriptionError } = await import('@/lib/voiceTranscription'));
+ devBind('S'); render(<PetPage />); summon('listen');
+ const input = await screen.findByRole('textbox');
+ await userEvent.type(input, 'typed before voice');
+ await userEvent.click(await screen.findByLabelText('pet.voice.open'));
+ expect(await screen.findByText(note)).toBeTruthy();
+ expect(pose()).not.toBe('listening');
+ expect((screen.getByRole('textbox') as HTMLTextAreaElement).value).toBe('typed before voice');
+ expect(api.sendSessionMessage).not.toHaveBeenCalled();
+ });
+ });
+
+ it('shows the microphone once transcription is turned on elsewhere and the window is reactivated', async () => {
+ let asr = false;
+ class Dictation { done = new Promise<string>(() => {}); start = async () => {}; finish = () => {}; abort = () => {}; }
+ await withVoiceEnv(() => asr, Dictation, async () => {
+ devBind('S'); render(<PetPage />); summon('listen');
+ await screen.findByRole('textbox');
+ await waitFor(() => expect(fetch).toHaveBeenCalled());
+ expect(screen.queryByLabelText('pet.voice.open')).toBeNull();
+ asr = true;
+ const focus = vi.spyOn(document, 'hasFocus').mockReturnValue(false);
+ act(() => { window.dispatchEvent(new Event('blur')); });
+ focus.mockReturnValue(true);
+ act(() => { window.dispatchEvent(new Event('focus')); });
+ expect(await screen.findByLabelText('pet.voice.open')).toBeTruthy();
+ focus.mockRestore();
+ });
+ });
+
it('discards voice preview and final text from the previous authorization epoch', async () => {
const done = deferred<string>();
let preview!: (text: string) => void;
diff --git a/ui/src/pet/PetPage.tsx b/ui/src/pet/PetPage.tsx
index c4c4b0da0..9f68f3ef1 100644
--- a/ui/src/pet/PetPage.tsx
+++ b/ui/src/pet/PetPage.tsx
@@ -13,6 +13,7 @@ import { harnessItemNativeId } from '@/lib/backgroundActivity';
import { usePageActive } from '@/lib/pageActivity';
import { usePendingVaultRequests } from '@/lib/usePendingVaultRequests';
import { ShowPageVoiceDictation } from '@/lib/showPageVoiceDictation';
+import { VoiceTranscriptionError } from '@/lib/voiceTranscription';
import { PetAvatar, type PetPose } from './PetAvatar';
import { petBridge, sessionLink, vaultRequestLink } from './petBridge';
@@ -392,6 +393,9 @@ const PetSurface: React.FC = () => {
onOpenComposer: openComposer,
});
+ const clearVoiceError = voice.clearError;
+ useEffect(() => { if (!composing) clearVoiceError(); }, [composing, clearVoiceError]);
+
const hotkeyHeld = useRef(false);
const onSummon = useCallback(() => {
const intent = petShell.takeSummon();
@@ -566,6 +570,8 @@ const PetSurface: React.FC = () => {
</div>
<SpeechSlot open={voice.listening || composing}>
{voice.listening ? <Marquee text={voice.preview} listeningLabel={t('pet.voice.listening')} /> : canChat && !uncertain ? (
+ <>
+ {voice.error && <p className="pet-mini-input-note" role="status">{t(`pet.voice.${voice.error}`)}</p>}
<form className="pet-mini-input" data-pet-hit onSubmit={(event) => { event.preventDefault(); void submit(); }}>
<textarea
ref={inputRef}
@@ -584,6 +590,7 @@ const PetSurface: React.FC = () => {
{voice.available && <button type="button" className="pet-mini-input-button" onClick={() => voice.start()} aria-label={t('pet.voiceContinue')}><MicIcon /></button>}
<button type="submit" className="pet-mini-send" disabled={!draft.trim() || sending || uncertain} aria-label={t('pet.send')}><SendIcon /></button>
</form>
+ </>
) : null}
</SpeechSlot>
</div>
@@ -711,18 +718,23 @@ function usePetVoice({ enabled, epoch, draft, onOpenComposer }: { enabled: boole
const [asr, setAsr] = useState<{ available: boolean; checked: boolean; maxFileBytes?: number }>({ available: false, checked: false });
const [preview, setPreview] = useState('');
const [listening, setListening] = useState(false);
+ // A failed capture or transcription falls back to text with this note.
+ const [error, setError] = useState<VoiceFailure | null>(null);
+ const pageActive = usePageActive();
type Capture = { dictation: ShowPageVoiceDictation; ending: boolean; cancelled: boolean };
const captureRef = useRef<Capture | null>(null);
const supported = typeof navigator !== 'undefined' && Boolean(navigator.locks && navigator.mediaDevices?.getUserMedia);
const available = enabled && supported && asr.available;
+ // The pet window outlives settings changes made elsewhere, so ASR status is
+ // re-read whenever the window becomes active again.
useEffect(() => {
- if (!enabled) return;
+ if (!enabled || !pageActive) return;
let alive = true;
void fetch('/api/asr/status', { credentials: 'same-origin' }).then((response) => response.ok ? response.json() : null).then((value) => {
if (alive && epoch.isCurrent()) setAsr({ available: Boolean(value?.available), checked: value !== null, maxFileBytes: value?.max_file_bytes });
}).catch(() => { if (alive && epoch.isCurrent()) setAsr({ available: false, checked: false }); });
return () => { alive = false; };
- }, [enabled, epoch]);
+ }, [enabled, epoch, pageActive]);
const stop = useCallback((commit: boolean) => {
const capture = captureRef.current;
if (!capture) return;
@@ -746,6 +758,7 @@ function usePetVoice({ enabled, epoch, draft, onOpenComposer }: { enabled: boole
};
captureRef.current = capture;
setListening(true);
+ setError(null);
const settle = () => {
if (captureRef.current !== capture || !epoch.isCurrent()) return false;
captureRef.current = null;
@@ -753,20 +766,38 @@ function usePetVoice({ enabled, epoch, draft, onOpenComposer }: { enabled: boole
setPreview('');
return !capture.cancelled;
};
+ // Whichever of start/done rejects first settles; the other is a no-op.
+ const fail = (reason: unknown) => {
+ if (!settle()) return;
+ if (reason instanceof VoiceTranscriptionError && reason.code === 'cancelled') return;
+ setError(voiceFailure(reason));
+ onOpenComposer(before);
+ };
void capture.dictation.done.then((text) => {
if (!settle()) return;
const next = text.trim();
onOpenComposer(next ? [before.trim(), next].filter(Boolean).join(' ') : before);
- }).catch(settle);
+ }).catch((reason: unknown) => fail(reason));
void capture.dictation.start().then(() => {
if (!epoch.isCurrent()) { capture.dictation.abort(); return; }
// Release may precede microphone permission or pipeline startup.
if (capture.ending && !capture.cancelled) capture.dictation.finish();
- }).catch(() => { settle(); capture.dictation.abort(); });
+ }).catch((reason: unknown) => { fail(reason); capture.dictation.abort(); });
}, [asr.maxFileBytes, available, epoch, draft, onOpenComposer]);
const cancel = useCallback(() => stop(false), [stop]);
+ const clearError = useCallback(() => setError(null), []);
useEffect(() => cancel, [epoch, enabled, cancel]);
- return { available, cloudRequired: asr.checked && !asr.available, listening, preview, start, stop, cancel };
+ useEffect(() => () => setError(null), [epoch, enabled]);
+ return { available, cloudRequired: asr.checked && !asr.available, listening, preview, error, clearError, start, stop, cancel };
+}
+
+type VoiceFailure = 'startFailed' | 'empty' | 'failed';
+
+// Anything that is not a transcription error comes from getting the
+// microphone itself (permission denied, no input device, capture setup).
+function voiceFailure(reason: unknown): VoiceFailure {
+ if (!(reason instanceof VoiceTranscriptionError)) return 'startFailed';
+ return reason.code === 'empty' ? 'empty' : 'failed';
}
const SpeechSlot: React.FC<{ open: boolean; children: ReactNode }> = ({ open, children }) => (
diff --git a/ui/src/pet/pet-mini.css b/ui/src/pet/pet-mini.css
index f6b86543a..9cded0280 100644
--- a/ui/src/pet/pet-mini.css
+++ b/ui/src/pet/pet-mini.css
@@ -683,3 +683,10 @@ to {
@media (prefers-reduced-motion: reduce) {
.pet-mini-cards:has(.pet-mini-swap.is-switching) { overflow: visible; }
}
+
+.pet-mini-input-note {
+ margin:0 0 6px;
+ padding:0 14px;
+ color:color-mix(in srgb,var(--pet-ink) 70%,transparent);
+ font-size:12px;
+ line-height:16px}