DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(voice): preserve math and isolate reply playback lifecycle
Bingxi Zhao (Frank)a3c76c9d27f2
diff --git a/deeptutor/services/voice/base.py b/deeptutor/services/voice/base.pyindex b14c89657..bd5bdf461 100644--- a/deeptutor/services/voice/base.py+++ b/deeptutor/services/voice/base.py@@ -159,9 +159,8 @@ _BLANK_LINES = re.compile(r"\n{3,}")def _unwrap_emphasis_for_speech(text: str) -> str:"""Turn Markdown bold/italic/strike into plain words.- Voice models otherwise speak ``*`` as "asterisk". Math must already have- been verbalized so TeX ``*`` / ``_`` inside ``$…$`` is not treated as- emphasis. Leftover unmatched ``**`` markers are dropped; a remaining+ Voice models otherwise speak ``*`` as "asterisk". Call this on verbalized+ math, or only on prose segments when math verbalization is disabled. Leftover unmatched ``**`` markers are dropped; a remainingsingle ``*`` is turned into a space so "asterisk" is never read."""out = _BOLD_STARS.sub(r"\1", text)@@ -200,8 +199,13 @@ def strip_markdown_for_speech(text: str, *, max_chars: int = 0, math_speak: bool# Math before emphasis: TeX uses `_` / `*` as scripts and products, and# the emphasis regex would otherwise pair a prose underscore with one# inside `$x_i$`.- out = verbalize_latex_for_speech(out, math_speak=math_speak)- out = _unwrap_emphasis_for_speech(out)+ out = verbalize_latex_for_speech(+ out,+ math_speak=math_speak,+ prose_transform=_unwrap_emphasis_for_speech if not math_speak else None,+ )+ if math_speak:+ out = _unwrap_emphasis_for_speech(out)out = _WHITESPACE.sub(" ", out)out = _BLANK_LINES.sub("\n\n", out).strip()if max_chars and len(out) > max_chars:diff --git a/deeptutor/services/voice/speech_text.py b/deeptutor/services/voice/speech_text.pyindex 00e61633e..76395057b 100644--- a/deeptutor/services/voice/speech_text.py+++ b/deeptutor/services/voice/speech_text.py@@ -18,6 +18,7 @@ Scope is deliberately Pareto, not a full MathSpeak engine:from __future__ import annotations+from collections.abc import Callableimport re# ── lexicons ──────────────────────────────────────────────────────────────@@ -735,7 +736,9 @@ def _emit_island(inner: str, *, math_speak: bool) -> str:return _verbalize_math(inner) if math_speak else inner-def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:+def verbalize_latex_for_speech(+ text: str, *, math_speak: bool = True, prose_transform: Callable[[str], str] | None = None+) -> str:"""Replace math islands with speakable prose, or just unwrap delimiters.``math_speak=True`` (default) verbalizes fractions, powers, Greek, and@@ -743,39 +746,46 @@ def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:wrappers so TTS never says "dollar", but leaves the inner TeX as-is.Bare underscores outside math are preserved (``file_name`` stays intact).- Leftover ``$`` from broken markup is dropped.+ Leftover ``$`` from broken markup is dropped. An optional prose transform+ cleans surrounding Markdown while leaving each math island intact."""if not text:return ""if "$" not in text and "\\" not in text:- return _replace_unicode(text) if math_speak else text+ spoken = _replace_unicode(text) if math_speak else text+ return prose_transform(spoken) if prose_transform else spokenout: list[str] = []+ prose: list[str] = []++ def flush_prose() -> None:+ segment = "".join(prose)+ out.append(prose_transform(segment) if prose_transform else segment)+ prose.clear()++ def append_island(inner: str) -> None:+ flush_prose()+ out.extend((" ", _emit_island(inner, math_speak=math_speak), " "))+i = 0n = len(text)while i < n:if text.startswith("$$", i) and not _is_escaped(text, i):end = text.find("$$", i + 2)if end != -1:- out.append(" ")- out.append(_emit_island(text[i + 2 : end], math_speak=math_speak))- out.append(" ")+ append_island(text[i + 2 : end])i = end + 2continueif text.startswith("\\[", i):end = text.find("\\]", i + 2)if end != -1:- out.append(" ")- out.append(_emit_island(text[i + 2 : end], math_speak=math_speak))- out.append(" ")+ append_island(text[i + 2 : end])i = end + 2continueif text.startswith("\\(", i):end = text.find("\\)", i + 2)if end != -1:- out.append(" ")- out.append(_emit_island(text[i + 2 : end], math_speak=math_speak))- out.append(" ")+ append_island(text[i + 2 : end])i = end + 2continuebegin = _BEGIN_ENV.match(text, i)@@ -784,9 +794,7 @@ def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:end_tag = f"\\end{{{env}}}"end = text.find(end_tag, begin.end())if end != -1:- out.append(" ")- out.append(_emit_island(text[begin.end() : end], math_speak=math_speak))- out.append(" ")+ append_island(text[begin.end() : end])i = end + len(end_tag)continueif text[i] == "$" and not _is_escaped(text, i):@@ -799,14 +807,13 @@ def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:breakj += 1if j > i:- out.append(" ")- out.append(_emit_island(text[i + 1 : j], math_speak=math_speak))- out.append(" ")+ append_island(text[i + 1 : j])i = j + 1continue- out.append(text[i])+ prose.append(text[i])i += 1+ flush_prose()spoken = "".join(out)if math_speak:spoken = _verbalize_loose_commands(spoken)diff --git a/tests/services/test_voice.py b/tests/services/test_voice.pyindex 3196e5a0f..8211509ca 100644--- a/tests/services/test_voice.py+++ b/tests/services/test_voice.py@@ -732,3 +732,21 @@ async def test_transcribe_audio_facade(monkeypatch: pytest.MonkeyPatch) -> None:)assert text == "transcribed"assert captured["json"]["input_audio"]["format"] == "webm"+++ "formula",+ ["$x*y*z$", "$$x*y*z$$", r"\(x*y*z\)", r"\[x*y*z\]", r"\begin{align}x*y*z\end{align}"],+)+def test_math_speak_off_preserves_products_while_cleaning_prose(formula):+ assert (+ strip_markdown_for_speech(f"**Multiply** {formula} and *compare*.", math_speak=False)+ == "Multiply x*y*z and compare."+ )+++def test_math_speak_off_keeps_tex_scripts_separate_from_prose_emphasis():+ assert (+ strip_markdown_for_speech(r"*Use* $x_i*y_j$ in file_name.", math_speak=False)+ == "Use x_i*y_j in file_name."+ )diff --git a/web/features/chat/messages/ChatMessageList.tsx b/web/features/chat/messages/ChatMessageList.tsxindex 0c33ef41e..3d2b20cfb 100644--- a/web/features/chat/messages/ChatMessageList.tsx+++ b/web/features/chat/messages/ChatMessageList.tsx@@ -1294,7 +1294,7 @@ export function CopyActionButton({// Speaker button: synthesizes this one reply and plays it. Auto-play of later// replies is a Settings preference (`autoPlayFresh`), not a first-click prompt.-let stopActivePlayback: (() => void) | null = null;+let activePlayback: { owner: object; stop: () => void } | null = null;async function ttsErrorMessage(resp: Response): Promise<string> {try {@@ -1327,6 +1327,7 @@ export function PlayAudioButton({const abortRef = useRef<AbortController | null>(null);const genRef = useRef(0);const autoPlayedRef = useRef(false);+ const playbackOwnerRef = useRef<object>({});const cleanup = useCallback(() => {abortRef.current?.abort();@@ -1345,16 +1346,16 @@ export function PlayAudioButton({genRef.current += 1;cleanup();setState("idle");- if (stopActivePlayback === stop) stopActivePlayback = null;+ if (activePlayback?.owner === playbackOwnerRef.current) activePlayback = null;}, [cleanup]);const play = useCallback(async () => {- stopActivePlayback?.();+ activePlayback?.stop();const gen = ++genRef.current;abortRef.current?.abort();const ac = new AbortController();abortRef.current = ac;- stopActivePlayback = stop;+ activePlayback = { owner: playbackOwnerRef.current, stop };setState("loading");try {const resp = await apiFetch(apiUrl("/api/voice/tts"), {@@ -1387,7 +1388,7 @@ export function PlayAudioButton({if (gen !== genRef.current) return;setState("idle");cleanup();- if (stopActivePlayback === stop) stopActivePlayback = null;+ if (activePlayback?.owner === playbackOwnerRef.current) activePlayback = null;};audio.onerror = () => {if (gen !== genRef.current) return;@@ -1430,7 +1431,11 @@ export function PlayAudioButton({return () => window.clearTimeout(id);}, [autoPlayFresh, autoplayEnabled, content, play]);- useEffect(() => cleanup, [cleanup]);+ useEffect(() => () => {+ genRef.current += 1;+ cleanup();+ if (activePlayback?.owner === playbackOwnerRef.current) activePlayback = null;+ }, [cleanup]);return (<div className="relative inline-flex">diff --git a/web/tests/chat-read-aloud-playback.spec.tsx b/web/tests/chat-read-aloud-playback.spec.tsxnew file mode 100644index 000000000..576ed20cc--- /dev/null+++ b/web/tests/chat-read-aloud-playback.spec.tsx@@ -0,0 +1,57 @@+import { act, render, screen } from "@testing-library/react";+import userEvent from "@testing-library/user-event";+import { afterEach, expect, it, vi } from "vitest";+import { PlayAudioButton } from "@/features/chat/messages/ChatMessageList";+import { apiFetch } from "@/lib/api";+import { initI18n } from "@/i18n/init";++vi.mock("@/lib/api", async (original) => ({+ ...(await original<typeof import("@/lib/api")>()),+ apiFetch: vi.fn(),+}));+vi.mock("@/hooks/useVoiceAutoplay", () => ({+ useVoiceAutoplay: () => ({ autoplayEnabled: false }),+}));+initI18n("en");+const instances: FakeAudio[] = [];+class FakeAudio {+ pause = vi.fn();+ play = vi.fn().mockResolvedValue(undefined);+ onended: (() => void) | null = null;+ onerror: (() => void) | null = null;+ constructor() { instances.push(this); }+}+function installAudio() {+ vi.stubGlobal("Audio", FakeAudio);+ Object.defineProperty(URL, "createObjectURL", { configurable: true, value: vi.fn(() => "blob:test") });+ Object.defineProperty(URL, "revokeObjectURL", { configurable: true, value: vi.fn() });+}+afterEach(() => { vi.unstubAllGlobals(); instances.length = 0; });++it("switches between reply buttons without overlapping speech or a first-click prompt", async () => {+ installAudio();+ vi.mocked(apiFetch).mockResolvedValue({ ok: true, blob: async () => new Blob(["audio"]) } as Response);+ render(<><PlayAudioButton content="First" autoPlayFresh={false} /><PlayAudioButton content="Second" autoPlayFresh={false} /></>);+ const user = userEvent.setup();+ await user.click(screen.getAllByRole("button", { name: "Play aloud" })[0]);+ expect(await screen.findByRole("button", { name: "Stop" })).toBeVisible();+ await user.click(screen.getByRole("button", { name: "Play aloud" }));+ expect(instances).toHaveLength(2);+ expect(instances[0].pause).toHaveBeenCalled();+ expect(instances[1].pause).not.toHaveBeenCalled();+ expect(screen.getAllByRole("button", { name: "Stop" })).toHaveLength(1);+ expect(screen.queryByText(/Automatically read/)).toBeNull();+});++it("does not play a late response body after leaving the chat", async () => {+ installAudio();+ let resolve!: (blob: Blob) => void;+ const body = new Promise<Blob>((yes) => { resolve = yes; });+ vi.mocked(apiFetch).mockResolvedValueOnce({ ok: true, blob: () => body } as Response);+ const { unmount } = render(<PlayAudioButton content="First" autoPlayFresh={false} />);+ await userEvent.setup().click(screen.getByRole("button", { name: "Play aloud" }));+ unmount();+ await act(async () => { resolve(new Blob(["audio"])); });+ expect(instances).toHaveLength(0);+ expect(URL.createObjectURL).not.toHaveBeenCalled();+});