Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(voice): preserve math and isolate reply playback lifecycle

Bingxi Zhao (Frank)a3c76c9d27f2
diff --git a/deeptutor/services/voice/base.py b/deeptutor/services/voice/base.py
index b14c89657..bd5bdf461 100644
--- a/deeptutor/services/voice/base.py
+++ b/deeptutor/services/voice/base.py
@@ -159,9 +159,8 @@ _BLANK_LINES = re.compile(r"\n{3,}")
def _unwrap_emphasis_for_speech(text: str) -> str:
"""Turn Markdown bold/italic/strike into plain words.
- Voice models otherwise speak ``*`` as "asterisk". Math must already have
- been verbalized so TeX ``*`` / ``_`` inside ``$…$`` is not treated as
- emphasis. Leftover unmatched ``**`` markers are dropped; a remaining
+ Voice models otherwise speak ``*`` as "asterisk". Call this on verbalized
+ math, or only on prose segments when math verbalization is disabled. Leftover unmatched ``**`` markers are dropped; a remaining
single ``*`` is turned into a space so "asterisk" is never read.
"""
out = _BOLD_STARS.sub(r"\1", text)
@@ -200,8 +199,13 @@ def strip_markdown_for_speech(text: str, *, max_chars: int = 0, math_speak: bool
# Math before emphasis: TeX uses `_` / `*` as scripts and products, and
# the emphasis regex would otherwise pair a prose underscore with one
# inside `$x_i$`.
- out = verbalize_latex_for_speech(out, math_speak=math_speak)
- out = _unwrap_emphasis_for_speech(out)
+ out = verbalize_latex_for_speech(
+ out,
+ math_speak=math_speak,
+ prose_transform=_unwrap_emphasis_for_speech if not math_speak else None,
+ )
+ if math_speak:
+ out = _unwrap_emphasis_for_speech(out)
out = _WHITESPACE.sub(" ", out)
out = _BLANK_LINES.sub("\n\n", out).strip()
if max_chars and len(out) > max_chars:
diff --git a/deeptutor/services/voice/speech_text.py b/deeptutor/services/voice/speech_text.py
index 00e61633e..76395057b 100644
--- a/deeptutor/services/voice/speech_text.py
+++ b/deeptutor/services/voice/speech_text.py
@@ -18,6 +18,7 @@ Scope is deliberately Pareto, not a full MathSpeak engine:
from __future__ import annotations
+from collections.abc import Callable
import re
# ── lexicons ──────────────────────────────────────────────────────────────
@@ -735,7 +736,9 @@ def _emit_island(inner: str, *, math_speak: bool) -> str:
return _verbalize_math(inner) if math_speak else inner
-def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:
+def verbalize_latex_for_speech(
+ text: str, *, math_speak: bool = True, prose_transform: Callable[[str], str] | None = None
+) -> str:
"""Replace math islands with speakable prose, or just unwrap delimiters.
``math_speak=True`` (default) verbalizes fractions, powers, Greek, and
@@ -743,39 +746,46 @@ def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:
wrappers so TTS never says "dollar", but leaves the inner TeX as-is.
Bare underscores outside math are preserved (``file_name`` stays intact).
- Leftover ``$`` from broken markup is dropped.
+ Leftover ``$`` from broken markup is dropped. An optional prose transform
+ cleans surrounding Markdown while leaving each math island intact.
"""
if not text:
return ""
if "$" not in text and "\\" not in text:
- return _replace_unicode(text) if math_speak else text
+ spoken = _replace_unicode(text) if math_speak else text
+ return prose_transform(spoken) if prose_transform else spoken
out: list[str] = []
+ prose: list[str] = []
+
+ def flush_prose() -> None:
+ segment = "".join(prose)
+ out.append(prose_transform(segment) if prose_transform else segment)
+ prose.clear()
+
+ def append_island(inner: str) -> None:
+ flush_prose()
+ out.extend((" ", _emit_island(inner, math_speak=math_speak), " "))
+
i = 0
n = len(text)
while i < n:
if text.startswith("$$", i) and not _is_escaped(text, i):
end = text.find("$$", i + 2)
if end != -1:
- out.append(" ")
- out.append(_emit_island(text[i + 2 : end], math_speak=math_speak))
- out.append(" ")
+ append_island(text[i + 2 : end])
i = end + 2
continue
if text.startswith("\\[", i):
end = text.find("\\]", i + 2)
if end != -1:
- out.append(" ")
- out.append(_emit_island(text[i + 2 : end], math_speak=math_speak))
- out.append(" ")
+ append_island(text[i + 2 : end])
i = end + 2
continue
if text.startswith("\\(", i):
end = text.find("\\)", i + 2)
if end != -1:
- out.append(" ")
- out.append(_emit_island(text[i + 2 : end], math_speak=math_speak))
- out.append(" ")
+ append_island(text[i + 2 : end])
i = end + 2
continue
begin = _BEGIN_ENV.match(text, i)
@@ -784,9 +794,7 @@ def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:
end_tag = f"\\end{{{env}}}"
end = text.find(end_tag, begin.end())
if end != -1:
- out.append(" ")
- out.append(_emit_island(text[begin.end() : end], math_speak=math_speak))
- out.append(" ")
+ append_island(text[begin.end() : end])
i = end + len(end_tag)
continue
if text[i] == "$" and not _is_escaped(text, i):
@@ -799,14 +807,13 @@ def verbalize_latex_for_speech(text: str, *, math_speak: bool = True) -> str:
break
j += 1
if j > i:
- out.append(" ")
- out.append(_emit_island(text[i + 1 : j], math_speak=math_speak))
- out.append(" ")
+ append_island(text[i + 1 : j])
i = j + 1
continue
- out.append(text[i])
+ prose.append(text[i])
i += 1
+ flush_prose()
spoken = "".join(out)
if math_speak:
spoken = _verbalize_loose_commands(spoken)
diff --git a/tests/services/test_voice.py b/tests/services/test_voice.py
index 3196e5a0f..8211509ca 100644
--- a/tests/services/test_voice.py
+++ b/tests/services/test_voice.py
@@ -732,3 +732,21 @@ async def test_transcribe_audio_facade(monkeypatch: pytest.MonkeyPatch) -> None:
)
assert text == "transcribed"
assert captured["json"]["input_audio"]["format"] == "webm"
+
+
+ "formula",
+ ["$x*y*z$", "$$x*y*z$$", r"\(x*y*z\)", r"\[x*y*z\]", r"\begin{align}x*y*z\end{align}"],
+)
+def test_math_speak_off_preserves_products_while_cleaning_prose(formula):
+ assert (
+ strip_markdown_for_speech(f"**Multiply** {formula} and *compare*.", math_speak=False)
+ == "Multiply x*y*z and compare."
+ )
+
+
+def test_math_speak_off_keeps_tex_scripts_separate_from_prose_emphasis():
+ assert (
+ strip_markdown_for_speech(r"*Use* $x_i*y_j$ in file_name.", math_speak=False)
+ == "Use x_i*y_j in file_name."
+ )
diff --git a/web/features/chat/messages/ChatMessageList.tsx b/web/features/chat/messages/ChatMessageList.tsx
index 0c33ef41e..3d2b20cfb 100644
--- a/web/features/chat/messages/ChatMessageList.tsx
+++ b/web/features/chat/messages/ChatMessageList.tsx
@@ -1294,7 +1294,7 @@ export function CopyActionButton({
// Speaker button: synthesizes this one reply and plays it. Auto-play of later
// replies is a Settings preference (`autoPlayFresh`), not a first-click prompt.
-let stopActivePlayback: (() => void) | null = null;
+let activePlayback: { owner: object; stop: () => void } | null = null;
async function ttsErrorMessage(resp: Response): Promise<string> {
try {
@@ -1327,6 +1327,7 @@ export function PlayAudioButton({
const abortRef = useRef<AbortController | null>(null);
const genRef = useRef(0);
const autoPlayedRef = useRef(false);
+ const playbackOwnerRef = useRef<object>({});
const cleanup = useCallback(() => {
abortRef.current?.abort();
@@ -1345,16 +1346,16 @@ export function PlayAudioButton({
genRef.current += 1;
cleanup();
setState("idle");
- if (stopActivePlayback === stop) stopActivePlayback = null;
+ if (activePlayback?.owner === playbackOwnerRef.current) activePlayback = null;
}, [cleanup]);
const play = useCallback(async () => {
- stopActivePlayback?.();
+ activePlayback?.stop();
const gen = ++genRef.current;
abortRef.current?.abort();
const ac = new AbortController();
abortRef.current = ac;
- stopActivePlayback = stop;
+ activePlayback = { owner: playbackOwnerRef.current, stop };
setState("loading");
try {
const resp = await apiFetch(apiUrl("/api/voice/tts"), {
@@ -1387,7 +1388,7 @@ export function PlayAudioButton({
if (gen !== genRef.current) return;
setState("idle");
cleanup();
- if (stopActivePlayback === stop) stopActivePlayback = null;
+ if (activePlayback?.owner === playbackOwnerRef.current) activePlayback = null;
};
audio.onerror = () => {
if (gen !== genRef.current) return;
@@ -1430,7 +1431,11 @@ export function PlayAudioButton({
return () => window.clearTimeout(id);
}, [autoPlayFresh, autoplayEnabled, content, play]);
- useEffect(() => cleanup, [cleanup]);
+ useEffect(() => () => {
+ genRef.current += 1;
+ cleanup();
+ if (activePlayback?.owner === playbackOwnerRef.current) activePlayback = null;
+ }, [cleanup]);
return (
<div className="relative inline-flex">
diff --git a/web/tests/chat-read-aloud-playback.spec.tsx b/web/tests/chat-read-aloud-playback.spec.tsx
new file mode 100644
index 000000000..576ed20cc
--- /dev/null
+++ b/web/tests/chat-read-aloud-playback.spec.tsx
@@ -0,0 +1,57 @@
+import { act, render, screen } from "@testing-library/react";
+import userEvent from "@testing-library/user-event";
+import { afterEach, expect, it, vi } from "vitest";
+import { PlayAudioButton } from "@/features/chat/messages/ChatMessageList";
+import { apiFetch } from "@/lib/api";
+import { initI18n } from "@/i18n/init";
+
+vi.mock("@/lib/api", async (original) => ({
+ ...(await original<typeof import("@/lib/api")>()),
+ apiFetch: vi.fn(),
+}));
+vi.mock("@/hooks/useVoiceAutoplay", () => ({
+ useVoiceAutoplay: () => ({ autoplayEnabled: false }),
+}));
+initI18n("en");
+const instances: FakeAudio[] = [];
+class FakeAudio {
+ pause = vi.fn();
+ play = vi.fn().mockResolvedValue(undefined);
+ onended: (() => void) | null = null;
+ onerror: (() => void) | null = null;
+ constructor() { instances.push(this); }
+}
+function installAudio() {
+ vi.stubGlobal("Audio", FakeAudio);
+ Object.defineProperty(URL, "createObjectURL", { configurable: true, value: vi.fn(() => "blob:test") });
+ Object.defineProperty(URL, "revokeObjectURL", { configurable: true, value: vi.fn() });
+}
+afterEach(() => { vi.unstubAllGlobals(); instances.length = 0; });
+
+it("switches between reply buttons without overlapping speech or a first-click prompt", async () => {
+ installAudio();
+ vi.mocked(apiFetch).mockResolvedValue({ ok: true, blob: async () => new Blob(["audio"]) } as Response);
+ render(<><PlayAudioButton content="First" autoPlayFresh={false} /><PlayAudioButton content="Second" autoPlayFresh={false} /></>);
+ const user = userEvent.setup();
+ await user.click(screen.getAllByRole("button", { name: "Play aloud" })[0]);
+ expect(await screen.findByRole("button", { name: "Stop" })).toBeVisible();
+ await user.click(screen.getByRole("button", { name: "Play aloud" }));
+ expect(instances).toHaveLength(2);
+ expect(instances[0].pause).toHaveBeenCalled();
+ expect(instances[1].pause).not.toHaveBeenCalled();
+ expect(screen.getAllByRole("button", { name: "Stop" })).toHaveLength(1);
+ expect(screen.queryByText(/Automatically read/)).toBeNull();
+});
+
+it("does not play a late response body after leaving the chat", async () => {
+ installAudio();
+ let resolve!: (blob: Blob) => void;
+ const body = new Promise<Blob>((yes) => { resolve = yes; });
+ vi.mocked(apiFetch).mockResolvedValueOnce({ ok: true, blob: () => body } as Response);
+ const { unmount } = render(<PlayAudioButton content="First" autoPlayFresh={false} />);
+ await userEvent.setup().click(screen.getByRole("button", { name: "Play aloud" }));
+ unmount();
+ await act(async () => { resolve(new Blob(["audio"])); });
+ expect(instances).toHaveLength(0);
+ expect(URL.createObjectURL).not.toHaveBeenCalled();
+});