Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
/**
* Manuscript Read-Aloud — full-prose TTS proofing (#1304).
*
* Reading prose aloud surfaces clunky rhythm, tongue-twisters, repeated words,
* and unnatural sentences that are invisible to the eye. This modal narrates the
* active manuscript section with the local TTS engines (Kokoro/Piper, same path
* as storyboard dialogue), highlighting each sentence karaoke-style as it plays.
*
* Sync: the server splits the prose into sentence segments and returns each
* one's audio filename + measured duration. We play the segments back-to-back
* via a single <audio> element and highlight the active sentence — no word-level
* timestamps needed. Each segment also carries a "hard to say" readability scan;
* flagged sentences get a wavy underline + tooltip so trouble-spots are visible
* before/while listening.
*
* Non-destructive: narration never mutates the manuscript — it only reads the
* section's current text.
*/
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { Play, Pause, Square, Loader2, Volume2, AlertTriangle } from 'lucide-react';
import Modal from '../../ui/Modal';
import VoicePicker from '../../voice/VoicePicker';
import toast from '../../ui/Toast';
import ProgressBar from '../../ui/ProgressBar';
import { formatDurationMs } from '../../../utils/formatters';
import SpokenCheckChip, { countLinesNeedingListen } from '../SpokenCheckChip';
import { acceptNarrationSegmentSpeech, narratePipelineProse, rerenderNarrationSegment } from '../../../services/api';
import { STAGE_LABEL } from './constants';
import { clickableProps, onActivateKeyDown } from '../../../lib/a11yKeyboard';
import { safeReadStorage, safeWriteStorage } from '../../../lib/safeStorage';
import useMounted from '../../../hooks/useMounted';
const NARRATOR_VOICE_KEY = 'portos.manuscript.narratorVoice';
const initialVoice = () => safeReadStorage(NARRATOR_VOICE_KEY) || '';
export default function ManuscriptReadAloud({ open, onClose, section }) {
const content = section?.content || '';
const [voiceId, setVoiceId] = useState(initialVoice);
const [segments, setSegments] = useState(null); // null = not synthesized yet
const [loading, setLoading] = useState(false);
// -1 = idle/no highlight; otherwise the segment currently cued.
const [currentIndex, setCurrentIndex] = useState(-1);
const [isPlaying, setIsPlaying] = useState(false);
const [elapsedMs, setElapsedMs] = useState(0);
// Segment index being re-rendered / re-scored (one at a time), or null.
const [busyIndex, setBusyIndex] = useState(null);
const audioRef = useRef(null);
// Gates the post-synthesis state writes so a narration that resolves after the
// modal is torn down doesn't apply audio/offsets into an unmounted tree.
const mountedRef = useMounted();
// Live mirror of the prose the modal is showing, so an in-flight narration
// response can detect that the section's text changed (edit, reformat, accepted
// fix, or a section swap) while it was synthesizing and discard itself rather
// than apply stale audio/offsets to different prose.
const contentRef = useRef(content);
contentRef.current = content;
// Live mirror of `open` so a narration that resolves after the modal was
// closed doesn't auto-start playback into an unmounted <audio> element.
const openRef = useRef(open);
openRef.current = open;
// Reset narration when the prose being read changes — a different issue/stage,
// or the SAME section edited under it. The cached segments + their char offsets
// belong to the exact text they were synthesized from; reusing them against
// changed prose would misalign the highlight and play the wrong sentence.
const resetNarration = () => {
setSegments(null);
setCurrentIndex(-1);
setIsPlaying(false);
setElapsedMs(0);
};
useEffect(() => { resetNarration(); }, [section?.issueId, section?.stageId, content]);
useEffect(() => { if (!open) { setIsPlaying(false); } }, [open]);
// Cumulative start offset (ms) per segment + total, for the progress bar.
const { cumulative, totalMs } = useMemo(() => {
const cum = [];
let acc = 0;
(segments || []).forEach((s) => { cum.push(acc); acc += s.durationMs || 0; });
return { cumulative: cum, totalMs: acc };
}, [segments]);
const listenCount = useMemo(() => countLinesNeedingListen(segments), [segments]);
const mismatchedSegments = useMemo(
() => (segments || []).filter((s) => s.verification?.status === 'mismatch'),
[segments],
);
const hardCount = useMemo(
() => (segments || []).filter((s) => s.readability?.hard).length,
[segments],
);
// Drive the <audio> element from (currentIndex, isPlaying). The src is bound
// in JSX from currentIndex, so changing index reloads + plays from the start
// of that segment; toggling isPlaying without an index change pauses/resumes.
useEffect(() => {
const audio = audioRef.current;
if (!audio) return;
if (currentIndex < 0) { audio.pause(); return; }
if (isPlaying) {
// play() rejects when the browser's autoplay policy blocks it — e.g. the
// user-activation window lapsed while a long section synthesized. Fall
// back to paused so the UI shows "Play" (a click is a fresh gesture the
// browser will honor) instead of a "Pause" button over silence.
audio.play().catch(() => { if (mountedRef.current) setIsPlaying(false); });
} else {
audio.pause();
}
}, [currentIndex, isPlaying]);
const runNarration = async () => {
if (!content.trim()) {
toast('There is no drafted prose to read aloud');
return;
}
setLoading(true);
const reqContent = content; // snapshot — discard if the prose changes mid-flight
const result = await narratePipelineProse(content, voiceId || undefined, { silent: true })
.catch((err) => {
toast.error(err.message || 'Failed to narrate');
return null;
});
if (!mountedRef.current) return;
setLoading(false);
// Drop the response if the section's text changed while it synthesized — its
// offsets belong to `reqContent`, not the current prose.
if (contentRef.current !== reqContent || !result) return;
const segs = Array.isArray(result.segments) ? result.segments : [];
setSegments(segs);
setElapsedMs(0);
// Only auto-start when the modal is still open — otherwise the <audio> is
// unmounted and the play effect would set "Pause" with nothing playing. The
// segments still load, so reopening shows the prose ready to Play.
if (segs.length && openRef.current) { setCurrentIndex(0); setIsPlaying(true); }
};
// Merge a per-segment update into the cached segments by index.
const patchSegment = (index, patch) => {
setSegments((prev) => (prev ? prev.map((seg) => (seg.index === index ? { ...seg, ...patch } : seg)) : prev));
};
// Re-synthesize one sentence (honoring any accepted spelling) and re-check it.
const rerenderSegment = async (seg) => {
setBusyIndex(seg.index);
const reqContent = content;
const result = await rerenderNarrationSegment(
reqContent.slice(seg.start, seg.end), voiceId || undefined, seg.expectedSpeech, { silent: true },
).catch((err) => {
toast.error(err.message || 'Failed to re-render sentence');
return null;
});
if (!mountedRef.current) return;
setBusyIndex(null);
if (!result || contentRef.current !== reqContent) return;
patchSegment(seg.index, {
filename: result.filename,
durationMs: result.durationMs,
verification: result.verification,
});
};
// "Accept as spoken": what speech-to-text heard becomes the expected read.
const acceptSegment = async (seg) => {
const heard = seg.verification?.heard;
if (!heard) return;
setBusyIndex(seg.index);
const reqContent = content;
const result = await acceptNarrationSegmentSpeech(
reqContent.slice(seg.start, seg.end), heard, heard, { silent: true },
).catch((err) => {
toast.error(err.message || 'Failed to accept as spoken');
return null;
});
if (!mountedRef.current) return;
setBusyIndex(null);
if (!result || contentRef.current !== reqContent) return;
patchSegment(seg.index, { expectedSpeech: result.expectedSpeech, verification: result.verification });
};
const togglePlay = () => {
if (!segments?.length) return;
if (currentIndex < 0) { setCurrentIndex(0); setIsPlaying(true); return; }
setIsPlaying((p) => !p);
};
const stop = () => {
setIsPlaying(false);
setCurrentIndex(-1);
setElapsedMs(0);
const audio = audioRef.current;
if (audio) audio.currentTime = 0;
};
// Memoized on currentIndex (its only non-stable closure) so `renderedProse`
// — which wires it onto every sentence span — stays cached between renders.
const jumpTo = useCallback((idx) => {
// Re-clicking the already-current sentence keeps both state values
// unchanged, so the play effect won't re-fire and the <audio> src won't
// reload — restart it imperatively so "play that line again" works.
const audio = audioRef.current;
if (idx === currentIndex && audio) {
audio.currentTime = 0;
audio.play().catch(() => {});
setIsPlaying(true);
return;
}
setCurrentIndex(idx);
setIsPlaying(true);
}, [currentIndex]);
const onEnded = () => {
if (currentIndex < (segments?.length || 0) - 1) {
setCurrentIndex(currentIndex + 1);
} else {
setIsPlaying(false);
setElapsedMs(totalMs);
}
};
const onTimeUpdate = () => {
const audio = audioRef.current;
if (!audio || currentIndex < 0) return;
setElapsedMs((cumulative[currentIndex] || 0) + audio.currentTime * 1000);
};
const changeVoice = (next) => {
setVoiceId(next);
safeWriteStorage(NARRATOR_VOICE_KEY, next || '');
// Cached segments were synthesized with the prior voice — invalidate so the
// next play re-synthesizes with the new narrator.
resetNarration();
};
// Render the prose with each sentence wrapped in a span, preserving the exact
// inter-sentence whitespace from the original text (the gaps between spans).
const renderedProse = useMemo(() => {
if (!segments?.length) return null;
const nodes = [];
let cursor = 0;
segments.forEach((seg) => {
if (seg.start > cursor) {
nodes.push(<span key={`gap-${cursor}`}>{content.slice(cursor, seg.start)}</span>);
}
const active = seg.index === currentIndex;
const reasons = seg.readability?.reasons || [];
const hard = seg.readability?.hard;
const misheard = seg.verification?.status === 'mismatch';
nodes.push(
<span
key={`seg-${seg.index}`}
{...clickableProps(() => jumpTo(seg.index))}
onClick={() => jumpTo(seg.index)}
onKeyDown={onActivateKeyDown(() => jumpTo(seg.index))}
title={misheard ? `Speech-to-text heard: "${seg.verification.heard || ''}"` : (hard ? `Hard to say: ${reasons.join('; ')}` : undefined)}
className={[
'cursor-pointer rounded transition-colors',
active ? 'bg-port-accent/30 text-white' : 'hover:bg-port-border/40',
hard || misheard ? 'underline decoration-wavy decoration-port-warning/70 underline-offset-2' : '',
].join(' ')}
>
{content.slice(seg.start, seg.end)}
</span>,
);
cursor = seg.end;
});
if (cursor < content.length) {
nodes.push(<span key={`gap-${cursor}`}>{content.slice(cursor)}</span>);
}
return nodes;
}, [segments, content, currentIndex, jumpTo]);
const currentSrc = (currentIndex >= 0 && segments?.[currentIndex])
? `/data/audio/${encodeURIComponent(segments[currentIndex].filename)}`
: undefined;
const progressPct = totalMs > 0 ? Math.min(100, (elapsedMs / totalMs) * 100) : 0;
return (
<Modal open={open} onClose={onClose} size="3xl" backdropClassName="bg-black/70 p-4" panelClassName="bg-port-card border border-port-border rounded-lg overflow-hidden flex flex-col">
<div className="flex flex-col min-h-0">
<header className="flex items-center gap-2 px-4 py-3 border-b border-port-border">
<Volume2 className="w-4 h-4 text-port-accent" />
<h2 className="text-sm font-semibold text-white">
Read aloud
{section ? (
<span className="text-gray-500 font-normal">
{' '}— #{section.number} {section.title ? `· ${section.title}` : ''} ({STAGE_LABEL[section.stageId] || section.stageId})
</span>
) : null}
</h2>
<button
type="button"
onClick={onClose}
className="ml-auto text-gray-500 hover:text-white text-sm px-2 min-h-[44px] min-w-[44px] flex items-center justify-center"
aria-label="Close read-aloud"
>
✕
</button>
</header>
<div className="px-4 py-3 border-b border-port-border space-y-2.5">
<div className="flex items-end gap-3 flex-wrap">
<div className="min-w-[220px]">
<VoicePicker
value={voiceId}
onChange={changeVoice}
label="Narrator voice"
placeholder="Default voice"
previewText="The morning fog burned off slow that day."
disabled={loading}
/>
</div>
<div className="flex items-center gap-2">
{!segments ? (
<button
type="button"
onClick={runNarration}
disabled={loading || !content.trim()}
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded text-sm font-medium border bg-port-bg text-port-accent border-port-border hover:border-port-accent/40 disabled:opacity-40"
>
{loading ? <Loader2 size={14} className="animate-spin" /> : <Play size={14} />}
{loading ? 'Synthesizing…' : 'Read aloud'}
</button>
) : (
<>
<button
type="button"
onClick={togglePlay}
className="inline-flex items-center gap-1.5 px-3 py-1.5 rounded text-sm font-medium border bg-port-bg text-port-accent border-port-border hover:border-port-accent/40"
>
{isPlaying ? <Pause size={14} /> : <Play size={14} />}
{isPlaying ? 'Pause' : 'Play'}
</button>
<button
type="button"
onClick={stop}
className="inline-flex items-center gap-1.5 px-2.5 py-1.5 rounded text-sm font-medium border bg-port-bg text-gray-400 border-port-border hover:text-white"
>
<Square size={13} /> Stop
</button>
</>
)}
</div>
</div>
{segments ? (
<div className="space-y-1">
<ProgressBar percent={progressPct} label="Narration progress" duration={150} />
<div className="flex items-center justify-between text-[11px] text-gray-500">
<span>
{segments.length} sentence{segments.length === 1 ? '' : 's'}
{currentIndex >= 0 ? ` · ${currentIndex + 1} of ${segments.length}` : ''}
</span>
<span>{formatDurationMs(elapsedMs)} / {formatDurationMs(totalMs)}</span>
</div>
{listenCount > 0 ? (
<p className="flex items-center gap-1.5 text-[11px] text-port-warning">
<AlertTriangle size={11} />
{listenCount} of {segments.length} sentence{segments.length === 1 ? '' : 's'} need a listen — speech-to-text heard something different from the text.
</p>
) : null}
{mismatchedSegments.length > 0 ? (
<ul className="max-h-32 overflow-y-auto space-y-1" aria-label="Sentences that need a listen">
{mismatchedSegments.map((seg) => (
<li key={seg.index} className="text-[11px] text-gray-400 flex items-center gap-2 flex-wrap">
<button
type="button"
onClick={() => jumpTo(seg.index)}
className="truncate max-w-[16rem] text-left underline decoration-dotted hover:text-white"
title={content.slice(seg.start, seg.end)}
>
{content.slice(seg.start, seg.end)}
</button>
<SpokenCheckChip
verification={seg.verification}
busy={busyIndex !== null}
onRerender={() => rerenderSegment(seg)}
onAccept={() => acceptSegment(seg)}
/>
</li>
))}
</ul>
) : null}
{hardCount > 0 ? (
<p className="flex items-center gap-1.5 text-[11px] text-port-warning">
<AlertTriangle size={11} />
{hardCount} possible trouble spot{hardCount === 1 ? '' : 's'} — underlined below; hover for why.
</p>
) : null}
</div>
) : null}
</div>
<div className="flex-1 overflow-y-auto px-5 py-4">
{segments ? (
<div className="whitespace-pre-wrap leading-relaxed text-[15px] text-gray-200">
{renderedProse}
</div>
) : (
<div className="whitespace-pre-wrap leading-relaxed text-[15px] text-gray-400">
{content.trim() ? content : <span className="italic text-gray-600">No drafted prose to read aloud yet.</span>}
</div>
)}
</div>
</div>
{/* Hidden engine — playback is driven by the controls above. */}
<audio ref={audioRef} src={currentSrc} onEnded={onEnded} onTimeUpdate={onTimeUpdate} className="hidden">
<track kind="captions" />
</audio>
</Modal>
);
}