Skip to content

The open-source AI voice studio. Clone, dictate, create.

feat(stories): m4b/mp3 export with auto-derived chapter markers

liubaofengfc8a6f890d0a
diff --git a/backend/routes/stories.py b/backend/routes/stories.py
index 0c6237a8..e250e913 100644
--- a/backend/routes/stories.py
+++ b/backend/routes/stories.py
@@ -206,29 +206,58 @@ async def set_story_item_version(
return item
+_EXPORT_MIME = {
+ "wav": "audio/wav",
+ "m4b": "audio/mp4",
+ "mp3": "audio/mpeg",
+}
+
+
@router.get("/stories/{story_id}/export-audio")
async def export_story_audio(
story_id: str,
+ format: str = "wav",
+ chapters: str = "none",
db: Session = Depends(get_db),
):
- """Export story as single mixed audio file."""
+ """Export story as a single mixed audio file.
+
+ Query params:
+ format: ``wav`` (default), ``m4b``, or ``mp3``.
+ chapters: ``none`` (default) or ``auto``. ``auto`` emits one chapter
+ per story item, titled from its generation text. WAV ignores this.
+ """
+ fmt = (format or "wav").lower()
+ if fmt not in _EXPORT_MIME:
+ raise HTTPException(status_code=400, detail=f"Unsupported format: {format}")
+ chapters_mode = (chapters or "none").lower()
+ if chapters_mode not in ("none", "auto"):
+ raise HTTPException(status_code=400, detail=f"Unsupported chapters mode: {chapters}")
+
try:
story = db.query(database.Story).filter_by(id=story_id).first()
if not story:
raise HTTPException(status_code=404, detail="Story not found")
- audio_bytes = await stories.export_story_audio(story_id, db)
+ try:
+ audio_bytes = await stories.export_story_audio(
+ story_id, db, fmt=fmt, chapters_mode=chapters_mode
+ )
+ except RuntimeError as e:
+ # Most likely: ffmpeg missing or ffmpeg returned non-zero.
+ raise HTTPException(status_code=503, detail=str(e))
+
if not audio_bytes:
raise HTTPException(status_code=400, detail="Story has no audio items")
safe_name = "".join(c for c in story.name if c.isalnum() or c in (" ", "-", "_")).strip()
if not safe_name:
safe_name = "story"
- filename = f"{safe_name}.wav"
+ filename = f"{safe_name}.{fmt}"
return StreamingResponse(
io.BytesIO(audio_bytes),
- media_type="audio/wav",
+ media_type=_EXPORT_MIME[fmt],
headers={"Content-Disposition": safe_content_disposition("attachment", filename)},
)
except HTTPException:
diff --git a/backend/services/stories.py b/backend/services/stories.py
index b1f0905d..d50b1caf 100644
--- a/backend/services/stories.py
+++ b/backend/services/stories.py
@@ -3,7 +3,11 @@ Story management module.
"""
from typing import List, Optional
+from dataclasses import dataclass
from datetime import UTC, datetime
+import re
+import shutil
+import subprocess
import uuid
import tempfile
from pathlib import Path
@@ -833,16 +837,145 @@ async def set_story_item_version(
return _build_item_detail(item, generation, profile.name if profile else "Unknown", db)
+@dataclass
+class _Chapter:
+ start_ms: int
+ end_ms: int
+ title: str
+
+
+# Latin sentences end with [.!?] + whitespace; CJK sentences end with [。!?]
+# and conventionally have no following whitespace, so the boundary is the mark
+# itself. Both lookbehinds are zero-width — split() just consumes whitespace
+# when present.
+_SENTENCE_BREAK = re.compile(r"(?<=[.!?])\s+|(?<=[。!?])")
+
+
+def _chapter_title_from_text(text: Optional[str], max_len: int = 80) -> str:
+ """Derive a chapter title from the linked generation's text.
+
+ Takes the first sentence (or the leading slice if the sentence is long).
+ """
+ cleaned = (text or "").strip()
+ if not cleaned:
+ return "Chapter"
+ first = _SENTENCE_BREAK.split(cleaned, maxsplit=1)[0].strip()
+ if not first:
+ return "Chapter"
+ if len(first) > max_len:
+ return first[:max_len].rstrip() + "…"
+ return first
+
+
+def _derive_chapters_auto(
+ segments: List[dict],
+ total_duration_ms: int,
+) -> List[_Chapter]:
+ """One chapter per story segment, ordered by start_time_ms.
+
+ Each segment dict must carry ``start_time_ms`` and ``text``. The final
+ chapter runs to ``total_duration_ms``. Segments with identical
+ ``start_time_ms`` are deduped (only the first contributes a chapter
+ boundary) so multi-track items don't produce zero-length chapters.
+ """
+ ordered = sorted(segments, key=lambda s: s["start_time_ms"])
+ chapters: List[_Chapter] = []
+ for seg in ordered:
+ start = int(seg["start_time_ms"])
+ if chapters and chapters[-1].start_ms == start:
+ continue
+ chapters.append(
+ _Chapter(start_ms=start, end_ms=0, title=_chapter_title_from_text(seg.get("text")))
+ )
+ for i, ch in enumerate(chapters):
+ ch.end_ms = chapters[i + 1].start_ms if i + 1 < len(chapters) else total_duration_ms
+ # Skip degenerate trailing chapter that would have zero duration.
+ return [ch for ch in chapters if ch.end_ms > ch.start_ms]
+
+
+def _escape_ffmetadata(value: str) -> str:
+ # Per https://ffmpeg.org/ffmpeg-formats.html#Metadata-1 — escape =, ;, #, \, newline.
+ return (
+ value.replace("\\", "\\\\")
+ .replace("=", "\\=")
+ .replace(";", "\\;")
+ .replace("#", "\\#")
+ .replace("\n", "\\n")
+ )
+
+
+def _write_ffmetadata(chapters: List[_Chapter], path: Path) -> None:
+ lines = [";FFMETADATA1"]
+ for ch in chapters:
+ lines.extend(
+ [
+ "[CHAPTER]",
+ "TIMEBASE=1/1000",
+ f"START={ch.start_ms}",
+ f"END={ch.end_ms}",
+ f"title={_escape_ffmetadata(ch.title)}",
+ ]
+ )
+ path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+
+
+def _ffmpeg_encode(
+ wav_path: Path,
+ out_path: Path,
+ fmt: str,
+ chapters: Optional[List[_Chapter]],
+) -> None:
+ """Transcode WAV → fmt with optional embedded chapter metadata.
+
+ Raises RuntimeError on missing ffmpeg or non-zero exit.
+ """
+ if shutil.which("ffmpeg") is None:
+ raise RuntimeError(
+ "ffmpeg is required for m4b/mp3 story export — install it or request format=wav"
+ )
+
+ cmd: List[str] = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", "-i", str(wav_path)]
+ meta_path: Optional[Path] = None
+ try:
+ if chapters:
+ meta_path = wav_path.with_suffix(".chapters.txt")
+ _write_ffmetadata(chapters, meta_path)
+ cmd.extend(["-i", str(meta_path), "-map_metadata", "1", "-map_chapters", "1"])
+
+ if fmt == "m4b":
+ # The 'ipod' muxer is ffmpeg's name for the .m4b container.
+ cmd.extend(["-map", "0:a", "-c:a", "aac", "-b:a", "128k", "-f", "ipod"])
+ elif fmt == "mp3":
+ cmd.extend(["-map", "0:a", "-c:a", "libmp3lame", "-b:a", "192k", "-f", "mp3"])
+ else:
+ raise ValueError(f"Unsupported export format: {fmt}")
+
+ cmd.append(str(out_path))
+ result = subprocess.run(cmd, capture_output=True, check=False)
+ if result.returncode != 0:
+ stderr = result.stderr.decode("utf-8", errors="replace").strip()
+ raise RuntimeError(f"ffmpeg exited {result.returncode}: {stderr}")
+ finally:
+ if meta_path is not None:
+ meta_path.unlink(missing_ok=True)
+
+
async def export_story_audio(
story_id: str,
db: Session,
+ fmt: str = "wav",
+ chapters_mode: str = "none",
) -> Optional[bytes]:
"""
- Export story as single mixed audio file with timecode-based mixing.
+ Export story as a single mixed audio file with timecode-based mixing.
Args:
story_id: Story ID
db: Database session
+ fmt: Output container — "wav" (default), "m4b", or "mp3".
+ chapters_mode: "none" (default) leaves chapter metadata off; "auto"
+ derives one chapter per story item, titled from its generation
+ text. WAV ignores this — chapters are an m4b/mp3 feature.
Returns:
Audio file bytes or None if story not found
@@ -918,6 +1051,7 @@ async def export_story_audio(
"audio": trimmed_audio,
"start_time_ms": start_time_ms,
"duration_ms": effective_duration_ms,
+ "text": generation.text,
}
)
except Exception:
@@ -961,18 +1095,26 @@ async def export_story_audio(
if max_val > 1.0:
final_audio = final_audio / max_val
- # Save to temporary file
- with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
- tmp_path = tmp.name
+ fmt = (fmt or "wav").lower()
+ if fmt not in ("wav", "m4b", "mp3"):
+ raise ValueError(f"Unsupported export format: {fmt}")
- try:
- save_audio(final_audio, tmp_path, sample_rate)
-
- # Read file bytes
- with open(tmp_path, "rb") as f:
- audio_bytes = f.read()
+ chapters: Optional[List[_Chapter]] = None
+ if fmt != "wav" and chapters_mode == "auto":
+ chapters = _derive_chapters_auto(audio_data, max_end_time_ms) or None
- return audio_bytes
+ wav_path = Path(tempfile.NamedTemporaryFile(suffix=".wav", delete=False).name)
+ out_path: Optional[Path] = None
+ try:
+ save_audio(final_audio, str(wav_path), sample_rate)
+ if fmt == "wav":
+ return wav_path.read_bytes()
+
+ out_suffix = ".m4b" if fmt == "m4b" else ".mp3"
+ out_path = Path(tempfile.NamedTemporaryFile(suffix=out_suffix, delete=False).name)
+ _ffmpeg_encode(wav_path, out_path, fmt, chapters)
+ return out_path.read_bytes()
finally:
- # Clean up temp file
- Path(tmp_path).unlink(missing_ok=True)
+ wav_path.unlink(missing_ok=True)
+ if out_path is not None:
+ out_path.unlink(missing_ok=True)
diff --git a/backend/tests/test_story_export.py b/backend/tests/test_story_export.py
new file mode 100644
index 00000000..8cc11f9b
--- /dev/null
+++ b/backend/tests/test_story_export.py
@@ -0,0 +1,166 @@
+"""Unit tests for story-export helpers — chapter derivation + FFMETADATA1 emission.
+
+These tests cover the pure-Python pieces of the m4b/mp3 export path that don't
+need a database session. The ffmpeg-driven encode path is exercised at the end
+under a shutil.which gate so the suite stays green on CI runners without ffmpeg.
+"""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from pathlib import Path
+
+import pytest
+
+from backend.services.stories import (
+ _Chapter,
+ _chapter_title_from_text,
+ _derive_chapters_auto,
+ _escape_ffmetadata,
+ _ffmpeg_encode,
+ _write_ffmetadata,
+)
+
+
+class TestChapterTitleFromText:
+ def test_returns_first_sentence(self):
+ assert _chapter_title_from_text("Hello world. Second sentence.") == "Hello world."
+
+ def test_truncates_long_sentence(self):
+ long = "a" * 200
+ title = _chapter_title_from_text(long, max_len=80)
+ assert title.endswith("…")
+ assert len(title) <= 81 # 80 chars + ellipsis
+
+ def test_handles_cjk_punctuation(self):
+ assert _chapter_title_from_text("第一章。第二章。") == "第一章。"
+
+ def test_empty_and_whitespace_fall_back(self):
+ assert _chapter_title_from_text("") == "Chapter"
+ assert _chapter_title_from_text(" ") == "Chapter"
+ assert _chapter_title_from_text(None) == "Chapter"
+
+
+class TestDeriveChaptersAuto:
+ def test_orders_by_start_time(self):
+ segs = [
+ {"start_time_ms": 5000, "text": "Two."},
+ {"start_time_ms": 0, "text": "One."},
+ ]
+ chapters = _derive_chapters_auto(segs, total_duration_ms=10000)
+ assert [c.start_ms for c in chapters] == [0, 5000]
+ assert chapters[0].title == "One."
+ assert chapters[1].end_ms == 10000
+
+ def test_fills_end_from_next_chapter(self):
+ segs = [
+ {"start_time_ms": 0, "text": "A."},
+ {"start_time_ms": 3000, "text": "B."},
+ {"start_time_ms": 7000, "text": "C."},
+ ]
+ chapters = _derive_chapters_auto(segs, total_duration_ms=10000)
+ assert chapters[0].end_ms == 3000
+ assert chapters[1].end_ms == 7000
+ assert chapters[2].end_ms == 10000
+
+ def test_dedupes_same_start_time(self):
+ # Two items at the same timecode (multi-track) → one chapter.
+ segs = [
+ {"start_time_ms": 0, "text": "Narrator."},
+ {"start_time_ms": 0, "text": "Music bed."},
+ {"start_time_ms": 4000, "text": "Next beat."},
+ ]
+ chapters = _derive_chapters_auto(segs, total_duration_ms=8000)
+ assert [c.start_ms for c in chapters] == [0, 4000]
+
+ def test_drops_zero_duration_trailing_chapter(self):
+ # An item starting at exactly the total duration would otherwise
+ # produce an end_ms == start_ms chapter, which ffmpeg rejects.
+ segs = [
+ {"start_time_ms": 0, "text": "Body."},
+ {"start_time_ms": 10000, "text": "Tail."},
+ ]
+ chapters = _derive_chapters_auto(segs, total_duration_ms=10000)
+ assert len(chapters) == 1
+
+
+class TestFFMetadataEscaping:
+ @pytest.mark.parametrize(
+ "raw,escaped",
+ [
+ ("simple", "simple"),
+ ("a=b", "a\\=b"),
+ ("a;b", "a\\;b"),
+ ("#hash", "\\#hash"),
+ ("back\\slash", "back\\\\slash"),
+ ("line1\nline2", "line1\\nline2"),
+ ],
+ )
+ def test_escapes_all_specials(self, raw, escaped):
+ assert _escape_ffmetadata(raw) == escaped
+
+
+class TestWriteFFMetadata:
+ def test_emits_valid_chapter_block(self, tmp_path: Path):
+ chapters = [
+ _Chapter(start_ms=0, end_ms=3000, title="Intro"),
+ _Chapter(start_ms=3000, end_ms=10000, title="Body; with = chars"),
+ ]
+ out = tmp_path / "meta.txt"
+ _write_ffmetadata(chapters, out)
+ content = out.read_text(encoding="utf-8")
+ assert content.startswith(";FFMETADATA1")
+ assert content.count("[CHAPTER]") == 2
+ assert "START=0\nEND=3000" in content
+ assert "title=Intro" in content
+ # Special chars must be escaped in the metadata value.
+ assert "Body\\; with \\= chars" in content
+
+
[email protected](shutil.which("ffmpeg") is None, reason="ffmpeg not installed")
+class TestFFmpegEncodeIntegration:
+ """End-to-end: synthesize a WAV, encode to M4B with chapters, read back with ffprobe."""
+
+ def _make_silent_wav(self, path: Path, seconds: float = 12.0, sample_rate: int = 24000) -> None:
+ import numpy as np # imported lazily — the rest of the suite shouldn't depend on numpy
+
+ from backend.utils.audio import save_audio
+
+ silence = np.zeros(int(seconds * sample_rate), dtype=np.float32)
+ save_audio(silence, str(path), sample_rate)
+
+ def test_m4b_round_trip_carries_chapters(self, tmp_path: Path):
+ if shutil.which("ffprobe") is None:
+ pytest.skip("ffprobe not installed")
+
+ wav_path = tmp_path / "in.wav"
+ out_path = tmp_path / "out.m4b"
+ self._make_silent_wav(wav_path)
+
+ chapters = [
+ _Chapter(start_ms=0, end_ms=4000, title="Opening"),
+ _Chapter(start_ms=4000, end_ms=12000, title="Closing"),
+ ]
+ _ffmpeg_encode(wav_path, out_path, fmt="m4b", chapters=chapters)
+ assert out_path.exists() and out_path.stat().st_size > 0
+
+ probe = subprocess.run(
+ [
+ "ffprobe",
+ "-v",
+ "error",
+ "-show_chapters",
+ "-of",
+ "default=noprint_wrappers=1",
+ str(out_path),
+ ],
+ capture_output=True,
+ check=True,
+ text=True,
+ )
+ chunks = probe.stdout.split("[CHAPTER]")
+ # ffprobe prints one [CHAPTER] block per chapter; expect two.
+ assert sum(1 for c in chunks if c.strip()) == 2
+ assert "Opening" in probe.stdout
+ assert "Closing" in probe.stdout