Skip to content

The animation engine for agents: blazingly fast math and 3D videos with Manim's Python API, on a Rust GPU core.

Python

Validate audio intervals and preserve speech source identity

Sina Atalay42f3743769fb
diff --git a/rust/engine/src/audio.rs b/rust/engine/src/audio.rs
index 359db90e..9680800e 100644
--- a/rust/engine/src/audio.rs
+++ b/rust/engine/src/audio.rs
@@ -44,18 +44,30 @@ fn kernel(cutoff: f64) -> Vec<f32> {
/// `input` (interleaved, `channels` channels) at `from` samples a second, resampled to `to`:
/// each output sample the band-limited interpolation of the input at its instant.
-pub fn resample(input: &[f32], channels: usize, from: f64, to: f64) -> Vec<f32> {
+pub fn resample(input: &[f32], channels: usize, from: f64, to: f64) -> Result<Vec<f32>, String> {
+ if channels == 0 || input.len() % channels != 0 {
+ return Err("audio samples must contain complete frames with a positive channel count".into());
+ }
+ if !from.is_finite() || !to.is_finite() || from <= 0.0 || to <= 0.0 {
+ return Err("audio sample rates must be finite and positive".into());
+ }
let frames = input.len() / channels;
if from == to || frames == 0 {
- return input.to_vec();
+ return Ok(input.to_vec());
}
let step = from / to; // input samples per output sample
+ if !step.is_finite() || step <= 0.0 {
+ return Err("audio sample-rate ratio is not representable".into());
+ }
let cutoff = 0.95 * (1.0 / step).min(1.0); // below the lower Nyquist
let table = kernel(cutoff);
- let reach = (ZEROS as f64 / cutoff).ceil() as isize; // input samples each side
+ let reach = (ZEROS as f64 / cutoff).ceil().min(frames as f64) as isize; // input samples each side
let count = ((frames as f64) / step).round() as usize;
- let mut out = vec![0.0f32; count * channels];
- let mut weights = Vec::with_capacity(2 * reach as usize + 1);
+ let length = count.checked_mul(channels).filter(|&n| n <= isize::MAX as usize / size_of::<f32>()).ok_or("resampled audio is too large")?;
+ let mut out = Vec::new();
+ out.try_reserve_exact(length).map_err(|_| "resampled audio is too large")?;
+ out.resize(length, 0.0f32);
+ let mut weights = Vec::with_capacity(frames.min(2 * reach as usize + 1));
for n in 0..count {
let x = n as f64 * step;
let center = x.floor() as isize;
@@ -81,5 +93,5 @@ pub fn resample(input: &[f32], channels: usize, from: f64, to: f64) -> Vec<f32>
}
}
}
- out
+ Ok(out)
}
diff --git a/rust/engine/src/speaker.rs b/rust/engine/src/speaker.rs
index 634a58d7..c2f11407 100644
--- a/rust/engine/src/speaker.rs
+++ b/rust/engine/src/speaker.rs
@@ -184,7 +184,7 @@ mod native {
let (load, playing, rate, out) = (track.loads, self.track.clone(), self.rate, self.channels);
std::thread::spawn(move || {
let Some((samples, from, channels)) = super::wav(&file) else { return };
- let resampled = crate::audio::resample(&samples, channels, f64::from(from), f64::from(rate));
+ let Ok(resampled) = crate::audio::resample(&samples, channels, f64::from(from), f64::from(rate)) else { return };
// its channels onto the device's: one to all, or each to its own (wrapping)
let frames = resampled.len() / channels;
let samples = (0..frames * out).map(|k| resampled[k / out * channels + k % out % channels]).collect();
@@ -341,6 +341,7 @@ mod web {
fn stop(&mut self) {
if let Some(playing) = self.playing.take() {
let _ = web_sys::AudioScheduledSourceNode::stop(&playing.source);
+ let _ = playing.source.disconnect();
}
}
@@ -381,9 +382,91 @@ mod web {
}
}
}
+
+ impl Drop for Speaker {
+ fn drop(&mut self) {
+ self.load(None); // stop playback, invalidate pending decodes, release the buffer
+ let _ = self.gain.disconnect();
+ }
+ }
+
+ #[cfg(test)]
+ mod tests {
+ use super::*;
+ use wasm_bindgen_test::*;
+
+ wasm_bindgen_test_configure!(run_in_browser);
+
+ // Observe calls on real browser audio nodes; their actual methods still execute.
+ #[wasm_bindgen(inline_js = r#"
+export function observeAudio(context, gain, source) {
+ const counts = [0, 0, 0];
+ const stop = source.stop, disconnect = source.disconnect, detach = gain.disconnect;
+ const decode = context.decodeAudioData;
+ let decoded;
+ source.stop = function (...args) { counts[0]++; return stop.apply(this, args); };
+ source.disconnect = function (...args) { counts[1]++; return disconnect.apply(this, args); };
+ gain.disconnect = function (...args) { counts[2]++; return detach.apply(this, args); };
+ context.decodeAudioData = function (bytes) { decoded = decode.call(this, bytes); return decoded; };
+ return { counts, decoded: () => decoded, restore: () => { context.decodeAudioData = decode; } };
+}
+export function audioDecoded(probe) { return probe.decoded(); }
+export function finishAudioProbe(probe) { probe.restore(); return probe.counts; }
+"#)]
+ extern "C" {
+ #[wasm_bindgen(js_name = observeAudio)]
+ fn observe_audio(context: &AudioContext, gain: &GainNode, source: &AudioBufferSourceNode) -> JsValue;
+ #[wasm_bindgen(js_name = audioDecoded)]
+ fn audio_decoded(probe: &JsValue) -> js_sys::Promise;
+ #[wasm_bindgen(js_name = finishAudioProbe)]
+ fn finish_audio_probe(probe: &JsValue) -> js_sys::Array;
+ }
+
+ #[wasm_bindgen_test]
+ async fn destruction_stops_owned_audio_and_invalidates_pending_decode() {
+ let mut speaker = Speaker::new().expect("browser AudioContext");
+ let context = speaker.context.clone();
+ let source = context.create_buffer_source().unwrap();
+ let playing_buffer = context.create_buffer(1, 48000, 48000.0).unwrap();
+ source.set_buffer(Some(&playing_buffer));
+ speaker.buffer.replace(Some(playing_buffer));
+ source.connect_with_audio_node(&speaker.gain).unwrap();
+ source.start_with_when(context.current_time() + 60.0).unwrap();
+ let probe = observe_audio(&context, &speaker.gain, &source);
+
+ // A valid PCM file, decoded by the real browser. The promise cannot complete
+ // before this synchronous owner is destroyed and JavaScript gets control back.
+ let mut wav = b"RIFF".to_vec();
+ wav.extend(38u32.to_le_bytes());
+ wav.extend(b"WAVEfmt ");
+ wav.extend(16u32.to_le_bytes());
+ wav.extend([1u16, 1].iter().flat_map(|v| v.to_le_bytes()));
+ wav.extend([48000u32, 96000].iter().flat_map(|v| v.to_le_bytes()));
+ wav.extend([2u16, 16].iter().flat_map(|v| v.to_le_bytes()));
+ wav.extend(b"data");
+ wav.extend(2u32.to_le_bytes());
+ wav.extend([0, 0]);
+ let mut decoding = Speaker::new().unwrap();
+ decoding.load(Some(&wav));
+ speaker.playing = Some(Playing { source, when: 0.0, offset: 0.0, rate: 1.0 });
+ let (playing_buffer, pending_buffer) = (speaker.buffer.clone(), decoding.buffer.clone());
+ let pending = audio_decoded(&probe);
+ drop(speaker);
+ drop(decoding);
+ JsFuture::from(pending).await.unwrap();
+ JsFuture::from(js_sys::Promise::resolve(&JsValue::UNDEFINED)).await.unwrap();
+ let counts = finish_audio_probe(&probe);
+ assert_eq!(counts.iter().map(|n| n.as_f64().unwrap()).collect::<Vec<_>>(), [1.0, 1.0, 1.0]);
+ assert!(playing_buffer.borrow().is_none(), "the finished owner releases its decoded buffer");
+ assert!(pending_buffer.borrow().is_none(), "a detached owner's pending decode must not be published");
+ assert_ne!(context.state(), AudioContextState::Closed);
+ let next = Speaker::new().unwrap();
+ assert_eq!(next.context, context, "another player keeps the shared page context");
+ }
+ }
}
-#[cfg(test)]
+#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
#[test]
fn a_takes_sound_is_read_as_written() {
diff --git a/src/manimgx/audio/fal.py b/src/manimgx/audio/fal.py
index 13fd72d0..2257a5d6 100644
--- a/src/manimgx/audio/fal.py
+++ b/src/manimgx/audio/fal.py
@@ -1,8 +1,7 @@
"""fal.ai's text-to-speech models, as voices: `Fal(model, **settings)`.
-[fal](https://fal.ai) runs text-to-speech models behind one API and one key. `Fal` speaks
-through the five most popular there (fal's "Trending" order, September 2026), one from each
-maker:
+[fal](https://fal.ai) runs text-to-speech models behind one API and one key. `Fal` supports
+the models below:
| Model | Its settings | Words timed |
| --- | --- | --- |
@@ -19,18 +18,21 @@ cache.
"""
import json
+import math
import os
import urllib.error
import urllib.request
from dataclasses import dataclass, field
-from typing import Literal, TypedDict, Unpack, overload
+from typing import Literal, Unpack, overload
+
+from typing_extensions import TypedDict
from manimgx.audio.sound import Speech, timed
# ── the models' settings, as their schemas on fal have them ────────────────────────────────
-class ElevenV3(TypedDict, total=False):
+class ElevenV3(TypedDict, total=False, closed=True):
"""The settings of ElevenLabs' Eleven v3 (`fal-ai/elevenlabs/tts/eleven-v3`), which times
its words. Its text may carry audio tags: `[whispers]`, `[laughs]`, `[excited]`…"""
@@ -115,7 +117,7 @@ class MiniMaxPronunciation(TypedDict, total=False):
`"燕少飞/(yan4)(shao3)(fei1)"`)."""
-class MiniMax(TypedDict, total=False):
+class MiniMax(TypedDict, total=False, closed=True):
"""The settings of MiniMax Speech 2.8 HD (`fal-ai/minimax/speech-2.8-hd`). Its text may
carry pauses, `<#0.5#>` (in seconds), and interjections: `(laughs)`, `(sighs)`, `(coughs)`,
`(clears throat)`, `(gasps)`, `(sniffs)`, `(groans)`, `(yawns)`."""
@@ -178,7 +180,7 @@ class MiniMax(TypedDict, total=False):
([`MiniMaxPronunciation`][manimgx.audio.fal.MiniMaxPronunciation])."""
-class Gemini(TypedDict, total=False):
+class Gemini(TypedDict, total=False, closed=True):
"""The settings of Google's Gemini 3.8 Flash TTS (`google/gemini-3.8-flash-tts`). Its text
may carry vocal events: `<laugh>`, `<sigh>`."""
@@ -220,7 +222,7 @@ class Gemini(TypedDict, total=False):
none)."""
-class Inworld(TypedDict, total=False):
+class Inworld(TypedDict, total=False, closed=True):
"""The settings of Inworld TTS-1.5 Max (`fal-ai/inworld-tts`), which speaks up to 2,000
characters a line."""
@@ -344,7 +346,7 @@ class Inworld(TypedDict, total=False):
"""Samples a second (default 48000)."""
-class Qwen(TypedDict, total=False):
+class Qwen(TypedDict, total=False, closed=True):
"""The settings of Alibaba's Qwen3-TTS 1.7B (`fal-ai/qwen-3-tts/text-to-speech/1.7b`)."""
voice: (
@@ -580,7 +582,7 @@ def _ordered(value: object) -> object:
def _pieces(stamps: object) -> list[tuple[str, float, float]]:
"""A model's timing of its text, as fal passes it on, as timed pieces of the text: items of
a text, a start and an end (words or characters), or ElevenLabs' alignments (characters,
- with their starts and ends). None if there is none, or if it runs backwards: the words
+ with their starts and ends). Empty if there is none, or if it runs backwards: the words
are then estimated."""
pieces: list[tuple[str, float, float]] = []
for item in stamps if isinstance(stamps, list) else ():
@@ -599,7 +601,8 @@ def _pieces(stamps: object) -> list[tuple[str, float, float]]:
elif {"text", "start", "end"} <= item.keys():
pieces.append((str(item["text"]), float(item["start"]), float(item["end"])))
starts = [a for _, a, _ in pieces]
- return pieces if starts == sorted(starts) else []
+ valid = all(0 <= a <= b and math.isfinite(b) for _, a, b in pieces)
+ return pieces if valid and starts == sorted(starts) else []
def _fetch(
diff --git a/src/manimgx/audio/sound.py b/src/manimgx/audio/sound.py
index 1db49b29..ae9e3604 100644
--- a/src/manimgx/audio/sound.py
+++ b/src/manimgx/audio/sound.py
@@ -1,12 +1,14 @@
"""Sounds and timed speech: sources, edits, placement and soundtrack mixing."""
import dataclasses
+import marshal
import math
import os
from collections.abc import Callable
from dataclasses import dataclass
from fractions import Fraction
from pathlib import Path
+from types import CodeType
from typing import TYPE_CHECKING, ClassVar, Self
from warnings import deprecated
@@ -31,8 +33,7 @@ CUT = 0.005
DUCK_ATTACK, DUCK_RELEASE = 0.15, 0.5
"""How long a ducking sound takes to dip before speech, and to come back after it (s)."""
BITRATE = 224_000
-"""The sound track's bitrate, a channel: AAC-LC at 224 kbps a channel, which PEAQ (ITU-R
-BS.1387) finds no listener could tell from the original, for speech, effects and music."""
+"""The soundtrack's bitrate per channel: AAC-LC at 224 kbps."""
type Source = str | os.PathLike[str] | bytes | np.ndarray
"""What a sound is made from: a file's path, a file's bytes (any common audio file: WAV, AIFF,
@@ -80,6 +81,8 @@ class Sound(Animation[Mobject]):
def __init__(self, source: Source, *, rate: int | None = None) -> None:
if isinstance(source, np.ndarray) != (rate is not None):
raise ValueError("give a rate with samples, and only with samples")
+ if rate is not None and (not math.isfinite(rate) or rate <= 0):
+ raise ValueError("the sample rate must be finite and positive")
if isinstance(source, str | os.PathLike) and not Path(source).is_file():
raise FileNotFoundError(f"no sound file at {os.fspath(source)!r}")
if isinstance(source, np.ndarray): # its own: the samples as they were given
@@ -119,8 +122,14 @@ class Sound(Animation[Mobject]):
return self._but(loop=math.inf if duration is None else duration)
def speed(self, factor: float) -> Self:
- """This sound played `factor` times as fast, as a tape is: higher and shorter."""
- return self._but(speed=self._edit.speed * factor)
+ """This sound played `factor` times as fast, as a tape is: higher and shorter.
+
+ The resulting speed must be finite and positive.
+ """
+ speed = self._edit.speed * factor
+ if not math.isfinite(speed) or speed <= 0:
+ raise ValueError("the resulting speed must be finite and positive")
+ return self._but(speed=speed)
def pan(self, position: float) -> Self:
"""This sound placed from -1 (left) through 0 (center) to 1 (right)."""
@@ -162,6 +171,14 @@ class Sound(Animation[Mobject]):
return self._edit.loop
return len(self.samples) / RATE
+ @property
+ def _duration(self) -> Fraction:
+ return (
+ Fraction(len(self.samples), RATE)
+ if self._edit.loop is None
+ else super()._duration
+ )
+
@property
def run_time(self) -> float:
"""Its duration: a sound plays for as long as it lasts. An endless one (`loop()`)
@@ -201,15 +218,18 @@ class Clip:
def stop(self, fade: float = 0.05) -> None:
"""Stop the sound: it fades out from now (the instant the scene is computing) over
`fade` seconds, and is silent after."""
- self.end, self.fade = clock.now + Fraction(fade).limit_denominator(RATE), fade
+ self.end, self.fade = clock.now + clock.rational(fade), fade
def _source_samples(sound: Sound) -> np.ndarray:
"""Source samples at RATE (2-D), possibly borrowed; consumers must not mutate them."""
source = sound.source
if isinstance(source, np.ndarray):
- data = source.astype(np.float32, copy=False).reshape(len(source), -1)
- rate = sound.rate or RATE
+ data = source.astype(np.float32, copy=False).reshape(
+ len(source), math.prod(source.shape[1:])
+ )
+ rate = sound.rate
+ assert rate is not None
return data if rate == RATE else _resample(data, rate, RATE)
raw = source if isinstance(source, bytes) else Path(source).read_bytes()
from manimgx._engine import digest
@@ -288,6 +308,8 @@ def mix(clips: list[Clip], duration: Fraction) -> np.ndarray | None:
continue
e, data = clip.sound._edit, clip.sound.samples
span = (total if clip.end is None else min(total, round(clip.end * RATE))) - at
+ if span <= 0:
+ continue
stop = 0.0 if clip.end is None else clip.fade
if e.loop == math.inf and len(data):
data = np.tile(data, (-(-span // len(data)), 1))[:span]
@@ -311,6 +333,10 @@ def _ducked(clips: list[Clip], decibels: float, at: int, length: int) -> np.ndar
if clip.sound.speaks:
a = float(clip.start)
b = a + clip.sound.duration
+ if clip.end is not None:
+ b = min(b, float(clip.end))
+ if b <= a:
+ continue
ease_in = np.clip((grid - (a - DUCK_ATTACK)) / DUCK_ATTACK, 0, 1)
ease_out = np.clip(((b + DUCK_RELEASE) - grid) / DUCK_RELEASE, 0, 1)
gain = np.minimum(gain, 1 - (1 - low) * np.minimum(ease_in, ease_out))
@@ -389,7 +415,7 @@ _WORD = re.compile(r"\S+")
def estimate(text: str, samples: np.ndarray) -> tuple[Word, ...]:
"""When each word of `text` is said in `samples`, estimated: the audio's voiced span
shared among the words by their letters, with a share more for the pause after a comma or
- a full stop. Against voices' own times, it is off by about 0.1 s on average."""
+ a full stop."""
words = _WORD.findall(text)
if not words:
return ()
@@ -503,7 +529,9 @@ def cached(voice: Voice, text: str, folder: Path) -> Speech:
audio = stem.with_suffix(record["suffix"]).read_bytes()
same = record.get("audio") == f"{digest(audio):016x}"
words = [Word(*w) for w in record["words"]] if same else []
- return Speech(audio, text=text, words=words)
+ speech = Speech(audio, text=text, words=words)
+ speech._edit = _Edit(**record.get("edit", {}))
+ return speech
speech = voice(text)
folder.mkdir(parents=True, exist_ok=True)
suffix, data = _stored(speech)
@@ -515,6 +543,7 @@ def cached(voice: Voice, text: str, folder: Path) -> Speech:
"voice": identity,
"suffix": suffix,
"audio": f"{digest(data):016x}",
+ "edit": dataclasses.asdict(speech._edit),
"words": [[w.text, w.start, w.end] for w in speech._words],
},
indent=1,
@@ -542,7 +571,7 @@ def _identity(voice: Voice) -> str:
return f"{type(voice).__module__}.{type(voice).__qualname__}{_fields(voice)!r}"
from manimgx._engine import digest
- body = digest(code.co_code, repr(code.co_consts).encode())
+ body = digest(marshal.dumps(_code(code)))
cells = tuple(c.cell_contents for c in getattr(fn, "__closure__", None) or ())
defaults = getattr(fn, "__defaults__", None) or ()
keywords = getattr(fn, "__kwdefaults__", None) or {}
@@ -556,6 +585,18 @@ def _identity(voice: Voice) -> str:
return f"{name}#{body:016x}{held!r}"
+def _code(code: CodeType) -> CodeType:
+ """A voice's code without source locations, including its nested functions."""
+ return code.replace(
+ co_filename="",
+ co_firstlineno=0,
+ co_linetable=b"",
+ co_consts=tuple(
+ _code(c) if isinstance(c, CodeType) else c for c in code.co_consts
+ ),
+ )
+
+
def _fields(owner: object) -> tuple[tuple[str, object], ...]:
"""An object's plain fields, in order of their names."""
plain = ((k, _plain(v)) for k, v in sorted(getattr(owner, "__dict__", {}).items()))
@@ -580,7 +621,7 @@ def _stored(speech: Speech) -> tuple[str, bytes]:
if isinstance(source, bytes):
return _sniff(source), source
if isinstance(source, np.ndarray):
- return ".wav", _wav(speech.samples)
+ return ".wav", _wav(_source_samples(speech))
path = Path(source)
return path.suffix or ".bin", path.read_bytes()
@@ -632,7 +673,7 @@ def timed(text: str, pieces: Sequence[tuple[str, float, float]]) -> tuple[Word,
owner += [(a, b)] * sum(c.isalnum() for c in piece)
letters = [c.lower() for c in text if c.isalnum()]
spelled = [c.lower() for piece, _, _ in pieces for c in piece if c.isalnum()]
- if letters != spelled:
+ if not letters or letters != spelled:
return ()
words, k = [], 0
for m in _WORD.finditer(text):
@@ -640,6 +681,7 @@ def timed(text: str, pieces: Sequence[tuple[str, float, float]]) -> tuple[Word,
if n:
words.append(Word(m.group(), owner[k][0], owner[k + n - 1][1]))
k += n
- elif words: # a word of no letters ("—"): at the end of the one before
- words.append(Word(m.group(), words[-1].end, words[-1].end))
+ else: # punctuation: beside the preceding word, or the first following word
+ at = words[-1].end if words else owner[0][0]
+ words.append(Word(m.group(), at, at))
return tuple(words)
diff --git a/tests/integration/test_audio_inputs.py b/tests/integration/test_audio_inputs.py
new file mode 100644
index 00000000..3a1b560b
--- /dev/null
+++ b/tests/integration/test_audio_inputs.py
@@ -0,0 +1,71 @@
+"""Audio time and interleaved frames are validated before native size arithmetic."""
+
+import io
+import math
+import wave
+
+import numpy as np
+import pytest
+
+import manimgx as m
+from manimgx import _engine
+
+
[email protected]("rate", [0, -1])
+def test_sample_rate_is_positive_where_samples_are_given(rate: int) -> None:
+ with pytest.raises(ValueError, match="sample rate"):
+ m.Sound(np.ones(1, np.float32), rate=rate)
+
+
[email protected]("factor", [0, -1, math.nan, math.inf, -math.inf])
+def test_speed_is_finite_and_positive_where_it_is_edited(factor: float) -> None:
+ sound = m.Sound(np.ones(1, np.float32), rate=48_000)
+ with pytest.raises(ValueError, match="speed"):
+ sound.speed(factor)
+
+
[email protected]("factor", [1e-200, 1e200])
+def test_composed_speed_must_remain_representable(factor: float) -> None:
+ sound = m.Sound(np.ones(1, np.float32), rate=48_000).speed(factor)
+ with pytest.raises(ValueError, match="speed"):
+ sound.speed(factor)
+
+
+ ("channels", "rate", "to"),
+ [
+ (0, 48_000, 48_000),
+ (2, 48_000, 48_000), # one sample cannot form a stereo frame
+ (1, 0, 48_000),
+ (1, 48_000, 0),
+ (1, -1, 48_000),
+ (1, math.nan, 48_000),
+ (1, 48_000, math.nan),
+ (1, math.inf, 48_000),
+ (1, 48_000, math.inf),
+ (1, 1e-100, 48_000), # finite metadata, unrepresentable output length
+ ],
+)
+def test_native_resampling_rejects_invalid_frames_and_time(
+ channels: int, rate: float, to: float
+) -> None:
+ with pytest.raises(ValueError, match="audio"):
+ _engine.resample_audio(np.ones(1, np.float32).tobytes(), channels, rate, to)
+
+
+def test_decoding_uses_the_same_sample_rate_contract() -> None:
+ encoded = io.BytesIO()
+ with wave.open(encoded, "wb") as out:
+ out.setnchannels(1)
+ out.setsampwidth(2)
+ out.setframerate(48_000)
+ out.writeframes(bytes(16))
+ with pytest.raises(ValueError, match="sample rates"):
+ _engine.decode_audio(encoded.getvalue(), 0)
+
+
[email protected]("channels", [1, 2, 3])
+def test_valid_empty_and_identity_resampling_remain_exact(channels: int) -> None:
+ samples = np.arange(5 * channels, dtype=np.float32).tobytes()
+ assert _engine.resample_audio(samples, channels, 48_000, 48_000) == samples
+ assert _engine.resample_audio(b"", channels, 24_000, 48_000) == b""
diff --git a/tests/integration/test_sound.py b/tests/integration/test_sound.py
index 9d2b8bd8..8b8a2502 100644
--- a/tests/integration/test_sound.py
+++ b/tests/integration/test_sound.py
@@ -19,6 +19,7 @@ import manimgx as m
from manimgx._engine import decode_audio, resample_audio
from manimgx.audio.sound import RATE
from manimgx.config import config
+from manimgx.rendering.film import X264Preset
def impulse() -> m.Sound:
@@ -62,11 +63,12 @@ pytestmark = pytest.mark.config(pixel_width=320, pixel_height=180)
@pytest.mark.parametrize("fps", [10, 24, 60])
[email protected]("preset", ["ultrafast", "slow"])
def test_each_sound_is_at_its_sample_and_lasts_the_film(
- tmp_path: Path, fps: int
+ tmp_path: Path, fps: int, preset: X264Preset
) -> None:
config.frame_rate = fps
- film = Beats().render(tmp_path / "beats.mp4")
+ film = Beats().render(tmp_path / "beats.mp4", preset=preset)
assert streams(tmp_path / "beats.mp4") == ["h264", "aac"]
x = decoded(tmp_path / "beats.mp4")
assert len(x) == film.frame_count * RATE // fps
diff --git a/tests/unit/audio/test_fal.py b/tests/unit/audio/test_fal.py
index c5475484..0031c7ea 100644
--- a/tests/unit/audio/test_fal.py
+++ b/tests/unit/audio/test_fal.py
@@ -111,7 +111,7 @@ class TestAsking:
assert speech.duration == pytest.approx(0.1)
def test_another_models_settings_are_refused(self) -> None:
- untyped: Callable[..., Fal] = Fal # as a caller no type checker reads
+ untyped = cast("Callable[..., Fal]", Fal) # as a caller no type checker reads
with pytest.raises(TypeError, match="takes"):
untyped("google/gemini-3.8-flash-tts", stability=0.5)
@@ -194,8 +194,21 @@ class TestTiming:
{"text": "you", "start": 0.0, "end": 0.02},
],
[{"text": "bye", "start": 0.0, "end": 0.02}],
+ [{"text": "hi you", "start": 0.05, "end": 0.0}],
+ [{"text": "hi you", "start": -0.05, "end": 0.02}],
+ [{"text": "hi you", "start": 0.0, "end": float("inf")}],
+ [{"text": "hi you", "start": 0.0, "end": float("nan")}],
+ ],
+ ids=[
+ "none",
+ "empty",
+ "backwards",
+ "other words",
+ "negative duration",
+ "negative start",
+ "infinite end",
+ "nan end",
],
- ids=["none", "empty", "backwards", "other words"],
)
def test_without_usable_timing_the_words_are_estimated(
self, service: Service, stamps: object
diff --git a/tests/unit/audio/test_sound.py b/tests/unit/audio/test_sound.py
index b9105f4e..717469cb 100644
--- a/tests/unit/audio/test_sound.py
+++ b/tests/unit/audio/test_sound.py
@@ -2,6 +2,7 @@
from fractions import Fraction
from pathlib import Path
+from typing import cast
import numpy as np
import pytest
@@ -39,6 +40,15 @@ def click() -> m.Sound:
return m.Sound(x, rate=RATE)
+def test_a_stop_keeps_its_time_until_audio_is_sampled(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr(clock, "now", Fraction(3, 10))
+ clip = Clip(click(), Fraction(0))
+ clip.stop(1e-8)
+ assert clip.end == Fraction(3, 10) + Fraction(1, 100000000)
+
+
def starts(scene: type[m.Scene], fps: float) -> list[Fraction]:
saved = config.frame_rate
config.frame_rate = fps
@@ -145,6 +155,12 @@ class TestPlacement:
class TestEdits:
+ @pytest.mark.parametrize("shape", [(0,), (0, 1), (0, 2)])
+ def test_empty_samples_have_zero_duration(self, shape: tuple[int, ...]) -> None:
+ sound = m.Sound(np.empty(shape, np.float32), rate=RATE)
+ assert sound.duration == 0
+ assert sound.samples.shape == (0, 1 if len(shape) == 1 else shape[1])
+
@given(decibels=st.floats(-60, 60), more=st.floats(-60, 60))
def test_gain_is_decibels(self, decibels: float, more: float) -> None:
a = tone()
@@ -193,6 +209,29 @@ class TestEdits:
class TestMix:
+ @pytest.mark.parametrize("loop", [False, True])
+ @pytest.mark.parametrize("end", [Fraction(0), Fraction(1)])
+ def test_a_clip_stopped_before_it_starts_is_silent(
+ self, loop: bool, end: Fraction
+ ) -> None:
+ sound = m.Sound(np.ones(2 * RATE, np.float32), rate=RATE)
+ if loop:
+ sound = sound.loop()
+ track = mix([Clip(sound, Fraction(1), end=end, fade=0)], Fraction(3))
+ assert track is not None
+ assert not track.any()
+
+ def test_ducking_releases_when_speech_is_stopped(self) -> None:
+ bed = m.Sound(np.ones(RATE, np.float32), rate=RATE).loop().duck(12)
+ speech = Speech(np.zeros(RATE, np.float32), text="quiet", rate=RATE)
+ track = mix(
+ [Clip(bed, Fraction(0)), Clip(speech, Fraction(2), end=Fraction(11, 5))],
+ Fraction(4),
+ )
+ assert track is not None
+ assert track[round(2.1 * RATE), 0] == pytest.approx(10 ** (-12 / 20))
+ assert track[round(2.8 * RATE), 0] == 1
+
@settings(max_examples=50)
@given(
starts=st.lists(
@@ -454,6 +493,70 @@ class TestVoice:
@pytest.mark.usefixtures("cache_here")
class TestCache:
+ def test_function_identity_keeps_code_but_not_source_locations(self) -> None:
+ def voice(source: str, filename: str) -> voices.Voice:
+ scope: dict[str, object] = {
+ "__name__": "recorded_voice",
+ "speak": said,
+ "first_speaker": said,
+ "second_speaker": said,
+ }
+ exec(compile(source, filename, "exec"), scope)
+ return cast("voices.Voice", scope["voice"])
+
+ source = (
+ "def voice(text):\n return speak(' '.join(w for w in text.split()))\n"
+ )
+ original = voices._identity(voice(source, "first.py"))
+ assert voices._identity(voice("\n\n" + source, "second.py")) == original
+ assert (
+ voices._identity(voice(source.replace("' '", "'_'"), "first.py"))
+ != original
+ )
+ first = "def voice(text):\n return first_speaker(text)\n"
+ second = first.replace("first_speaker", "second_speaker")
+ assert voices._identity(voice(first, "voice.py")) != voices._identity(
+ voice(second, "voice.py")
+ )
+
+ @pytest.mark.parametrize("kind", ["array", "bytes", "file"])
+ def test_cached_speech_preserves_edits_and_word_times(
+ self, kind: str, cache_here: Path, tmp_path: Path
+ ) -> None:
+ samples = np.linspace(-0.5, 0.5, RATE, dtype=np.float32)
+ audio = voices._wav(samples)
+ path = tmp_path / "source.wav"
+ path.write_bytes(audio)
+
+ def voice(text: str) -> Speech:
+ source = samples if kind == "array" else audio if kind == "bytes" else path
+ return (
+ Speech(
+ source,
+ text=text,
+ words=[Word(text, 0.2, 0.3)],
+ rate=RATE if kind == "array" else None,
+ )
+ .trim(0.1, 0.4)
+ .speed(2)
+ .gain(-6)
+ .pan(0.25)
+ .fade_in(0.01)
+ .fade_out(0.02)
+ .loop(0.35)
+ .duck(9)
+ )
+
+ first = cached(voice, "hello", cache_here)
+ again = cached(voice, "hello", cache_here)
+ assert first.duration == again.duration == 0.35
+ assert first.words == again.words
+ np.testing.assert_array_equal(first.samples, again.samples)
+ np.testing.assert_array_equal(
+ mix([Clip(first, Fraction(0))], Fraction(1)),
+ mix([Clip(again, Fraction(0))], Fraction(1)),
+ )
+
def test_a_voice_speaks_each_text_once(self, cache_here: Path) -> None:
calls: list[str] = []
@@ -505,6 +608,17 @@ class TestCache:
@pytest.mark.usefixtures("cache_here")
class TestWords:
+ def test_leading_punctuation_keeps_its_place_in_timed_text(self) -> None:
+ text = "— hi"
+ speech = Speech(
+ np.zeros(RATE, np.float32),
+ text=text,
+ words=timed(text, [("hi", 0.1, 0.2)]),
+ rate=RATE,
+ )
+ assert speech.words == (Word("—", 0.1, 0.1), Word("hi", 0.1, 0.2))
+ assert times(speech, (2, 4)) == (0.1, 0.2)
+
def test_pieces_time_the_words(self) -> None:
pieces = [(c, i * 0.1, i * 0.1 + 0.1) for i, c in enumerate("hi, you")]
hi, you = timed("hi, you", pieces)