Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

Pad LTX audio-to-video audio past the clip so the render no longer crashes in RoPE

Claude68136144ca9a
diff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.py
index 4e597cc9a1..f1e5e1b282 100644
--- a/scripts/generate_ltx2.py
+++ b/scripts/generate_ltx2.py
@@ -1776,6 +1776,47 @@ def run_extend(args: argparse.Namespace) -> str:
return pipe._decode_and_save_video(video_latent, audio_latent, args.output)
+# Trailing silence added past the clip, so the audio VAE always has more source
+# than the `num_frames / fps * 25` tokens the pipeline slices and positions.
+A2V_AUDIO_TAIL_PAD_SEC = 0.25
+
+
+def a2v_audio_covering_clip(audio_path: str, start_sec: float, num_frames: int, fps: float,
+ out_dir: str) -> tuple[str, float, str | None]:
+ """Return (audio path, start offset, temp path to delete) whose audio covers the clip.
+
+ A2VidPipelineTwoStage reads at most `num_frames / fps` seconds, encodes it,
+ and slices `round(num_frames / fps * 25)` audio tokens while sizing RoPE
+ positions for exactly that count. Audio that ends at (or a frame before)
+ the clip's end encodes to fewer tokens, and the render dies in
+ `apply_rope_split` with a broadcast error. A Music Video window is cut to
+ `(num_frames - 1) / fps` (the federated wire contract), so it is always a
+ frame short. Re-cut it from `start_sec` and pad with silence to the clip
+ plus `A2V_AUDIO_TAIL_PAD_SEC`; the pipeline discards tokens past the clip.
+ Without ffmpeg the original audio is returned unchanged.
+ """
+ import shutil
+ import subprocess
+ import tempfile
+ if not shutil.which("ffmpeg"):
+ emit_status("ffmpeg not on PATH — passing a2v audio unpadded")
+ return audio_path, start_sec, None
+ cover_sec = num_frames / fps + A2V_AUDIO_TAIL_PAD_SEC
+ fd, padded = tempfile.mkstemp(prefix="a2v-audio-", suffix=".wav", dir=out_dir)
+ os.close(fd)
+ proc = subprocess.run(
+ ["ffmpeg", "-y", "-loglevel", "error", "-ss", f"{max(0.0, start_sec):.6f}", "-i", audio_path, "-vn",
+ "-af", f"apad=whole_dur={cover_sec:.6f}", "-t", f"{cover_sec:.6f}",
+ "-ar", "48000", "-ac", "2", "-c:a", "pcm_s16le", padded],
+ check=False,
+ )
+ if proc.returncode != 0 or Path(padded).stat().st_size == 0:
+ Path(padded).unlink(missing_ok=True)
+ emit_status(f"a2v audio padding failed (exit {proc.returncode}); passing audio unpadded")
+ return audio_path, start_sec, None
+ return padded, 0.0, padded
+
+
def run_a2v(args: argparse.Namespace) -> str:
"""Audio-to-video — generate a clip whose motion + audio track sync to an
input WAV/MP3. Two-stage pipeline (dev model + CFG at half-res, then
@@ -1809,12 +1850,14 @@ def run_a2v(args: argparse.Namespace) -> str:
args.teacache_thresh)
if _A2V_TC_CONFIG["enable"]:
emit_status(f"TeaCache active on A2V Stage 1 (thresh={_teacache_thresh_label(args.teacache_thresh)})")
+ audio_path, audio_start, padded_audio = a2v_audio_covering_clip(
+ args.audio, args.audio_start, args.num_frames, args.fps, str(Path(args.output).parent))
restore_image_rate = _patch_a2v_image_conditioning_rate(args.fps)
try:
return _run_with_ltx_stepwise_preview(pipe, args, lambda: pipe.generate_and_save(
prompt=args.prompt,
output_path=args.output,
- audio_path=args.audio,
+ audio_path=audio_path,
image=args.image,
height=args.height,
width=args.width,
@@ -1823,12 +1866,14 @@ def run_a2v(args: argparse.Namespace) -> str:
stage1_steps=stage1_steps,
stage2_steps=args.stage2_steps,
cfg_scale=args.cfg_scale if args.cfg_scale is not None else 3.0,
- audio_start_time=args.audio_start,
+ audio_start_time=audio_start,
**_rate_kwargs(pipe.generate_and_save, args.fps),
))
finally:
restore_image_rate()
_A2V_TC_CONFIG = None
+ if padded_audio:
+ Path(padded_audio).unlink(missing_ok=True)
def run_ic_lora(args: argparse.Namespace) -> str:
diff --git a/scripts/generate_ltx2.test.js b/scripts/generate_ltx2.test.js
index 33a6d8aaad..10409166fe 100644
--- a/scripts/generate_ltx2.test.js
+++ b/scripts/generate_ltx2.test.js
@@ -12,6 +12,7 @@ const script = join(dirname(fileURLToPath(import.meta.url)), 'generate_ltx2.py')
// failing. Every helper exercised here is import-free by design (stdlib only),
// so no mlx/ltx wheel is needed.
const pyBin = resolveTestPython();
+const hasFfmpeg = (() => { try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })();
const runPython = (source) => execFileSync(pyBin, ['-c', source, script], { encoding: 'utf8' });
const importRunner = [
@@ -1710,3 +1711,34 @@ describe.skipIf(!pyBin)('generate_ltx2.py block streaming (#6499)', () => {
expect(out.trim().split(/\r?\n/).map((l) => l.trimEnd())).toEqual(['auto', 'stream', 'resident']);
});
});
+
+// A Music Video supplied-audio window is cut to (numFrames - 1) / fps on the
+// federated wire, one frame short of the num_frames / fps the A2V pipeline
+// slices tokens for. Unpadded, LTX dies in RoPE with a broadcast error.
+describe.skipIf(!pyBin || !hasFfmpeg)('generate_ltx2.py a2v audio coverage', () => {
+ it('re-cuts the window from its offset and pads it past the clip with silence', () => {
+ const out = runPython(`${importRunner}
+import subprocess, tempfile, wave, os
+tmp = tempfile.mkdtemp()
+src = os.path.join(tmp, "song.wav")
+subprocess.run(["ffmpeg", "-v", "error", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=6",
+ "-ac", "2", "-c:a", "pcm_s16le", "-y", src], check=True)
+# The window starts 2s in and holds (97 - 1) / 24 = 4.0s of audio in the song.
+path, start, temp = runner.a2v_audio_covering_clip(src, 2.0, 97, 24.0, tmp)
+with wave.open(path) as w:
+ print(round(w.getnframes() / w.getframerate(), 4), start, temp == path)
+# The tail is silence once the song runs out (a 1s source, a 97-frame clip).
+short = os.path.join(tmp, "short.wav")
+subprocess.run(["ffmpeg", "-v", "error", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=1",
+ "-ac", "2", "-c:a", "pcm_s16le", "-y", short], check=True)
+path, _, _ = runner.a2v_audio_covering_clip(short, 0.0, 97, 24.0, tmp)
+with wave.open(path) as w:
+ w.setpos(w.getnframes() - 1)
+ print(round(w.getnframes() / w.getframerate(), 4), w.readframes(1) == b"\\x00" * 4)
+import shutil
+shutil.rmtree(tmp)
+`);
+ // 97 / 24 + 0.25s of tail, offset consumed into the cut.
+ expect(out.trim().split(/\r?\n/)).toEqual(['4.2917 0.0 True', '4.2917 True']);
+ });
+});