Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
Pad LTX audio-to-video audio past the clip so the render no longer crashes in RoPE
Claude68136144ca9a
diff --git a/scripts/generate_ltx2.py b/scripts/generate_ltx2.pyindex 4e597cc9a1..f1e5e1b282 100644--- a/scripts/generate_ltx2.py+++ b/scripts/generate_ltx2.py@@ -1776,6 +1776,47 @@ def run_extend(args: argparse.Namespace) -> str:return pipe._decode_and_save_video(video_latent, audio_latent, args.output)+# Trailing silence added past the clip, so the audio VAE always has more source+# than the `num_frames / fps * 25` tokens the pipeline slices and positions.+A2V_AUDIO_TAIL_PAD_SEC = 0.25+++def a2v_audio_covering_clip(audio_path: str, start_sec: float, num_frames: int, fps: float,+ out_dir: str) -> tuple[str, float, str | None]:+ """Return (audio path, start offset, temp path to delete) whose audio covers the clip.++ A2VidPipelineTwoStage reads at most `num_frames / fps` seconds, encodes it,+ and slices `round(num_frames / fps * 25)` audio tokens while sizing RoPE+ positions for exactly that count. Audio that ends at (or a frame before)+ the clip's end encodes to fewer tokens, and the render dies in+ `apply_rope_split` with a broadcast error. A Music Video window is cut to+ `(num_frames - 1) / fps` (the federated wire contract), so it is always a+ frame short. Re-cut it from `start_sec` and pad with silence to the clip+ plus `A2V_AUDIO_TAIL_PAD_SEC`; the pipeline discards tokens past the clip.+ Without ffmpeg the original audio is returned unchanged.+ """+ import shutil+ import subprocess+ import tempfile+ if not shutil.which("ffmpeg"):+ emit_status("ffmpeg not on PATH — passing a2v audio unpadded")+ return audio_path, start_sec, None+ cover_sec = num_frames / fps + A2V_AUDIO_TAIL_PAD_SEC+ fd, padded = tempfile.mkstemp(prefix="a2v-audio-", suffix=".wav", dir=out_dir)+ os.close(fd)+ proc = subprocess.run(+ ["ffmpeg", "-y", "-loglevel", "error", "-ss", f"{max(0.0, start_sec):.6f}", "-i", audio_path, "-vn",+ "-af", f"apad=whole_dur={cover_sec:.6f}", "-t", f"{cover_sec:.6f}",+ "-ar", "48000", "-ac", "2", "-c:a", "pcm_s16le", padded],+ check=False,+ )+ if proc.returncode != 0 or Path(padded).stat().st_size == 0:+ Path(padded).unlink(missing_ok=True)+ emit_status(f"a2v audio padding failed (exit {proc.returncode}); passing audio unpadded")+ return audio_path, start_sec, None+ return padded, 0.0, padded++def run_a2v(args: argparse.Namespace) -> str:"""Audio-to-video — generate a clip whose motion + audio track sync to aninput WAV/MP3. Two-stage pipeline (dev model + CFG at half-res, then@@ -1809,12 +1850,14 @@ def run_a2v(args: argparse.Namespace) -> str:args.teacache_thresh)if _A2V_TC_CONFIG["enable"]:emit_status(f"TeaCache active on A2V Stage 1 (thresh={_teacache_thresh_label(args.teacache_thresh)})")+ audio_path, audio_start, padded_audio = a2v_audio_covering_clip(+ args.audio, args.audio_start, args.num_frames, args.fps, str(Path(args.output).parent))restore_image_rate = _patch_a2v_image_conditioning_rate(args.fps)try:return _run_with_ltx_stepwise_preview(pipe, args, lambda: pipe.generate_and_save(prompt=args.prompt,output_path=args.output,- audio_path=args.audio,+ audio_path=audio_path,image=args.image,height=args.height,width=args.width,@@ -1823,12 +1866,14 @@ def run_a2v(args: argparse.Namespace) -> str:stage1_steps=stage1_steps,stage2_steps=args.stage2_steps,cfg_scale=args.cfg_scale if args.cfg_scale is not None else 3.0,- audio_start_time=args.audio_start,+ audio_start_time=audio_start,**_rate_kwargs(pipe.generate_and_save, args.fps),))finally:restore_image_rate()_A2V_TC_CONFIG = None+ if padded_audio:+ Path(padded_audio).unlink(missing_ok=True)def run_ic_lora(args: argparse.Namespace) -> str:diff --git a/scripts/generate_ltx2.test.js b/scripts/generate_ltx2.test.jsindex 33a6d8aaad..10409166fe 100644--- a/scripts/generate_ltx2.test.js+++ b/scripts/generate_ltx2.test.js@@ -12,6 +12,7 @@ const script = join(dirname(fileURLToPath(import.meta.url)), 'generate_ltx2.py')// failing. Every helper exercised here is import-free by design (stdlib only),// so no mlx/ltx wheel is needed.const pyBin = resolveTestPython();+const hasFfmpeg = (() => { try { execFileSync('ffmpeg', ['-version'], { stdio: 'ignore' }); return true; } catch { return false; } })();const runPython = (source) => execFileSync(pyBin, ['-c', source, script], { encoding: 'utf8' });const importRunner = [@@ -1710,3 +1711,34 @@ describe.skipIf(!pyBin)('generate_ltx2.py block streaming (#6499)', () => {expect(out.trim().split(/\r?\n/).map((l) => l.trimEnd())).toEqual(['auto', 'stream', 'resident']);});});++// A Music Video supplied-audio window is cut to (numFrames - 1) / fps on the+// federated wire, one frame short of the num_frames / fps the A2V pipeline+// slices tokens for. Unpadded, LTX dies in RoPE with a broadcast error.+describe.skipIf(!pyBin || !hasFfmpeg)('generate_ltx2.py a2v audio coverage', () => {+ it('re-cuts the window from its offset and pads it past the clip with silence', () => {+ const out = runPython(`${importRunner}+import subprocess, tempfile, wave, os+tmp = tempfile.mkdtemp()+src = os.path.join(tmp, "song.wav")+subprocess.run(["ffmpeg", "-v", "error", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=6",+ "-ac", "2", "-c:a", "pcm_s16le", "-y", src], check=True)+# The window starts 2s in and holds (97 - 1) / 24 = 4.0s of audio in the song.+path, start, temp = runner.a2v_audio_covering_clip(src, 2.0, 97, 24.0, tmp)+with wave.open(path) as w:+ print(round(w.getnframes() / w.getframerate(), 4), start, temp == path)+# The tail is silence once the song runs out (a 1s source, a 97-frame clip).+short = os.path.join(tmp, "short.wav")+subprocess.run(["ffmpeg", "-v", "error", "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=1",+ "-ac", "2", "-c:a", "pcm_s16le", "-y", short], check=True)+path, _, _ = runner.a2v_audio_covering_clip(short, 0.0, 97, 24.0, tmp)+with wave.open(path) as w:+ w.setpos(w.getnframes() - 1)+ print(round(w.getnframes() / w.getframerate(), 4), w.readframes(1) == b"\\x00" * 4)+import shutil+shutil.rmtree(tmp)+`);+ // 97 / 24 + 0.25s of tail, offset consumed into the cut.+ expect(out.trim().split(/\r?\n/)).toEqual(['4.2917 0.0 True', '4.2917 True']);+ });+});