Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
"""Extend delivery: the clip you already had, untouched, plus the part the model made.
Issue #48 — "progressive audio/video distortion when using Extend".
WHY THIS EXISTS. Extend conditions the model on the source clip, and the
model never re-denoises that region (LTX carries it with ``denoise_mask = 0``;
H3 holds it as window 0 of the timeline). But both engines used to DELIVER the
source region as the model pipeline's own copy of it, so every round put the
whole accumulated clip through one more lossy generation:
* LTX: the entire clip was VAE-encoded and decoded again, its audio resampled
to 16 kHz for the audio VAE and re-synthesised to 48 kHz by the vocoder
(everything above 8 kHz discarded and re-invented), and the result written
at the output preset (yuv420p / CRF 18 by default) — per round, over the
whole clip, compounding.
* H3: the source was decoded, cover-scaled onto the canvas, its audio
resampled to 32 kHz, and the whole clip re-encoded at CRF 18 + AAC — again
per round, and the gallery's 720p export (which the Extend picker offers)
added a lanczos down-and-up resample on top.
The fix keeps the model's conditioning exactly as it was and changes only the
DELIVERY: the source region is the source file itself, the new region is what
the model made, joined at the seam.
* Video — the source's H.264 packets are copied bit-exact (concat demuxer,
``-c copy``) whenever the source can be continued that way: same codec,
canvas and frame rate, every one of its frames used by the model, and the
new part's encoder parameter sets (``avcC``) byte-identical to the source's.
Otherwise (a first hop from a foreign file, a downscaled copy, a changed
output preset) the source is decoded ONCE from the file the user picked and
encoded once together with the new part — a single generation, after which
the next round is a copy again.
* Audio — AAC cannot be concatenated without its priming/padding showing at
the splice, and re-encoding the previous round's AAC accumulates (measured:
34.1 -> 32.2 -> 29.8 dB SNR over three tandem rounds at 192 kb/s). So every
extend output keeps a lossless 48 kHz FLAC "audio master" in a dot-folder,
and each round's AAC is encoded ONCE from master PCM. The source region of
that PCM is byte-identical round after round, so the delivered audio is
always exactly one generation from what the user had — never two.
* Seam — a short linear crossfade (40 ms, ending on the seam) from the
source audio into the generated track. Linear, not equal-power: just before
the seam the generated track is the model's re-synthesis of the SAME sound,
so the two signals are correlated and an equal-power pair would bump the
level. No video blend: the first new frame continues the model's decode of
the last source frame, and the measured seam jump is reported per job.
This module is pure apart from ffmpeg/ffprobe: every ffmpeg call goes through
the ``run(cmd, label)`` callable the caller passes (the panel passes its
cancellable ``run_ffmpeg_tracked``), so it is unit-testable on synthetic clips
without a GPU.
"""
from __future__ import annotations
import json
import subprocess
from dataclasses import dataclass
from pathlib import Path
from typing import Callable
import numpy as np
SAMPLE_RATE = 48000
CHANNELS = 2
FADE_SECONDS = 0.040
AUDIO_BITRATE = "192k"
# LTX's video VAE is causal in time: latent frame 0 holds one pixel frame and
# every later latent holds 8, so the extend pipeline only ever sees the first
# 1 + 8k frames of its source (retake.py `_encode_source_video`).
VAE_TIME_STRIDE = 8
Runner = Callable[[list, str], object]
class SpliceError(RuntimeError):
"""The splice could not produce a verified file (the caller keeps the
model's own output, i.e. the pre-#48 delivery, and says so)."""
# --------------------------------------------------------------------------
# Layout: which frames of which file go where
# --------------------------------------------------------------------------
@dataclass(frozen=True)
class Layout:
source_frames: int # frames of the SOURCE file the delivery keeps
new_start: int # first generated frame that is new
new_end: int # one past the last generated frame that is new
source_first: bool # True = extend after, False = extend before
@property
def new_frames(self) -> int:
return self.new_end - self.new_start
@property
def total_frames(self) -> int:
return self.source_frames + self.new_frames
@property
def seam_frame(self) -> int:
"""Index in the DELIVERED clip of the first frame after the seam."""
return self.source_frames if self.source_first else self.new_frames
def ltx_source_frames(n_source: int) -> int:
"""How many leading frames of an n-frame source the LTX extend pipeline
encodes (1 + 8k, rounded down; any remainder is dropped by the model)."""
if n_source < 1:
raise SpliceError("the source has no frames")
return 1 + VAE_TIME_STRIDE * max(0, (n_source - 1) // VAE_TIME_STRIDE)
def ltx_layout(n_source: int, n_generated: int, direction: str = "after") -> Layout:
"""LTX extend: the decoded output holds the K = 1+8k source frames plus
8 per new latent, new ones after (or before) the source region."""
k = ltx_source_frames(n_source)
new = n_generated - k
if new < 1:
raise SpliceError(f"the model returned {n_generated} frames for a "
f"{k}-frame source — no new frames to splice")
if direction == "before":
return Layout(source_frames=k, new_start=0, new_end=new, source_first=False)
return Layout(source_frames=k, new_start=k, new_end=n_generated, source_first=True)
def h3_layout(n_generated: int, new_frames: int) -> Layout:
"""H3 extend: the runner delivers its source frames followed by exactly
``new_frames`` new ones (`--extend-frames`). Forward only."""
src = n_generated - int(new_frames)
if src < 1 or new_frames < 1:
raise SpliceError(f"the H3 runner returned {n_generated} frames for "
f"{new_frames} new ones — nothing to splice")
return Layout(source_frames=src, new_start=src, new_end=n_generated, source_first=True)
# --------------------------------------------------------------------------
# Audio: pure numpy
# --------------------------------------------------------------------------
def crossfade_splice(source: np.ndarray, generated: np.ndarray, seam: int,
fade: int, source_first: bool = True) -> np.ndarray:
"""Join source PCM and generated PCM at sample ``seam`` of the delivered
timeline, both ``(channels, samples)`` float32.
``source_first`` (extend after): ``generated`` is on the delivered
timeline (sample 0 = clip start, its first ``seam`` samples are the
model's copy of the source). Output = source[:seam-fade], a linear fade
from source to generated over [seam-fade, seam), then generated[seam:].
A source shorter than the seam hands over where it ends instead.
Extend before: ``generated`` holds the new part in [0, seam) followed by
the model's copy of the source; ``source`` starts at the seam. Output =
generated[:seam], a fade from generated into the source over its first
``fade`` samples, then the rest of the source.
"""
source = np.asarray(source, dtype=np.float32)
generated = np.asarray(generated, dtype=np.float32)
if source.ndim != 2 or generated.ndim != 2 or source.shape[0] != generated.shape[0]:
raise SpliceError("source and generated audio must both be (channels, samples)")
seam = max(0, int(seam))
if source_first:
hand = min(seam, source.shape[1], generated.shape[1])
f = max(0, min(int(fade), hand))
head = source[:, :hand - f]
if f:
w = ((np.arange(f, dtype=np.float32) + 0.5) / f)[None, :]
mid = source[:, hand - f:hand] * (1.0 - w) + generated[:, hand - f:hand] * w
else:
mid = np.zeros((source.shape[0], 0), np.float32)
return np.concatenate([head, mid, generated[:, hand:]], axis=1)
hand = min(seam, generated.shape[1])
f = max(0, min(int(fade), source.shape[1], generated.shape[1] - hand))
if f:
w = ((np.arange(f, dtype=np.float32) + 0.5) / f)[None, :]
mid = generated[:, hand:hand + f] * (1.0 - w) + source[:, :f] * w
else:
mid = np.zeros((source.shape[0], 0), np.float32)
return np.concatenate([generated[:, :hand], mid, source[:, f:]], axis=1)
def fit_length(pcm: np.ndarray, samples: int) -> np.ndarray:
"""Pad with silence or cut to exactly ``samples``."""
samples = max(0, int(samples))
if pcm.shape[1] >= samples:
return np.ascontiguousarray(pcm[:, :samples])
return np.pad(pcm, ((0, 0), (0, samples - pcm.shape[1])))
# --------------------------------------------------------------------------
# Probing
# --------------------------------------------------------------------------
def probe(ffprobe: str | Path, path: str | Path) -> dict:
"""Codec, canvas, pix_fmt, frame rate, frame count (packets) and whether
there is an audio stream. Raises SpliceError on an unreadable file."""
r = subprocess.run(
[str(ffprobe), "-v", "error", "-count_packets", "-show_entries",
"stream=codec_type,codec_name,width,height,pix_fmt,avg_frame_rate,"
"r_frame_rate,nb_read_packets,start_time", "-of", "json", str(path)],
capture_output=True, text=True, errors="replace", timeout=120)
try:
streams = json.loads(r.stdout or "{}").get("streams") or []
except ValueError:
streams = []
v = next((s for s in streams if s.get("codec_type") == "video"), None)
if r.returncode != 0 or v is None:
raise SpliceError(f"cannot read the video stream of {Path(path).name}")
def _rate(s: str | None) -> float:
num, _, den = str(s or "0/1").partition("/")
try:
return float(num) / float(den or 1)
except (ValueError, ZeroDivisionError):
return 0.0
fps = _rate(v.get("avg_frame_rate")) or _rate(v.get("r_frame_rate"))
a = next((s for s in streams if s.get("codec_type") == "audio"), None)
def _start(s: dict | None) -> float:
try:
return float((s or {}).get("start_time") or 0.0)
except (TypeError, ValueError):
return 0.0
return {
# Where the audio starts on the VIDEO's clock (Codex 4.19.0): an
# imported clip whose sound starts later (or earlier) than its first
# frame must not have that offset dropped when it is decoded to PCM.
"audio_offset": (_start(a) - _start(v)) if a is not None else 0.0,
"codec": v.get("codec_name"), "width": int(v.get("width") or 0),
"height": int(v.get("height") or 0), "pix_fmt": v.get("pix_fmt"),
"fps": fps, "frames": int(v.get("nb_read_packets") or 0),
"has_audio": any(s.get("codec_type") == "audio" for s in streams),
}
def avcc(path: str | Path) -> bytes | None:
"""The H.264 decoder configuration record (SPS + PPS) of an mp4, or None.
Two streams can only be joined packet-for-packet when this is
byte-identical: the joined file carries ONE sample description, so a new
part whose parameter sets differ (another CRF, profile, colour tags, x264
build) would be decoded with the source's and break."""
try:
data = Path(path).read_bytes()
except OSError:
return None
i = data.find(b"avcC")
if i < 4:
return None
size = int.from_bytes(data[i - 4:i], "big")
if size < 8 or i - 4 + size > len(data):
return None
return data[i + 4:i - 4 + size]
# --------------------------------------------------------------------------
# Assembly
# --------------------------------------------------------------------------
def _read_pcm(run: Runner, ffmpeg: str, src: Path, raw: Path, label: str,
offset: float = 0.0) -> np.ndarray:
"""The audio as (channels, samples) on the VIDEO's timeline: ``offset``
seconds (audio start minus video start) are padded with silence when
positive and trimmed when negative, so sample 0 is the first frame's."""
run([ffmpeg, "-y", "-v", "error", "-i", str(src), "-map", "0:a:0", "-vn",
"-ac", str(CHANNELS), "-ar", str(SAMPLE_RATE), "-f", "f32le", str(raw)], label)
data = np.fromfile(raw, dtype="<f4")
data = data[: data.size - data.size % CHANNELS]
pcm = np.ascontiguousarray(data.reshape(-1, CHANNELS).T)
shift = int(round(float(offset or 0.0) * SAMPLE_RATE))
if shift > 0:
pcm = np.concatenate([np.zeros((CHANNELS, shift), dtype=pcm.dtype), pcm], axis=1)
elif shift < 0:
pcm = pcm[:, -shift:]
return np.ascontiguousarray(pcm)
def _write_master(run: Runner, ffmpeg: str, pcm: np.ndarray, raw: Path, master: Path) -> None:
np.ascontiguousarray(pcm.T, dtype="<f4").tofile(raw)
master.parent.mkdir(parents=True, exist_ok=True)
partial = master.with_name(master.name + ".partial")
run([ffmpeg, "-y", "-v", "error", "-f", "f32le", "-ar", str(SAMPLE_RATE),
"-ac", str(CHANNELS), "-i", str(raw), "-c:a", "flac", "-sample_fmt", "s16",
"-f", "flac", str(partial)], "Extend: audio master")
partial.replace(master)
def _master_matches(master: Path | None, seconds: float) -> bool:
"""A master is only trusted when it still describes the source: same
length to within a frame (a clip trimmed or replaced since must not get
the old soundtrack back)."""
if not master:
return False
try:
size = master.stat().st_size
except OSError:
return False
return size > 0 and abs(_flac_seconds(master) - seconds) <= 0.05
def _flac_seconds(path: Path) -> float:
"""Duration of a FLAC file from its STREAMINFO block (no ffprobe)."""
try:
with open(path, "rb") as fh:
head = fh.read(42)
if head[:4] != b"fLaC":
return -1.0
info = head[8:8 + 34]
sr = (info[10] << 12) | (info[11] << 4) | (info[12] >> 4)
total = ((info[13] & 0x0F) << 32) | int.from_bytes(info[14:18], "big")
return total / float(sr) if sr else -1.0
except OSError:
return -1.0
def splice(*, ffmpeg: str | Path, ffprobe: str | Path, source: Path, generated: Path,
generated_audio: Path | None, out: Path, layout: Layout, work: Path,
pix_fmt: str, crf: str, x264_args: list[str] | None = None,
generated_lossless: bool = True, source_master: Path | None = None,
master_out: Path | None = None, run: Runner | None = None) -> dict:
"""Write ``out`` = the source's own frames + the generated new frames,
with the audio spliced from lossless masters and encoded once.
``generated`` is the model's full output (source region + new), on the
delivered canvas; ``generated_lossless`` says whether its video is a
lossless intermediate (re-encode the new part once) or already the
delivery codec (H3's runner without ``--crf 0``). ``generated_audio`` is
a lossless wav of the same timeline when the engine leaves one (the
helper's vocoder output, the H3 runner's wav), else the audio is read
from ``generated``. Returns a report dict for the sidecar.
"""
run = run or _default_run
ffmpeg, ffprobe = str(ffmpeg), str(ffprobe)
x264 = list(x264_args or [])
work.mkdir(parents=True, exist_ok=True)
src = probe(ffprobe, source)
gen = probe(ffprobe, generated)
fps = gen["fps"] or src["fps"]
if not fps:
raise SpliceError("unknown frame rate")
if gen["frames"] < layout.new_end:
raise SpliceError(f"the model's output has {gen['frames']} frames, the "
f"splice needs {layout.new_end}")
report = {"version": 1, "source": str(source), "source_frames": layout.source_frames,
"new_frames": layout.new_frames, "seam_frame": layout.seam_frame,
"seam_seconds": round(layout.seam_frame / fps, 4),
"direction": "after" if layout.source_first else "before"}
same_canvas = (src["width"], src["height"]) == (gen["width"], gen["height"])
same_rate = abs((src["fps"] or 0) - fps) < 0.01
enc = ["-c:v", "libx264", "-pix_fmt", pix_fmt, "-crf", str(crf), *x264]
# ---- video --------------------------------------------------------------
video = work / "video.mp4"
mode = None
why = ""
if not (same_canvas and same_rate):
why = (f"source {src['width']}x{src['height']}@{src['fps']:.3f} differs from the "
f"render {gen['width']}x{gen['height']}@{fps:.3f}")
elif src["frames"] < layout.source_frames:
why = f"source has {src['frames']} frames, the model used {layout.source_frames}"
if why:
# Nothing of the source file can stand in for the model's input here:
# deliver the model's own frames (its source region was decoded once
# from the picked file), exactly the pre-#48 delivery for this round.
mode = "generated"
if generated_lossless:
run([ffmpeg, "-y", "-v", "error", "-i", str(generated), "-map", "0:v:0", "-an",
*enc, str(video)], "Extend: encode")
else:
run([ffmpeg, "-y", "-v", "error", "-i", str(generated), "-map", "0:v:0", "-an",
"-c:v", "copy", str(video)], "Extend: video")
else:
new_part = work / "new.mp4"
sel = f"select='between(n\\,{layout.new_start}\\,{layout.new_end - 1})',setpts=N/FRAME_RATE/TB"
run([ffmpeg, "-y", "-v", "error", "-i", str(generated), "-map", "0:v:0", "-an",
"-vf", sel, "-frames:v", str(layout.new_frames), *enc, str(new_part)],
"Extend: new frames")
copy_ok = (src["codec"] == "h264" and src["frames"] == layout.source_frames
and avcc(new_part) is not None and avcc(new_part) == avcc(source))
if copy_ok:
src_v = work / "source_video.mp4"
run([ffmpeg, "-y", "-v", "error", "-i", str(source), "-map", "0:v:0", "-an",
"-c", "copy", str(src_v)], "Extend: source video")
parts = [src_v, new_part] if layout.source_first else [new_part, src_v]
lst = work / "concat.txt"
lst.write_text("".join("file '" + str(p).replace("'", "'\\''") + "'\n"
for p in parts), encoding="utf-8")
try:
run([ffmpeg, "-y", "-v", "error", "-f", "concat", "-safe", "0",
"-i", str(lst), "-c", "copy", str(video)], "Extend: join")
got = probe(ffprobe, video)["frames"]
except Exception: # noqa: BLE001
got = -1
if got == layout.total_frames:
mode = "copy"
else:
why = f"stream-copy join came out at {got} frames, not {layout.total_frames}"
else:
why = ("source is not an H.264 stream the new part can continue "
"(codec, frame count or encoder parameter sets differ)")
if mode is None:
# Decode the picked file ONCE (never a re-decoded copy of it) and
# encode it together with the new frames: one generation.
a, b = layout.new_start, layout.new_end
chains = (f"[0:v]trim=end_frame={layout.source_frames},setpts=PTS-STARTPTS,"
f"setsar=1,format={pix_fmt}[s];"
f"[1:v]trim=start_frame={a}:end_frame={b},setpts=PTS-STARTPTS,"
f"setsar=1,format={pix_fmt}[n];")
chains += "[s][n]concat=n=2:v=1:a=0[v]" if layout.source_first \
else "[n][s]concat=n=2:v=1:a=0[v]"
run([ffmpeg, "-y", "-v", "error", "-i", str(source), "-i", str(generated),
"-filter_complex", chains, "-map", "[v]", "-an", *enc, str(video)],
"Extend: re-encode once")
mode = "reencode"
report["video"] = mode
if why:
report["video_note"] = why
total = probe(ffprobe, video)["frames"]
if mode != "generated" and total != layout.total_frames:
raise SpliceError(f"spliced video has {total} frames, expected {layout.total_frames}")
report["frames"] = total
# ---- audio --------------------------------------------------------------
seconds_total = total / fps
n_total = int(round(seconds_total * SAMPLE_RATE))
g_src = generated_audio if (generated_audio and Path(generated_audio).is_file()) else generated
gen_pcm = _read_pcm(run, ffmpeg, Path(g_src), work / "gen.f32", "Extend: generated audio",
offset=gen.get("audio_offset", 0.0) if g_src == generated else 0.0)
# The audio is spliced whatever happened to the picture: it is a time-domain
# signal, and the source region spans the same seconds either way.
src_seconds = src["frames"] / (src["fps"] or fps)
if _master_matches(source_master, src_seconds):
src_pcm = _read_pcm(run, ffmpeg, Path(source_master), work / "src.f32",
"Extend: source audio master")
report["audio"] = "master"
elif src["has_audio"]:
src_pcm = _read_pcm(run, ffmpeg, source, work / "src.f32", "Extend: source audio",
offset=src.get("audio_offset", 0.0))
report["audio"] = "decoded"
else:
src_pcm = np.zeros((CHANNELS, int(round(src_seconds * SAMPLE_RATE))), np.float32)
report["audio"] = "silent-source"
# Only the stretch the model used: a source longer than its 1+8k frames
# loses the same tail in the audio as in the picture.
src_pcm = src_pcm[:, :int(round(layout.source_frames / fps * SAMPLE_RATE))]
fade = int(round(FADE_SECONDS * SAMPLE_RATE))
seam = int(round(layout.seam_frame / fps * SAMPLE_RATE))
pcm = crossfade_splice(src_pcm, gen_pcm, seam, fade, layout.source_first)
report["fade_ms"] = round(1000 * fade / SAMPLE_RATE, 1)
pcm = fit_length(pcm, n_total)
master = master_out or (work / "master.flac")
_write_master(run, ffmpeg, pcm, work / "master.f32", master)
report["audio_master"] = str(master)
# ---- mux ----------------------------------------------------------------
partial = out.with_name(out.name + ".partial")
run([ffmpeg, "-y", "-v", "error", "-i", str(video), "-i", str(master),
"-map", "0:v:0", "-map", "1:a:0", "-c:v", "copy", "-c:a", "aac",
"-b:a", AUDIO_BITRATE, "-movflags", "+faststart", "-f", "mp4", str(partial)],
"Extend: mux")
final = probe(ffprobe, partial)
if final["frames"] != total or not final["has_audio"]:
raise SpliceError(f"muxed file has {final['frames']} frames / audio "
f"{final['has_audio']}, expected {total} + audio")
partial.replace(out)
return report
def _default_run(cmd: list, label: str) -> None:
r = subprocess.run([str(c) for c in cmd], capture_output=True, text=True,
errors="replace")
if r.returncode != 0:
raise SpliceError(f"{label} failed: {(r.stderr or r.stdout).strip()[-300:]}")