Free, open-source AI video generator for Mac. Run LTX-2.5 and MiniMax H3 (Hailuo) locally on Apple Silicon: text-to-video and image-to-video with sound, lip-sync, character LoRA training, songs from lyrics (YuE2) and image editing (Qwen-Image-Edit). One-click install via Pinokio. Offline: no cloud, no API key, no NVIDIA GPU.
Python ai-filmmakingai-videoai-video-generatorapple-siliconhailuoimage-to-videolip-synclocal-ailora-trainingltx-2ltx-2-5ltx-videomacosminimax-h3mlxmusic-generationoffline-aipinokiotext-to-videovideo-generation
#!/usr/bin/env python3
"""LoRA inference for the YuE2 MLX engine — Phosphene's own code, not the engine's.
The vendored engine (`lyra`, from vanch007/mlx-Yue) has no adapter support and
is installed from a pinned SHA by `scripts/pinokio/music_sync.sh`, so anything
written inside `yue2-mlx/` is lost on the next sync. This module therefore
lives in OUR repo and works on the models `lyra` hands back, from the runner.
WHAT AN ADAPTER IS HERE
-----------------------
`y = W x + strength · B (A x)`, with `lora_A: [rank, in]` and `lora_B: [out,
rank]`, in BF16, applied at inference and never merged into the base weights.
Not merging is the point: the AR model runs 8-bit by default in Phosphene, and
a merge would either force BF16 (5 GB more resident) or dequantise-requantise
a 3B model per song. A wrapper costs two small matmuls per projection.
HOW IT ATTACHES WITHOUT RENAMING ANYTHING
-----------------------------------------
`nar.load_nar()` validates the shared AR model by walking
`tree_flatten(ar_model.parameters())` and demanding the exact upstream names
(`model.layers.0.self_attn.q_proj.weight`, ...). A wrapper module holding the
original linear as a child would rename every one of them to `...q_proj.base.
weight` and the acoustic model would refuse to load. So the projection object
is kept and its CLASS is swapped for a generated subclass that adds the delta
after the base call; the adapter matrices hang off the instance through
`object.__setattr__`, the way `lyra` itself keeps runtime constants off the
parameter tree. The parameter tree comes out byte-identical, quantised bases
work unchanged, and removal is one `__class__` assignment back.
MODES (Maestro's semantics, replicated)
---------------------------------------
* `joint` — the AR and NAR deltas are ordinary additive LoRAs. The decoder
I/O projections are NOT touched, even when the file carries them
(`nar_lora_joint_v9` does; we ignore them and say so).
* `separate`— an artist bundle whose NAR file is a complete decoder companion:
the deltas still add, and `vae2llm` / `llm2vae` are REPLACED by a
convex blend of every separate companion, weighted by strength.
Summing them would double the shared decoder adaptation.
Key spellings accepted: the Mothersuperior `layers.{i}...lora_A` layout, an
optional `model.` prefix, PEFT / AI-Toolkit `base_model.model.` prefixes,
`.lora_A.weight` / `.lora_A.default.weight` spellings, kohya `lora_down` /
`lora_up`, and Maestro's own `.A` / `.B`. A PEFT `alpha` scalar, when present,
is folded into B as `alpha / rank` (the trap that flattened the first H3 LoRA
imports); files without one scale by strength alone, as Mothersuperior's do.
"""
from __future__ import annotations
from dataclasses import dataclass, field
import json
from pathlib import Path
import re
import sys
import mlx.core as mx
# The pack layout (which files, which revisions, how a pick is spelled) is
# STDLIB-ONLY and lives beside the fetcher, because the panel imports it at
# boot to draw the picker and must not pull MLX onto that path. One definition,
# two processes.
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from scripts.pinokio.music_lora_fetch import ( # noqa: E402
INSTRUMENTAL_ADAPTER, JOINT_NAR_ADAPTER, MAX_STRENGTH, USER_SUBDIR,
pack_adapters, parse_field, parse_picks, parse_spec, resolve_in_pack,
sha256_of,
validate_strength,
)
__all__ = [
"Adapter", "LoRARuntime", "apply_adapters", "branch_of", "check_against",
"describe_stack", "instrumental_lyrics", "instrumental_recipe", "is_adapted",
"linear_shape", "load_stack", "normalize_key", "read_adapter",
"remove_adapters", "style_with_triggers",
"INSTRUMENTAL_ADAPTER", "JOINT_NAR_ADAPTER", "MAX_STRENGTH", "USER_SUBDIR",
"pack_adapters", "parse_field", "parse_picks", "parse_spec",
"resolve_in_pack", "sha256_of",
"validate_strength",
]
#: One rank ceiling, same as Maestro's. Anything outside this is not a YuE2 LoRA.
MAX_RANK = 128
#: Which attention / MLP container each branch keeps its projections in.
BRANCH_GROUPS = {"ar": ("self_attn", "mlp"), "nar": ("nar_self_attn", "nar_mlp")}
ATTN_PROJECTIONS = ("q_proj", "k_proj", "v_proj", "o_proj")
MLP_PROJECTIONS = ("gate_proj", "up_proj", "down_proj")
#: The acoustic decoder's input/output projections — full weights, not deltas.
IO_MODULES = ("vae2llm", "llm2vae")
MODES = ("joint", "separate")
_ADAPTER_ATTR = "_phosphene_lora"
_ORIGINAL_CLASS_ATTR = "_phosphene_lora_base_class"
_lora_classes: dict[type, type] = {}
# ---------------------------------------------------------------- key mapping
_PREFIXES = ("base_model.model.model.", "base_model.model.", "base_model.",
"transformer.", "model.")
# Longest first: `.lora_A.default.weight` must not be eaten by `.lora_A.weight`.
_SUFFIXES = (
(".lora_A.default.weight", ".lora_A"), (".lora_B.default.weight", ".lora_B"),
(".lora_A.weight", ".lora_A"), (".lora_B.weight", ".lora_B"),
(".lora_down.weight", ".lora_A"), (".lora_up.weight", ".lora_B"),
(".lora_down", ".lora_A"), (".lora_up", ".lora_B"),
(".A", ".lora_A"), (".B", ".lora_B"),
)
_TARGET_RE = re.compile(
r"^layers\.(\d+)\.(self_attn|mlp|nar_self_attn|nar_mlp)\."
r"(q_proj|k_proj|v_proj|o_proj|gate_proj|up_proj|down_proj)$")
def normalize_key(key: str) -> str:
"""One spelling for the half-dozen an adapter file may arrive in."""
key = str(key)
for prefix in _PREFIXES:
if key.startswith(prefix):
key = key[len(prefix):]
break
for suffix, canonical in _SUFFIXES:
if key.endswith(suffix):
return key[: -len(suffix)] + canonical
return key
def _describe(name: str) -> tuple[int, str, str] | None:
match = _TARGET_RE.match(name)
if match is None:
return None
return int(match.group(1)), match.group(2), match.group(3)
def branch_of(names) -> str:
"""`nar` when the file targets the acoustic branch, `ar` otherwise."""
for name in names:
described = _describe(name.rsplit(".", 1)[0]) if name.endswith((".lora_A", ".lora_B")) else None
if described is not None and described[1].startswith("nar_"):
return "nar"
return "ar"
# ------------------------------------------------------------------- loading
@dataclass
class Adapter:
"""One adapter file, read and checked, ready to hang on a model."""
path: Path
branch: str
strength: float
mode: str
rank: int
deltas: dict[str, tuple[mx.array, mx.array]]
io: dict[str, mx.array] = field(default_factory=dict)
trigger: str = ""
sha256: str = ""
ignored_io: bool = False
@property
def name(self) -> str:
return self.path.name
def summary(self) -> dict:
"""What the sidecar records: enough to reproduce, nothing heavy."""
return {"file": self.name, "path": str(self.path), "branch": self.branch,
"strength": round(float(self.strength), 4), "mode": self.mode,
"rank": self.rank, "targets": len(self.deltas),
"decoder_io": bool(self.io), "decoder_io_ignored": self.ignored_io,
"trigger": self.trigger, "sha256": self.sha256}
def read_adapter(path, *, strength: float = 1.0, mode: str = "joint",
trigger: str = "", with_sha: bool = False) -> Adapter:
"""Read one `.safetensors` adapter into BF16 deltas, or say exactly why not."""
path = Path(path).expanduser()
if not path.is_file():
raise FileNotFoundError(f"No LoRA at {path}")
if path.suffix.lower() != ".safetensors":
raise ValueError(f"{path.name}: a YuE2 LoRA is a .safetensors file")
if mode not in MODES:
raise ValueError(f"Unsupported LoRA mode {mode!r}; use {' or '.join(MODES)}")
strength = validate_strength(strength)
raw = mx.load(str(path))
if not isinstance(raw, dict) or not raw:
raise ValueError(f"{path.name}: no named tensors")
tensors: dict[str, mx.array] = {}
for key, value in raw.items():
canonical = normalize_key(key)
if canonical in tensors:
raise ValueError(f"{path.name}: two tensors map to {canonical!r}")
tensors[canonical] = value
branch = branch_of(tensors)
pairs: dict[str, list] = {}
alphas: dict[str, float] = {}
io: dict[str, mx.array] = {}
unexpected: list[str] = []
for name, value in tensors.items():
stem, _, leaf = name.rpartition(".")
if leaf in ("lora_A", "lora_B") and _describe(stem) is not None:
pairs.setdefault(stem, [None, None])[0 if leaf == "lora_A" else 1] = value
elif leaf == "alpha" and _describe(stem) is not None:
alphas[stem] = float(value.reshape(-1)[0].item())
elif stem in IO_MODULES and leaf in ("weight", "bias"):
io[name] = value
else:
unexpected.append(name)
if unexpected:
raise ValueError(f"{path.name}: unsupported tensor targets: "
f"{', '.join(sorted(unexpected)[:5])}")
if not pairs:
raise ValueError(f"{path.name}: no LoRA matrices for the AR or acoustic branch")
ranks: set[int] = set()
deltas: dict[str, tuple[mx.array, mx.array]] = {}
for target in sorted(pairs):
a, b = pairs[target]
if a is None or b is None:
raise ValueError(f"{path.name}: {target} is missing its "
f"{'lora_A' if a is None else 'lora_B'} half")
if a.ndim != 2 or b.ndim != 2 or b.shape[1] != a.shape[0]:
raise ValueError(f"{path.name}: {target} is not a [rank,in]/[out,rank] pair")
rank = int(a.shape[0])
if not 1 <= rank <= MAX_RANK:
raise ValueError(f"{path.name}: rank {rank} is outside 1–{MAX_RANK}")
if not (mx.issubdtype(a.dtype, mx.floating) and mx.issubdtype(b.dtype, mx.floating)):
raise ValueError(f"{path.name}: {target} is not floating point")
ranks.add(rank)
a, b = a.astype(mx.bfloat16), b.astype(mx.bfloat16)
if target in alphas:
# PEFT / AI-Toolkit exports carry the scale separately. Folding it
# here keeps "strength 1.0" meaning the adapter as it was trained.
b = (b.astype(mx.float32) * (alphas[target] / rank)).astype(mx.bfloat16)
deltas[target] = (a, b)
if len(ranks) != 1:
raise ValueError(f"{path.name}: one rank per adapter, found {sorted(ranks)}")
for name, value in io.items():
io[name] = value.astype(mx.bfloat16)
if io and set(io) != {f"{m}.{k}" for m in IO_MODULES for k in ("weight", "bias")}:
raise ValueError(f"{path.name}: an incomplete decoder companion "
f"({', '.join(sorted(io))})")
if branch == "ar" and io:
raise ValueError(f"{path.name}: decoder projections belong to an acoustic adapter")
arrays = [x for pair in deltas.values() for x in pair] + list(io.values())
mx.eval(arrays)
for array in arrays:
if not bool(mx.isfinite(array).all()):
raise ValueError(f"{path.name}: adapter weights contain non-finite values")
ignored = bool(io) and mode == "joint"
return Adapter(path=path, branch=branch, strength=strength, mode=mode,
rank=ranks.pop(), deltas=deltas, io={} if ignored else io,
trigger=str(trigger or "").strip(),
sha256=sha256_of(path) if with_sha else "",
ignored_io=ignored)
# ------------------------------------------------------------------ applying
def _lora_class(base: type) -> type:
"""A subclass of the projection's own class that adds the deltas."""
if base in _lora_classes:
return _lora_classes[base]
class _LoRAProjection(base): # type: ignore[misc]
def __call__(self, x, *args, **kwargs):
out = super().__call__(x, *args, **kwargs)
for a, b, scale in object.__getattribute__(self, _ADAPTER_ATTR):
delta = (x.astype(a.dtype) @ a.T) @ b.T
out = out + delta.astype(out.dtype) * scale
return out
_LoRAProjection.__name__ = f"LoRA{base.__name__}"
_LoRAProjection.__qualname__ = _LoRAProjection.__name__
_lora_classes[base] = _LoRAProjection
return _LoRAProjection
def _projection_slots(model, branch: str) -> dict[str, tuple[object, str]]:
"""`layers.{i}.{group}.{proj}` → (owning module, attribute name)."""
if branch not in BRANCH_GROUPS:
raise ValueError(f"branch must be one of {', '.join(BRANCH_GROUPS)}")
attn_name, mlp_name = BRANCH_GROUPS[branch]
backbone = getattr(model, "model", None)
layers = getattr(backbone, "layers", None)
if layers is None:
raise TypeError("This model has no transformer layers to adapt")
slots: dict[str, tuple[object, str]] = {}
for index, layer in enumerate(layers):
for group, projections in ((attn_name, ATTN_PROJECTIONS),
(mlp_name, MLP_PROJECTIONS)):
container = getattr(layer, group, None)
if container is None:
continue
for projection in projections:
if hasattr(container, projection):
slots[f"layers.{index}.{group}.{projection}"] = (container, projection)
return slots
def linear_shape(module) -> tuple[int, int]:
"""(out_features, in_features) for a plain or a quantised linear."""
weight = module.weight
out = int(weight.shape[0])
scales = module.get("scales") if hasattr(module, "get") else None
if scales is not None:
return out, int(scales.shape[1]) * int(module.group_size)
return out, int(weight.shape[1])
def check_against(adapter: Adapter, model) -> list[str]:
"""Shape-check every delta against the model that will run it."""
slots = _projection_slots(model, adapter.branch)
problems: list[str] = []
for target, (a, b) in adapter.deltas.items():
slot = slots.get(target)
if slot is None:
problems.append(f"{target} is not a projection of this model")
continue
out_features, in_features = linear_shape(getattr(slot[0], slot[1]))
if int(a.shape[1]) != in_features or int(b.shape[0]) != out_features:
problems.append(
f"{target} expects [rank,{in_features}]/[{out_features},rank], "
f"got {tuple(a.shape)}/{tuple(b.shape)}")
if adapter.io:
for name, value in adapter.io.items():
module = getattr(model, name.split(".")[0], None)
reference = None if module is None else module.get(name.split(".")[1])
if reference is None:
problems.append(f"{name} has no counterpart on this model")
elif tuple(value.shape) != tuple(reference.shape):
problems.append(f"{name} is {tuple(value.shape)}, "
f"expected {tuple(reference.shape)}")
return problems
def apply_adapters(model, branch: str, adapters) -> int:
"""Hang every adapter for `branch` on `model`. Returns projections wrapped.
Idempotent per model: applying twice replaces the stack rather than
doubling it, so a pipeline that reloads a model mid-song cannot drift.
"""
adapters = [a for a in adapters if a.branch == branch and a.strength > 0]
remove_adapters(model)
if not adapters:
return 0
for adapter in adapters:
problems = check_against(adapter, model)
if problems:
raise ValueError(f"{adapter.name}: " + "; ".join(problems[:3]))
stacks: dict[str, list] = {}
for adapter in adapters:
for target, (a, b) in adapter.deltas.items():
stacks.setdefault(target, []).append((a, b, float(adapter.strength)))
slots = _projection_slots(model, branch)
for target, stack in stacks.items():
container, attribute = slots[target]
module = getattr(container, attribute)
object.__setattr__(module, _ORIGINAL_CLASS_ATTR, type(module))
object.__setattr__(module, _ADAPTER_ATTR, tuple(stack))
module.__class__ = _lora_class(type(module))
# A complete decoder companion is not a delta: the separate bundles blend
# their I/O projections by relative strength, and replace the base ones.
companions = [a for a in adapters if a.mode == "separate" and a.io]
if companions:
total = sum(a.strength for a in companions)
blended: dict[str, mx.array] = {}
for adapter in companions:
share = (adapter.strength / total) if total else (1.0 / len(companions))
for name, value in adapter.io.items():
scaled = value.astype(mx.float32) * share
blended[name] = scaled if name not in blended else blended[name] + scaled
for name, value in blended.items():
module_name, leaf = name.split(".")
module = getattr(model, module_name)
original = getattr(model, "_phosphene_lora_io", None)
if original is None:
original = {}
object.__setattr__(model, "_phosphene_lora_io", original)
original.setdefault(name, module[leaf])
module[leaf] = value.astype(mx.bfloat16)
mx.eval([getattr(model, m)[k] for m in IO_MODULES for k in ("weight", "bias")])
return len(stacks)
def remove_adapters(model) -> int:
"""Put every adapted projection (and any replaced decoder I/O) back."""
restored = 0
for branch in BRANCH_GROUPS:
try:
slots = _projection_slots(model, branch)
except TypeError:
return restored
for container, attribute in slots.values():
module = getattr(container, attribute)
original = getattr(module, _ORIGINAL_CLASS_ATTR, None)
if original is None:
continue
module.__class__ = original
object.__delattr__(module, _ORIGINAL_CLASS_ATTR)
object.__delattr__(module, _ADAPTER_ATTR)
restored += 1
saved = getattr(model, "_phosphene_lora_io", None)
if saved:
for name, value in saved.items():
module_name, leaf = name.split(".")
getattr(model, module_name)[leaf] = value
object.__delattr__(model, "_phosphene_lora_io")
return restored
def is_adapted(model) -> bool:
try:
slots = _projection_slots(model, "ar")
slots.update(_projection_slots(model, "nar"))
except TypeError:
return False
return any(hasattr(getattr(c, a), _ORIGINAL_CLASS_ATTR) for c, a in slots.values())
# ------------------------------------------------------- pipeline attachment
class LoRARuntime:
"""Keeps a pipeline's models adapted across its own lazy reloads.
`YuE2Pipeline._load_model()` builds the AR model for planning, a second
BF16 AR for acoustic conditioning, and the acoustic model itself — each on
first use, and the 8-bit path drops and rebuilds them mid-song. Wrapping
that one method is the only hook that sees every model the song will use.
"""
def __init__(self, adapters):
self.adapters = list(adapters)
self._pipeline = None
self._original = None
self._applied: dict[int, int] = {}
@property
def active(self) -> bool:
return bool(self.adapters)
def summary(self) -> list[dict]:
return [a.summary() for a in self.adapters]
def attach(self, pipeline):
if not self.adapters:
return self
if self._pipeline is not None:
raise RuntimeError("This LoRA runtime is already attached")
self._pipeline, self._original = pipeline, pipeline._load_model
def load_model(for_nar=False):
model = self._original(for_nar=for_nar)
self.refresh()
return model
pipeline._load_model = load_model
return self
def refresh(self) -> None:
"""Apply the stack to whichever models the pipeline holds right now."""
pipeline = self._pipeline
if pipeline is None:
return
for attribute, branch in (("_ar", "ar"), ("_bf16_ar", "ar"), ("_nar", "nar")):
model = getattr(pipeline, attribute, None)
if model is None:
continue
token = id(model)
if self._applied.get(token) == len(self.adapters) and is_adapted(model):
continue
count = apply_adapters(model, branch, self.adapters)
if count:
self._applied[token] = len(self.adapters)
def detach(self) -> None:
pipeline, self._pipeline = self._pipeline, None
if pipeline is None:
return
pipeline._load_model = self._original
self._original = None
for attribute in ("_ar", "_bf16_ar", "_nar"):
model = getattr(pipeline, attribute, None)
if model is not None:
remove_adapters(model)
self._applied.clear()
def __enter__(self):
return self
def __exit__(self, *exc):
self.detach()
return False
# --------------------------------------------------------- instrumental mode
#: Maestro's own prompt. A bare `[instrumental]` lets the model pick sections.
INSTRUMENTAL_PROMPT = "[instrumental]"
INSTRUMENTAL_COT = "full"
_SECTION_RE = re.compile(
r"\[(intro|verse|pre-chorus|chorus|bridge|outro)(?: (\d+):([0-5]\d)-(\d+):([0-5]\d))?\]",
re.IGNORECASE)
def instrumental_lyrics(lyrics) -> str:
"""Bare section tags survive; anything else becomes `[instrumental]`.
Sung prose fed to this adapter is what produces a vocal-like track, so the
three caption forms Maestro accepts are the only ones that pass through.
"""
lines = str(lyrics or "").strip().splitlines()
plan: list[str] = []
previous_end = 0
for line in lines:
if not line.strip():
continue
match = _SECTION_RE.fullmatch(line.strip())
if match is None:
return INSTRUMENTAL_PROMPT
tag, start_min, start_sec, end_min, end_sec = match.groups()
if start_min is not None:
start = int(start_min) * 60 + int(start_sec)
end = int(end_min) * 60 + int(end_sec)
if start < previous_end or end <= start:
return INSTRUMENTAL_PROMPT
previous_end = end
plan.append(line.strip().lower())
return "\n".join(plan) or INSTRUMENTAL_PROMPT
def instrumental_recipe(lora_root, lyrics="") -> dict:
"""Maestro's recipe: the AR adapter at 1.0, full score planning, stock decoder.
Returns the pieces the runner applies; `adapter` is None when the pack is
not downloaded, and the caller falls back to the prompt-only instrumental
that shipped in v4.16 rather than failing a queued song.
"""
adapter = None
if lora_root is not None:
candidate = Path(lora_root).expanduser() / INSTRUMENTAL_ADAPTER
if candidate.is_file():
adapter = candidate
return {"adapter": adapter, "strength": 1.0, "cot": INSTRUMENTAL_COT,
"lyrics": instrumental_lyrics(lyrics) if adapter else None,
"decoder": "stock", "pauses_other_loras": True}
def style_with_triggers(style: str, adapters) -> str:
"""Add each adapter's training trigger to the style prompt once."""
style = str(style or "")
folded = style.casefold()
missing: list[str] = []
for adapter in adapters:
trigger = (adapter.trigger or "").strip()
if trigger and trigger.casefold() not in folded and \
trigger.casefold() not in {m.casefold() for m in missing}:
missing.append(trigger)
if not missing:
return style
return ", ".join([*missing, style] if style else missing)
def load_stack(specs, *, mode="joint", triggers=(), with_sha=True) -> list[Adapter]:
"""`[(path, strength), ...]` → checked adapters, in the order given."""
triggers = list(triggers)
adapters = []
for index, (path, strength) in enumerate(specs):
adapters.append(read_adapter(
path, strength=strength, mode=mode, with_sha=with_sha,
trigger=triggers[index] if index < len(triggers) else ""))
return adapters
def describe_stack(adapters, *, instrumental=None) -> dict:
"""The sidecar block. Says what ran, at what strength, from which file."""
return {"adapters": [a.summary() for a in adapters],
"modes": sorted({a.mode for a in adapters}),
"branches": sorted({a.branch for a in adapters}),
"triggers": [a.trigger for a in adapters if a.trigger],
"instrumental": instrumental}
if __name__ == "__main__": # pragma: no cover
import argparse
parser = argparse.ArgumentParser(description="Inspect a YuE2 LoRA file.")
parser.add_argument("path", type=Path, nargs="+")
parsed = parser.parse_args()
for one in parsed.path:
adapter = read_adapter(one, with_sha=True)
print(json.dumps(adapter.summary(), indent=2))