Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python

fix: render.py merges both LoRAs into the weights at once (their deltas summed in float32, rounded into bf16 a single time, nothing running on top: sharper, a clean background, faster); a causal stream past the training clip renders every later chunk as a trained one was (prompt, first frame and audio shifted forward by whole 5-latent steps)

Enderfgab7e7a3656b12
diff --git a/scripts/render.py b/scripts/render.py
index d2de7fc..a48e2a1 100644
--- a/scripts/render.py
+++ b/scripts/render.py
@@ -138,7 +138,7 @@ def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool
import torch
from solarwm.backends.minimax_h3.artifacts import extend_silence, load_silence_latents
- from solarwm.backends.minimax_h3.lora import inject_h3_lora, merge_lora_delta
+ from solarwm.backends.minimax_h3.lora import merge_lora_delta
from solarwm.backends.minimax_h3.observations import observation_profile_from_train_config
from solarwm.backends.minimax_h3.optional import load_conditioners, load_transformer
from solarwm.backends.minimax_h3.stage0p5 import H3Stage0p5Core
@@ -152,17 +152,20 @@ def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool
"adapter": {"rank": RANK, "alpha": ALPHA, "dropout": 0.0},
}
modules = load_transformer(model_cfg, device=torch.device(device))
- # One LoRA is merged into the weights, the other runs on top, as each pair was trained: the bidirectional four-step
- # LoRA under the control LoRA, the causal control LoRA under its four-step student. Merging the other way rounds
- # the top LoRA into bf16 weights and the picture goes hazy.
- under, top = (control, fewstep) if causal else (fewstep, control)
- pairs, alpha = lora_pairs(under)
- rank = int(next(v for k, v in pairs.items() if k.endswith("lora_A.weight")).shape[0])
- merged = merge_lora_delta(modules.transformer, pairs, alpha=alpha, rank=rank, scale=1.0)
- print(f"{Path(under).name} merged into {merged} weights", flush=True)
- wrapped, lora = inject_h3_lora(modules.transformer, model_cfg["adapter"], base_identity={})
- pairs, _alpha = lora_pairs(top)
- lora.load_state_dict({f"base_model.model.{k}": v for k, v in pairs.items()}, broadcast=False)
+ # Both LoRAs are merged into the weights at once: their deltas summed in float32 and rounded into bf16 a single
+ # time, nothing left running on top (a LoRA computed beside bf16 weights, or merged one at a time, softens the
+ # picture). Each pair's up projection carries its own alpha/rank, so the stack adds exactly B1 A1 + B2 A2.
+ stacked: dict = {}
+ for path in (fewstep, control):
+ pairs, alpha = lora_pairs(path)
+ rank = int(next(v for k, v in pairs.items() if k.endswith("lora_A.weight")).shape[0])
+ for key, value in pairs.items():
+ value = value if key.endswith("lora_A.weight") else value.float() * (alpha / rank)
+ stacked[key] = torch.cat((stacked[key], value.float()), dim=0 if key.endswith("lora_A.weight") else 1) \
+ if key in stacked else value.float()
+ merged = merge_lora_delta(modules.transformer, stacked, alpha=None, rank=None, scale=1.0)
+ print(f"both LoRAs merged into {merged} weights", flush=True)
+ wrapped = modules.transformer
conditioners = load_conditioners(
model_cfg, device=torch.device(device), qwen=False, video_vae=True, audio_vae=False, schedulers=False
)
diff --git a/src/solarwm/backends/minimax_h3/omni_causal.py b/src/solarwm/backends/minimax_h3/omni_causal.py
index 6eee7ed..208d01b 100644
--- a/src/solarwm/backends/minimax_h3/omni_causal.py
+++ b/src/solarwm/backends/minimax_h3/omni_causal.py
@@ -32,6 +32,7 @@ from .layout import AUDIO_TAG, VIDEO_TAG, H3PackedLayout
STATIC_TIME = 1.0 # text rows: fixed, so a stream can keep them
CLEAN_LEVEL = 0.999 # given frames and observations, as the first frame always was
+TRAIN_LATENTS = 47 # the training clip, 158 frames: a longer stream renders every later chunk as one of these was trained
# --- attention plan -------------------------------------------------------------------------------------------------
@@ -525,6 +526,14 @@ class H3OmniCausalMixin:
window = int(window or self.causal.get("window", 6))
prompt, layout, observation_rows, rows, (frames, grid) = self._document(batch, generator)
audio = self._audio(frames, generator)
+ # Past the training clip nothing extrapolates: a stream longer than the training clip hears the training
+ # clip's length of audio, and a chunk beyond the training clip's last full chunk is rendered as that chunk was
+ # trained - the prompt, first frame and audio shifted forward with it by whole 5-latent steps.
+ shifts, audio_keep = self._frozen_positions(layout, rows, frames)
+ if any(shifts) and cache_structure and rows.structure:
+ raise ValueError("a stream past the training length recomputes its structure window (cache_structure=False)")
+ audio = audio.index_select(1, torch.from_numpy(audio_keep - rows.audio[0]).to(self.device))
+ audio_keep = audio_keep.astype(np.int64)
cache = H3ChunkKVCache(window=window)
source = layout.source
n_cond = int(source.num_condition_video_rows)
@@ -541,12 +550,15 @@ class H3OmniCausalMixin:
static_obs = [s for s in rows.static if s[0] >= rows.text[1]] # the first frame (and other static pictures)
- def sub_document(target: tuple[int, int], extra: list = ()) -> tuple[Any, np.ndarray, list, tuple[int, int], Any]:
+ def sub_document(
+ target: tuple[int, int], extra: list = (), shift: float = 0.0
+ ) -> tuple[Any, np.ndarray, list, tuple[int, int], Any, tuple[int, int]]:
"""The rows a stream recomputes for one forward: the static rows, the audio, ``extra`` (the structure window
- when it is not cached) and ``target`` (a structure or target chunk) - everything else comes from the cache."""
+ when it is not cached) and ``target`` (a structure or target chunk) - everything else comes from the cache.
+ The static rows and the audio sit ``shift`` later in time; the audio's rows come back as the last item."""
- spans = sorted([*rows.static, rows.audio, *extra, target])
- keep = np.concatenate([np.arange(a, b) for a, b in spans])
+ spans = sorted([*rows.static, *extra, target])
+ keep = np.sort(np.concatenate([*(np.arange(a, b) for a, b in spans), audio_keep]))
where = np.full(rows.sequence_length, -1, dtype=np.int64)
where[keep] = np.arange(keep.size)
kept_video = np.asarray([r for r in video_doc_rows[:n_cond] if where[r] >= 0], dtype=np.int64)
@@ -554,13 +566,20 @@ class H3OmniCausalMixin:
times = np.full(int(keep.size), CLEAN_LEVEL, dtype=np.float32)
text = _remap(rows.text, where)
times[text[0] : text[1]] = STATIC_TIME
- return select_rows(layout, keep), where, pick, _remap(target, where), times
+ arrays = select_rows(layout, keep)
+ heard = (int(where[audio_keep[0]]), int(where[audio_keep[-1]]) + 1)
+ if shift:
+ positions = arrays["position_ids"].clone()
+ for a, b in [*(_remap(s, where) for s in rows.static), heard]:
+ positions[a:b, 0] += shift
+ arrays["position_ids"] = positions
+ return arrays, where, pick, _remap(target, where), times, heard
def plan_for(
- where: np.ndarray, mine: tuple[int, int], size: int, keys: list, cached: tuple, commit: Any, extra: list = ()
+ where: np.ndarray, mine: tuple[int, int], size: int, keys: list, cached: tuple, commit: Any, extra: list = (),
+ *, audio: tuple[int, int],
) -> H3SegmentPlan:
static = [_remap(s, where) for s in rows.static]
- audio = _remap(rows.audio, where)
pieces = [(span, static) for span in static] + [(audio, [*static, audio])]
# structure chunks recomputed in this forward see the static rows and themselves, as always
pieces += [(_remap(e, where), [*static, _remap(e, where)]) for e in extra]
@@ -578,16 +597,16 @@ class H3OmniCausalMixin:
extra = [] if cache_structure or not rows.structure else [rows.structure[j] for j in _window(c, window)]
if rows.structure and cache_structure:
# The chunk's structure, once: it sees the static rows and itself, so its keys and values never change.
- arrays, where, pick, mine, times = sub_document(rows.structure[c])
+ arrays, where, pick, mine, times, heard = sub_document(rows.structure[c])
with torch.no_grad():
self._run(
video_rows=observation_rows.index_select(1, pick), audio_rows=audio, prompt=prompt,
- arrays=arrays, times=times, plan=plan_for(where, mine, len(times), [], (), ("structure", c)),
- model=model,
+ arrays=arrays, times=times,
+ plan=plan_for(where, mine, len(times), [], (), ("structure", c), audio=heard), model=model,
)
- arrays, where, pick, mine, times = sub_document(rows.target[c], extra)
+ arrays, where, pick, mine, times, heard = sub_document(rows.target[c], extra, shifts[c])
condition = observation_rows.index_select(1, pick)
- audio_span = [_remap(rows.audio, where), *(_remap(e, where) for e in extra)]
+ audio_span = [heard, *(_remap(e, where) for e in extra)]
shape = (1, 24, f1 - f0, *grid)
current = draw(shape)
for index in range(evaluations):
@@ -596,7 +615,7 @@ class H3OmniCausalMixin:
rows_out = self._run(
video_rows=torch.cat((condition, patchify_video(current)), dim=1), audio_rows=audio,
prompt=prompt, arrays=arrays, times=times.copy(),
- plan=plan_for(where, mine, len(times), audio_span, history, None, extra), model=model,
+ plan=plan_for(where, mine, len(times), audio_span, history, None, extra, audio=heard), model=model,
)
velocity = unpatchify_video(
rows_out[:, int(condition.shape[1]) :], f1 - f0, *grid, channels=24
@@ -620,10 +639,36 @@ class H3OmniCausalMixin:
self._run(
video_rows=torch.cat((condition, patchify_video(CLEAN_LEVEL * current + (1 - CLEAN_LEVEL) * jitter)), dim=1),
audio_rows=audio, prompt=prompt, arrays=arrays, times=times.copy(),
- plan=plan_for(where, mine, len(times), audio_span, history, ("clean", c), extra), model=model,
+ plan=plan_for(where, mine, len(times), audio_span, history, ("clean", c), extra, audio=heard),
+ model=model,
)
return clip, last_inputs, last_times, jitters
+ def _frozen_positions(self, layout: Any, rows: H3CausalRows, frames: int) -> tuple[list[float], np.ndarray]:
+ """Per chunk, how far in time a stream moves the prompt, first frame and audio forward (0 within the training
+ clip, 47 latents), and the document rows of the audio it hears: each channel's first training-clip length."""
+
+ train = TRAIN_LATENTS
+ chunk = int(self.causal.get("chunk", 5))
+ per = self.audio_latents(frames)
+ size = rows.audio[1] - rows.audio[0]
+ heard = min(per, self.audio_latents(train)) if frames > train else per
+ audio_keep = np.concatenate([rows.audio[0] + k * per + np.arange(heard) for k in range(size // per)])
+ trained = chunk_frames(train, chunk)
+ last = max(i for i, (a, b) in enumerate(trained) if b - a == chunk)
+ starts = [float(layout.position_ids[a, 0]) for a, _b in rows.target]
+ shifts = []
+ for c in range(rows.chunks):
+ if frames <= train or c <= last:
+ shifts.append(0.0)
+ continue
+ # the latest trained chunk a whole number of 5-latent steps (17 frames, one even time step) behind
+ match = [i for i in range(last + 1) if ((c - i) * chunk) % 5 == 0]
+ if not match:
+ raise ValueError(f"no trained chunk lies a whole number of 5-latent steps behind chunk {c}")
+ shifts.append(starts[c] - starts[max(match)])
+ return shifts, audio_keep
+
def _core_class() -> type:
from .stage0p5 import H3Stage0p5Core