SolarRenderer: Few-Step Generative Reference Rendering
Python
schema: solarwm.run.v1
action: train
name: causal_structure_guided_4step
model:
family: minimax_h3
architecture: minimax-h3-33b
checkpoint_path: /path/to/MiniMax-H3/ref2va
transformer_subfolder: transformer
transformer_device_map: null
training_mode: lora
torch_dtype: bfloat16
load_conditioners: false
camera_attention_mode: fused_prope
attention_head_dim: 128
camera_prope_head_dim_start: 96
camera_prope_head_dim_end: 128
camera_translation_transform: logd4
camera_intrinsics_mode: wan_fixed
latent_channels: 24
latent_height: 48
latent_width: 84
rows_per_latent: 1008
num_frames_per_block: 5
max_prior_clean_chunks: 5
attention_backend: flash
adapter:
type: lora
target: block_qkvo_ffn
rank: 384
alpha: 384
dropout: 0.0
bias: none
dtype: bfloat16
expected_target_linear_modules: 312
expected_trainable_parameters: 2075394048
data:
input_mode: preencoded
format: webdataset
dataset_name: h3_preencoded_wds
transport:
kind: local
root: /path/to/encoded_samples
index_root: /path/to/encoded_samples
train_index: index.jsonl
test_index: index.jsonl
silence_latents_path: assets/h3_silence_153_158_170.safetensors
encoder_contract_path: assets/encoder_contract.json
preencode_version: h3.158f.v1
pixel_frames: 158
encoded_latents: 47
train_target_latents: 47
height: 768
width: 1344
latent_channels: 24
latent_height: 48
latent_width: 84
frame_sampling: contiguous
source_fps_policy: audit_only
random_start: false
resolution_label: default
max_relative_translation: 20.0
max_camera_absolute_value: 20.0
seed: 42
num_workers: 8
prefetch_factor: 2
omni_root: /path/to/encoded_samples
omni_index: /path/to/encoded_samples/train.jsonl
omni_length_buckets: true
distributed:
world_size: 8
sequence_parallel_size: 8
context_parallel_size: 1
rank_partition: node_shard
sp_peers_share_sample: true
sp_peers_share_rng: true
train:
stage: stage0p5
causal_mode: bidirectional # the camera-model attention route, unused here: block-causal rendering is set by train.causal_omni (control) or train.dmd.causal_stream (four-step)
objective: dmd
precision: bfloat16
micro_batch_size: 1
gradient_accumulation_steps: 1
global_batch_size: 1
max_steps: 1600
video_timestep_shift: 12.0
audio_timestep_shift: 3.0
keyframe_noise_augmentation: 0.999
audio_loss_weight: 0.0
optimizer:
name: fp32_master_adamw
learning_rate: 2.0e-05
betas:
- 0.0
- 0.9
epsilon: 1.0e-08
weight_decay: 0.01
gradient_clip: 1.0
warmup_steps: 0
min_lr_ratio: 1.0
fsdp:
sharding_strategy: FULL_SHARD
activation_checkpointing: true
preserve_checkpoint_dtype: true
param_dtype: null
reduce_dtype: float32
buffer_dtype: null
saved_attention_blocks: 0
frozen_base_shard_size: 8
observations:
patterns:
shared: 1.0
dropout:
observation: 0.0
objective_variant: v2
critic_optimizer:
name: fp32_master_adamw
learning_rate: 5.0e-05
betas:
- 0.9
- 0.999
epsilon: 1.0e-08
weight_decay: 0.0
gradient_clip: 1.0
warmup_steps: 0
min_lr_ratio: 1.0
dmd:
# the causal recipe: the causal control LoRA (step 4 of docs/training.md) is frozen into the student's base and a separate
# acceleration LoRA, starting as the released bidirectional four-step LoRA, is distilled on the student's own
# block-by-block streams (replayed as one document for the gradient) with structure guidance; the bidirectional
# control LoRA teaches, scoring the whole clip
mode: causal
decoupled: true
student_init: /path/to/causal_control_lora_checkpoint
accel_init: /path/to/solarrenderer_4step_lora.safetensors
control: /path/to/control_lora_checkpoint
causal_stream:
chunk: 5
window: 6
# the critic learns faster than the student (2e-5 / 5e-5): with a slower critic the stream drifts to
# over-contrast within a few updates
recipe: plug_h3
forwards: 4
critic_updates_per_student: 5
flow_matching_weight: 0.0
score_shift: 12.0
structure_guidance: 1.5
validation:
sample_count: 2
selection_seed: 42
noise_seed: 42
pixel_frames: 158
latent_frames: 47
fps: 24
num_inference_steps: 30
max_relative_translation: 20.0
max_camera_absolute_value: 20.0
validate_every_steps: 10000000
smoke_step: 10000000
passes:
- live
- ema
checkpoint:
save_every_steps: 400
save_optimizer: true
ema:
enabled: true
dtype: float32
sharded: true
decay: 0.9999
start_step: 0
update_every_steps: 1
initialization: {}
runtime:
output_dir: /path/to/runs/causal_structure_guided_4step