Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
4.6 KB
Raw Blame
schema: solarwm.run.v1
action: train
name: causal_structure_guided_4step
model:
  family: minimax_h3
  architecture: minimax-h3-33b
  checkpoint_path: /path/to/MiniMax-H3/ref2va
  transformer_subfolder: transformer
  transformer_device_map: null
  training_mode: lora
  torch_dtype: bfloat16
  load_conditioners: false
  camera_attention_mode: fused_prope
  attention_head_dim: 128
  camera_prope_head_dim_start: 96
  camera_prope_head_dim_end: 128
  camera_translation_transform: logd4
  camera_intrinsics_mode: wan_fixed
  latent_channels: 24
  latent_height: 48
  latent_width: 84
  rows_per_latent: 1008
  num_frames_per_block: 5
  max_prior_clean_chunks: 5
  attention_backend: flash
  adapter:
    type: lora
    target: block_qkvo_ffn
    rank: 384
    alpha: 384
    dropout: 0.0
    bias: none
    dtype: bfloat16
    expected_target_linear_modules: 312
    expected_trainable_parameters: 2075394048
data:
  input_mode: preencoded
  format: webdataset
  dataset_name: h3_preencoded_wds
  transport:
    kind: local
    root: /path/to/encoded_samples
  index_root: /path/to/encoded_samples
  train_index: index.jsonl
  test_index: index.jsonl
  silence_latents_path: assets/h3_silence_153_158_170.safetensors
  encoder_contract_path: assets/encoder_contract.json
  preencode_version: h3.158f.v1
  pixel_frames: 158
  encoded_latents: 47
  train_target_latents: 47
  height: 768
  width: 1344
  latent_channels: 24
  latent_height: 48
  latent_width: 84
  frame_sampling: contiguous
  source_fps_policy: audit_only
  random_start: false
  resolution_label: default
  max_relative_translation: 20.0
  max_camera_absolute_value: 20.0
  seed: 42
  num_workers: 8
  prefetch_factor: 2
  omni_root: /path/to/encoded_samples
  omni_index: /path/to/encoded_samples/train.jsonl
  omni_length_buckets: true
distributed:
  world_size: 8
  sequence_parallel_size: 8
  context_parallel_size: 1
  rank_partition: node_shard
  sp_peers_share_sample: true
  sp_peers_share_rng: true
train:
  stage: stage0p5
  causal_mode: bidirectional  # the camera-model attention route, unused here: block-causal rendering is set by train.causal_omni (control) or train.dmd.causal_stream (four-step)
  objective: dmd
  precision: bfloat16
  micro_batch_size: 1
  gradient_accumulation_steps: 1
  global_batch_size: 1
  max_steps: 1600
  video_timestep_shift: 12.0
  audio_timestep_shift: 3.0
  keyframe_noise_augmentation: 0.999
  audio_loss_weight: 0.0
  optimizer:
    name: fp32_master_adamw
    learning_rate: 2.0e-05
    betas:
    - 0.0
    - 0.9
    epsilon: 1.0e-08
    weight_decay: 0.01
    gradient_clip: 1.0
    warmup_steps: 0
    min_lr_ratio: 1.0
  fsdp:
    sharding_strategy: FULL_SHARD
    activation_checkpointing: true
    preserve_checkpoint_dtype: true
    param_dtype: null
    reduce_dtype: float32
    buffer_dtype: null
    saved_attention_blocks: 0
    frozen_base_shard_size: 8
  observations:
    patterns:
      shared: 1.0
    dropout:
      observation: 0.0
  objective_variant: v2
  critic_optimizer:
    name: fp32_master_adamw
    learning_rate: 5.0e-05
    betas:
    - 0.9
    - 0.999
    epsilon: 1.0e-08
    weight_decay: 0.0
    gradient_clip: 1.0
    warmup_steps: 0
    min_lr_ratio: 1.0
  dmd:
    # the causal recipe: the causal control LoRA (step 4 of docs/training.md) is frozen into the student's base and a separate
    # acceleration LoRA, starting as the released bidirectional four-step LoRA, is distilled on the student's own
    # block-by-block streams (replayed as one document for the gradient) with structure guidance; the bidirectional
    # control LoRA teaches, scoring the whole clip
    mode: causal
    decoupled: true
    student_init: /path/to/causal_control_lora_checkpoint
    accel_init: /path/to/solarrenderer_4step_lora.safetensors
    control: /path/to/control_lora_checkpoint
    causal_stream:
      chunk: 5
      window: 6
    # the critic learns faster than the student (2e-5 / 5e-5): with a slower critic the stream drifts to
    # over-contrast within a few updates
    recipe: plug_h3
    forwards: 4
    critic_updates_per_student: 5
    flow_matching_weight: 0.0
    score_shift: 12.0
    structure_guidance: 1.5
validation:
  sample_count: 2
  selection_seed: 42
  noise_seed: 42
  pixel_frames: 158
  latent_frames: 47
  fps: 24
  num_inference_steps: 30
  max_relative_translation: 20.0
  max_camera_absolute_value: 20.0
  validate_every_steps: 10000000
  smoke_step: 10000000
  passes:
  - live
  - ema
checkpoint:
  save_every_steps: 400
  save_optimizer: true
  ema:
    enabled: true
    dtype: float32
    sharded: true
    decay: 0.9999
    start_step: 0
    update_every_steps: 1
  initialization: {}
runtime:
  output_dir: /path/to/runs/causal_structure_guided_4step