SolarRenderer: Few-Step Generative Reference Rendering
Python
SolarRenderer: Few-Step Generative Reference Rendering
Enderfga5a5da36c2cc0
diff --git a/.gitignore b/.gitignorenew file mode 100644index 0000000..c8a054b--- /dev/null+++ b/.gitignore@@ -0,0 +1,7 @@+__pycache__/+*.pyc+*.egg-info/+/weights/+/MiniMax-H3/+*.mp4+!examples/**/*.mp4diff --git a/LICENSE b/LICENSEnew file mode 100644index 0000000..261eeb9--- /dev/null+++ b/LICENSE@@ -0,0 +1,201 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.diff --git a/NOTICE b/NOTICEnew file mode 100644index 0000000..b6eab81--- /dev/null+++ b/NOTICE@@ -0,0 +1,32 @@+SolarRenderer+Copyright 2026 SolarRenderer authors++The code in this repository is licensed under the Apache License, Version 2.0 (see LICENSE). It builds on the SolarWM+training framework (https://github.com/Junchao-cs/SolarWM, Apache-2.0).++--------------------------------------------------------------------------------+MiniMax-H3+--------------------------------------------------------------------------------++MiniMax H3 is licensed under the MiniMax H3 Community License Agreement, Copyright Β© 2026 MiniMax. All Rights Reserved.++SolarRenderer runs on MiniMax-H3, which is not distributed here; obtain it from its publisher+(https://huggingface.co/MiniMaxAI/MiniMax-H3) under its own license and Acceptable Use Policy.++--------------------------------------------------------------------------------+Third-party source included in this repository+--------------------------------------------------------------------------------++ PRoPE: Projective Positional Encoding for Multiview Transformers, MIT+ Copyright (c) the PRoPE authors.+ Adapted as src/solarwm/backends/minimax_h3/torch_prope.py.++Each adapted file keeps its upstream copyright header and states that it was modified.++--------------------------------------------------------------------------------+Evaluation dependencies (not included; fetched by eval/setup.sh)+--------------------------------------------------------------------------------++ DOVER (github.com/VQAssessment/DOVER), CoTracker3 (github.com/facebookresearch/co-tracker), the LAION aesthetic+ predictor (github.com/christophschuhmann/improved-aesthetic-predictor) and VideoScore2 (TIGER-Lab/VideoScore2) keep+ their own licenses; DOVER's is non-commercial.diff --git a/README.md b/README.mdnew file mode 100644index 0000000..51b7ed7--- /dev/null+++ b/README.md@@ -0,0 +1,105 @@+<p align="center"><img src="assets/teaser.gif" width="100%"/></p>++<h1 align="center">SolarRenderer: Few-Step Generative Reference Rendering</h1>++<p align="center">+ π <a href="https://enderfga.cn/SolarRenderer">Project Page</a> β’+ π <a href="https://enderfga.cn/data/solarrenderer.pdf">Paper</a> β’+ π€ <a href="https://huggingface.co/Enderfga/SolarRenderer">Weights</a> β’+ π€ <a href="https://huggingface.co/datasets/Enderfga/SolarRenderer-Data">Data</a> β’+ π» <a href="https://github.com/Enderfga/SolarRenderer/tree/data">Data code</a>+</p>++**SolarRenderer** turns a structure video from a game engine or an agent (grey box, semantic map, skeletons, depth,+body mesh, or a sketch it never saw in training) into a finished video, with an optional first frame and a prompt+deciding the look. It runs on **MiniMax-H3** with two LoRAs and renders a clip in **four steps**.+A block-causal pair of the same LoRAs streams the clip block by block.++## News++- **[2026-10]** Code, weights, the shareable training data and the benchmark released.++## Highlights++- π§± **Any structure, one model**: no extra branch and no condition-type label; the prompt names the structure.+- β‘ **Four steps**: structure-guided post-training, 12.3Γ faster sampling than 50 steps.+- π¨ **Any look**: the same structure renders as photoreal, anime, pixel art or ink, set by the first frame and prompt.+- π **Streaming**: a block-causal version of both LoRAs renders the video block by block, each block shown as soon as+ it is done ([guide](docs/inference.md#stream-a-clip)).+- π **Open benchmark**: 208 hard cases and 53 held-out clips, with scoring code, our renders and every judge verdict.++## Gallery++Structure video (left) and SolarRenderer (right).++| Scene grey box | Semantic segmentation |+|:---:|:---:|+| <img src="assets/gallery/greybox.gif" width="100%"> | <img src="assets/gallery/semantic.gif" width="100%"> |+| **Object masks + skeletons** | **SMPL body mesh** |+| <img src="assets/gallery/masks_skeletons.gif" width="100%"> | <img src="assets/gallery/smpl.gif" width="100%"> |+| **Near-field shaded geometry** | **Pencil storyboard** (unseen) |+| <img src="assets/gallery/near_depth.gif" width="100%"> | <img src="assets/gallery/unseen_pencil.gif" width="100%"> |+| **Point cloud** (unseen) | **Child's crayon drawing** (unseen) |+| <img src="assets/gallery/unseen_pointcloud.gif" width="100%"> | <img src="assets/gallery/unseen_crayon.gif" width="100%"> |+| **Voxels** (unseen) | **Wireframe** (unseen) |+| <img src="assets/gallery/unseen_voxels.gif" width="100%"> | <img src="assets/gallery/unseen_wireframe.gif" width="100%"> |++More structure types, styles and comparisons are on the [project page](https://enderfga.cn/SolarRenderer/#structures).++## Quick Start++```bash+git clone https://github.com/Enderfga/SolarRenderer && cd SolarRenderer+pip install -e . && pip install flash-attn==2.7.4.post1 --no-build-isolation+```++```bash+python scripts/prepare_h3.py --dir ./MiniMax-H3+hf download Enderfga/SolarRenderer solarrenderer_control_lora.safetensors solarrenderer_4step_lora.safetensors \+ --local-dir ./weights+```++```bash+python scripts/render.py --h3 ./MiniMax-H3 \+ --control weights/solarrenderer_control_lora.safetensors \+ --fewstep weights/solarrenderer_4step_lora.safetensors \+ --structure examples/sem-highway_synthwave/structure.mp4 \+ --first examples/sem-highway_synthwave/first.png \+ --prompt examples/sem-highway_synthwave/prompt.txt --out render.mp4+```++The example is a [synthwave highway run](examples/sem-highway_synthwave) (semantic map in, neon render out).+Add `--causal` with the two causal LoRAs to stream the clip block by block+([details](docs/inference.md#stream-a-clip)).++- βοΈ **Prompts and first frames**: give [`skills/solarrenderer-caption`](skills/solarrenderer-caption) to your coding+ agent ([guide](docs/prompts.md)).+- π¬ **Inference details**: inputs, sizes and timing ([guide](docs/inference.md)).+- ποΈ **Training** on one 8-GPU node ([guide](docs/training.md)).+- π **Benchmark scoring** ([guide](eval/README.md)).++## Citation++```bibtex+@article{fang2026solarrenderer,+ title = {SolarRenderer: Few-Step Generative Reference Rendering},+ author = {Fang, Guian and Qian, Shengju and Lu, Dongyue and+ Huang, Junchao and Jiang, Lutao and Wang, Xin and+ Song, Yiren and Jiang, Li and Shou, Mike Zheng},+ year = {2026}+}+```++<details>+<summary>Acknowledgement</summary>++SolarRenderer builds on [MiniMax-H3](https://huggingface.co/MiniMaxAI/MiniMax-H3), starts its four-step LoRA from+[DMAD](https://huggingface.co/ZhengmingYu/DMAD), and shares its training framework with+[SolarWM](https://github.com/Junchao-cs/SolarWM).+</details>++## License++Code: Apache 2.0. Weights: [MiniMax H3 Community License](https://huggingface.co/Enderfga/SolarRenderer/blob/main/LICENSE),+including its territorial restrictions (EU, UK, Republic of Korea and US excluded) and Acceptable Use Policy; the+four-step LoRA is fine-tuned from DMAD under the same license.diff --git a/assets/encoder_contract.json b/assets/encoder_contract.jsonnew file mode 100644index 0000000..df70b82--- /dev/null+++ b/assets/encoder_contract.json@@ -0,0 +1,100 @@+{+ "camera_convention": "absolute_c2w+normalized_K",+ "extras": {+ "encoder_identity": "official-minimax-h3-codec",+ "frame_sampling": "exact_contiguous",+ "qwen_hidden_state": 50,+ "qwen_presentation": "<Picture 1> joint image+caption",+ "silence_artifact_profile": {+ "artifact": "official-audiovae-encoded-stereo-zero-waveform",+ "audio_hop_length": 800,+ "audio_sampling_rate": 32000,+ "format": "h3.158f.v1",+ "normalization": "audio_vae.config.latents_mean_std",+ "posterior": "mode",+ "schema": "solarwm.minimax-h3-silence-profile.v1",+ "tensor": {+ "dtype": "bfloat16",+ "key": "silence_158f",+ "shape": [+ 2,+ 32,+ 263+ ]+ },+ "waveform": "real_stereo_zero_waveform"+ },+ "source_fps_policy": "audit_only",+ "video_vae_normalization": "config.latents_mean_std",+ "video_vae_posterior": "sample(seed=42)-fp16-round"+ },+ "family": "minimax_h3",+ "format_version": "h3.158f.v1",+ "height": 768,+ "latent_frames": 47,+ "pixel_frames": 158,+ "schema": "solarwm.encoder.v1",+ "tensors": [+ {+ "dtype": "bfloat16",+ "name": "target_latents",+ "shape": [+ 24,+ 47,+ 48,+ 84+ ]+ },+ {+ "dtype": "bfloat16",+ "name": "anchor_latents",+ "shape": [+ 24,+ 1,+ 48,+ 84+ ]+ },+ {+ "dtype": "bfloat16",+ "name": "prompt_embeds",+ "shape": [+ null,+ 5120+ ]+ },+ {+ "dtype": "int64",+ "name": "text_token_tags",+ "shape": [+ null+ ]+ },+ {+ "dtype": "int64",+ "name": "source_frame_indices",+ "shape": [+ 158+ ]+ },+ {+ "dtype": "float32",+ "name": "camera_c2w",+ "shape": [+ 158,+ 4,+ 4+ ]+ },+ {+ "dtype": "float32",+ "name": "camera_K",+ "shape": [+ 158,+ 3,+ 3+ ]+ }+ ],+ "width": 1344+}diff --git a/assets/gallery/greybox.gif b/assets/gallery/greybox.gifnew file mode 100644index 0000000..f541bb7Binary files /dev/null and b/assets/gallery/greybox.gif differdiff --git a/assets/gallery/masks_skeletons.gif b/assets/gallery/masks_skeletons.gifnew file mode 100644index 0000000..f679097Binary files /dev/null and b/assets/gallery/masks_skeletons.gif differdiff --git a/assets/gallery/near_depth.gif b/assets/gallery/near_depth.gifnew file mode 100644index 0000000..c2dbc97Binary files /dev/null and b/assets/gallery/near_depth.gif differdiff --git a/assets/gallery/semantic.gif b/assets/gallery/semantic.gifnew file mode 100644index 0000000..00f66a1Binary files /dev/null and b/assets/gallery/semantic.gif differdiff --git a/assets/gallery/smpl.gif b/assets/gallery/smpl.gifnew file mode 100644index 0000000..ab31c1eBinary files /dev/null and b/assets/gallery/smpl.gif differdiff --git a/assets/gallery/unseen_crayon.gif b/assets/gallery/unseen_crayon.gifnew file mode 100644index 0000000..1f70db1Binary files /dev/null and b/assets/gallery/unseen_crayon.gif differdiff --git a/assets/gallery/unseen_pencil.gif b/assets/gallery/unseen_pencil.gifnew file mode 100644index 0000000..a5a418eBinary files /dev/null and b/assets/gallery/unseen_pencil.gif differdiff --git a/assets/gallery/unseen_pointcloud.gif b/assets/gallery/unseen_pointcloud.gifnew file mode 100644index 0000000..8bd0791Binary files /dev/null and b/assets/gallery/unseen_pointcloud.gif differdiff --git a/assets/gallery/unseen_voxels.gif b/assets/gallery/unseen_voxels.gifnew file mode 100644index 0000000..998a85eBinary files /dev/null and b/assets/gallery/unseen_voxels.gif differdiff --git a/assets/gallery/unseen_wireframe.gif b/assets/gallery/unseen_wireframe.gifnew file mode 100644index 0000000..ad0bf9eBinary files /dev/null and b/assets/gallery/unseen_wireframe.gif differdiff --git a/assets/h3_silence_153_158_170.safetensors b/assets/h3_silence_153_158_170.safetensorsnew file mode 100644index 0000000..76c1729Binary files /dev/null and b/assets/h3_silence_153_158_170.safetensors differdiff --git a/assets/logo.png b/assets/logo.pngnew file mode 100644index 0000000..b4894e8Binary files /dev/null and b/assets/logo.png differdiff --git a/assets/teaser.gif b/assets/teaser.gifnew file mode 100644index 0000000..5b901deBinary files /dev/null and b/assets/teaser.gif differdiff --git a/configs/causal_control_lora.yaml b/configs/causal_control_lora.yamlnew file mode 100644index 0000000..d15f05b--- /dev/null+++ b/configs/causal_control_lora.yaml@@ -0,0 +1,152 @@+schema: solarwm.run.v1+action: train+name: causal_control_lora+model:+ family: minimax_h3+ architecture: minimax-h3-33b+ checkpoint_path: /path/to/MiniMax-H3/ref2va+ transformer_subfolder: transformer+ transformer_device_map: null+ training_mode: lora+ torch_dtype: bfloat16+ load_conditioners: false+ camera_attention_mode: fused_prope+ attention_head_dim: 128+ camera_prope_head_dim_start: 96+ camera_prope_head_dim_end: 128+ camera_translation_transform: logd4+ camera_intrinsics_mode: wan_fixed+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ rows_per_latent: 1008+ num_frames_per_block: 5+ max_prior_clean_chunks: 5+ attention_backend: flash+ adapter:+ type: lora+ target: block_qkvo_ffn+ rank: 384+ alpha: 384+ dropout: 0.0+ bias: none+ dtype: bfloat16+ expected_target_linear_modules: 312+ expected_trainable_parameters: 2075394048+data:+ input_mode: preencoded+ format: webdataset+ dataset_name: h3_preencoded_wds+ transport:+ kind: local+ root: /path/to/encoded_samples+ index_root: /path/to/encoded_samples+ train_index: index.jsonl+ test_index: index.jsonl+ silence_latents_path: assets/h3_silence_153_158_170.safetensors+ encoder_contract_path: assets/encoder_contract.json+ preencode_version: h3.158f.v1+ pixel_frames: 158+ encoded_latents: 47+ train_target_latents: 47+ height: 768+ width: 1344+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ frame_sampling: contiguous+ source_fps_policy: audit_only+ random_start: false+ resolution_label: default+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ seed: 42+ num_workers: 8+ prefetch_factor: 2+ omni_root: /path/to/encoded_samples+ omni_index: /path/to/encoded_samples/train.jsonl+ omni_length_buckets: true+distributed:+ world_size: 8+ sequence_parallel_size: 4+ context_parallel_size: 1+ rank_partition: node_shard+ sp_peers_share_sample: true+ sp_peers_share_rng: true+train:+ stage: stage0p5+ causal_mode: bidirectional # the camera-model attention route, unused here: block-causal rendering is set by train.causal_omni (control) or train.dmd.causal_stream (four-step)+ objective: flow_matching+ precision: bfloat16+ micro_batch_size: 1+ gradient_accumulation_steps: 1+ global_batch_size: 2+ max_steps: 19200+ video_timestep_shift: 12.0+ audio_timestep_shift: 3.0+ keyframe_noise_augmentation: 0.999+ audio_loss_weight: 0.0+ optimizer:+ name: fp32_master_adamw+ learning_rate: 3.0e-05+ betas:+ - 0.9+ - 0.95+ epsilon: 1.0e-08+ weight_decay: 0.01+ gradient_clip: 1.0+ warmup_steps: 50+ min_lr_ratio: 1.0+ fsdp:+ sharding_strategy: FULL_SHARD+ activation_checkpointing: true+ preserve_checkpoint_dtype: true+ param_dtype: null+ reduce_dtype: float32+ buffer_dtype: null+ saved_attention_blocks: 0+ # block-causal rendering: blocks of at most 5 latents, each seeing the prompt, the first frame, the structure of+ # its last 6 blocks and the clean frames of the 5 blocks before it (teacher forcing), never a later block+ causal_omni:+ mode: teacher+ chunk: 5+ window: 6+ corrupt: 0.0+ observations:+ patterns:+ shared: 1.0+ dropout:+ observation: 0.0+validation:+ sample_count: 2+ selection_seed: 42+ noise_seed: 42+ pixel_frames: 158+ latent_frames: 47+ fps: 24+ num_inference_steps: 30+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ validate_every_steps: 10000000+ smoke_step: 10000000+ passes:+ - live+ - ema+checkpoint:+ # the causal control LoRA starts as the trained bidirectional control LoRA (its checkpoint directory)+ initialization:+ student:+ path: /path/to/control_lora_checkpoint+ stage: stage0p5+ weight_source: live+ save_every_steps: 1600+ save_optimizer: true+ ema:+ enabled: true+ dtype: float32+ sharded: true+ decay: 0.9999+ start_step: 0+ update_every_steps: 1+runtime:+ output_dir: /path/to/runs/causal_control_loradiff --git a/configs/causal_structure_guided_4step.yaml b/configs/causal_structure_guided_4step.yamlnew file mode 100644index 0000000..a292137--- /dev/null+++ b/configs/causal_structure_guided_4step.yaml@@ -0,0 +1,174 @@+schema: solarwm.run.v1+action: train+name: causal_structure_guided_4step+model:+ family: minimax_h3+ architecture: minimax-h3-33b+ checkpoint_path: /path/to/MiniMax-H3/ref2va+ transformer_subfolder: transformer+ transformer_device_map: null+ training_mode: lora+ torch_dtype: bfloat16+ load_conditioners: false+ camera_attention_mode: fused_prope+ attention_head_dim: 128+ camera_prope_head_dim_start: 96+ camera_prope_head_dim_end: 128+ camera_translation_transform: logd4+ camera_intrinsics_mode: wan_fixed+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ rows_per_latent: 1008+ num_frames_per_block: 5+ max_prior_clean_chunks: 5+ attention_backend: flash+ adapter:+ type: lora+ target: block_qkvo_ffn+ rank: 384+ alpha: 384+ dropout: 0.0+ bias: none+ dtype: bfloat16+ expected_target_linear_modules: 312+ expected_trainable_parameters: 2075394048+data:+ input_mode: preencoded+ format: webdataset+ dataset_name: h3_preencoded_wds+ transport:+ kind: local+ root: /path/to/encoded_samples+ index_root: /path/to/encoded_samples+ train_index: index.jsonl+ test_index: index.jsonl+ silence_latents_path: assets/h3_silence_153_158_170.safetensors+ encoder_contract_path: assets/encoder_contract.json+ preencode_version: h3.158f.v1+ pixel_frames: 158+ encoded_latents: 47+ train_target_latents: 47+ height: 768+ width: 1344+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ frame_sampling: contiguous+ source_fps_policy: audit_only+ random_start: false+ resolution_label: default+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ seed: 42+ num_workers: 8+ prefetch_factor: 2+ omni_root: /path/to/encoded_samples+ omni_index: /path/to/encoded_samples/train.jsonl+ omni_length_buckets: true+distributed:+ world_size: 8+ sequence_parallel_size: 8+ context_parallel_size: 1+ rank_partition: node_shard+ sp_peers_share_sample: true+ sp_peers_share_rng: true+train:+ stage: stage0p5+ causal_mode: bidirectional # the camera-model attention route, unused here: block-causal rendering is set by train.causal_omni (control) or train.dmd.causal_stream (four-step)+ objective: dmd+ precision: bfloat16+ micro_batch_size: 1+ gradient_accumulation_steps: 1+ global_batch_size: 1+ max_steps: 1600+ video_timestep_shift: 12.0+ audio_timestep_shift: 3.0+ keyframe_noise_augmentation: 0.999+ audio_loss_weight: 0.0+ optimizer:+ name: fp32_master_adamw+ learning_rate: 2.0e-05+ betas:+ - 0.0+ - 0.9+ epsilon: 1.0e-08+ weight_decay: 0.01+ gradient_clip: 1.0+ warmup_steps: 0+ min_lr_ratio: 1.0+ fsdp:+ sharding_strategy: FULL_SHARD+ activation_checkpointing: true+ preserve_checkpoint_dtype: true+ param_dtype: null+ reduce_dtype: float32+ buffer_dtype: null+ saved_attention_blocks: 0+ frozen_base_shard_size: 8+ observations:+ patterns:+ shared: 1.0+ dropout:+ observation: 0.0+ objective_variant: v2+ critic_optimizer:+ name: fp32_master_adamw+ learning_rate: 5.0e-05+ betas:+ - 0.9+ - 0.999+ epsilon: 1.0e-08+ weight_decay: 0.0+ gradient_clip: 1.0+ warmup_steps: 0+ min_lr_ratio: 1.0+ dmd:+ # the causal recipe: the causal control LoRA (step 4 of docs/training.md) is frozen into the student's base and a separate+ # acceleration LoRA, starting as the released bidirectional four-step LoRA, is distilled on the student's own+ # block-by-block streams (replayed as one document for the gradient) with structure guidance; the bidirectional+ # control LoRA teaches, scoring the whole clip+ mode: causal+ decoupled: true+ student_init: /path/to/causal_control_lora_checkpoint+ accel_init: /path/to/solarrenderer_4step_lora.safetensors+ control: /path/to/control_lora_checkpoint+ causal_stream:+ chunk: 5+ window: 6+ # the critic learns faster than the student (2e-5 / 5e-5): with a slower critic the stream drifts to+ # over-contrast within a few updates+ recipe: plug_h3+ forwards: 4+ critic_updates_per_student: 5+ flow_matching_weight: 0.0+ score_shift: 12.0+ structure_guidance: 1.5+validation:+ sample_count: 2+ selection_seed: 42+ noise_seed: 42+ pixel_frames: 158+ latent_frames: 47+ fps: 24+ num_inference_steps: 30+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ validate_every_steps: 10000000+ smoke_step: 10000000+ passes:+ - live+ - ema+checkpoint:+ save_every_steps: 400+ save_optimizer: true+ ema:+ enabled: true+ dtype: float32+ sharded: true+ decay: 0.9999+ start_step: 0+ update_every_steps: 1+ initialization: {}+runtime:+ output_dir: /path/to/runs/causal_structure_guided_4stepdiff --git a/configs/control_lora.yaml b/configs/control_lora.yamlnew file mode 100644index 0000000..07ab0ea--- /dev/null+++ b/configs/control_lora.yaml@@ -0,0 +1,138 @@+schema: solarwm.run.v1+action: train+name: control_lora+model:+ family: minimax_h3+ architecture: minimax-h3-33b+ checkpoint_path: /path/to/MiniMax-H3/ref2va+ transformer_subfolder: transformer+ transformer_device_map: null+ training_mode: lora+ torch_dtype: bfloat16+ load_conditioners: false+ camera_attention_mode: fused_prope+ attention_head_dim: 128+ camera_prope_head_dim_start: 96+ camera_prope_head_dim_end: 128+ camera_translation_transform: logd4+ camera_intrinsics_mode: wan_fixed+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ rows_per_latent: 1008+ num_frames_per_block: 5+ max_prior_clean_chunks: 5+ attention_backend: _native_cudnn+ adapter:+ type: lora+ target: block_qkvo_ffn+ rank: 384+ alpha: 384+ dropout: 0.0+ bias: none+ dtype: bfloat16+ expected_target_linear_modules: 312+ expected_trainable_parameters: 2075394048+data:+ input_mode: preencoded+ format: webdataset+ dataset_name: h3_preencoded_wds+ transport:+ kind: local+ root: /path/to/encoded_samples+ index_root: /path/to/encoded_samples+ train_index: index.jsonl+ test_index: index.jsonl+ silence_latents_path: assets/h3_silence_153_158_170.safetensors+ encoder_contract_path: assets/encoder_contract.json+ preencode_version: h3.158f.v1+ pixel_frames: 158+ encoded_latents: 47+ train_target_latents: 47+ height: 768+ width: 1344+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ frame_sampling: contiguous+ source_fps_policy: audit_only+ random_start: false+ resolution_label: default+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ seed: 42+ num_workers: 8+ prefetch_factor: 2+ omni_root: /path/to/encoded_samples+ omni_index: /path/to/encoded_samples/train.jsonl+ omni_length_buckets: true+distributed:+ world_size: 8+ sequence_parallel_size: 2+ context_parallel_size: 1+ rank_partition: node_shard+ sp_peers_share_sample: true+ sp_peers_share_rng: true+train:+ stage: stage0p5+ causal_mode: bidirectional+ objective: flow_matching+ precision: bfloat16+ micro_batch_size: 1+ gradient_accumulation_steps: 1+ global_batch_size: 4+ max_steps: 38400+ video_timestep_shift: 12.0+ audio_timestep_shift: 3.0+ keyframe_noise_augmentation: 0.999+ audio_loss_weight: 0.0+ optimizer:+ name: fp32_master_adamw+ learning_rate: 0.0001+ betas:+ - 0.9+ - 0.95+ epsilon: 1.0e-08+ weight_decay: 0.01+ gradient_clip: 1.0+ warmup_steps: 16000+ fsdp:+ sharding_strategy: FULL_SHARD+ activation_checkpointing: true+ preserve_checkpoint_dtype: true+ param_dtype: null+ reduce_dtype: float32+ buffer_dtype: null+ saved_attention_blocks: 20+ observations:+ patterns:+ shared: 1.0+ dropout:+ observation: 0.0+validation:+ sample_count: 2+ selection_seed: 42+ noise_seed: 42+ pixel_frames: 158+ latent_frames: 47+ fps: 24+ num_inference_steps: 30+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ validate_every_steps: 10000000+ smoke_step: 10000000+ passes:+ - live+ - ema+checkpoint:+ save_every_steps: 1600+ save_optimizer: true+ ema:+ enabled: true+ dtype: float32+ sharded: true+ decay: 0.9999+ start_step: 0+ update_every_steps: 1+runtime:+ output_dir: /path/to/runs/control_loradiff --git a/configs/structure_guided_4step.yaml b/configs/structure_guided_4step.yamlnew file mode 100644index 0000000..fd7053c--- /dev/null+++ b/configs/structure_guided_4step.yaml@@ -0,0 +1,167 @@+schema: solarwm.run.v1+action: train+name: structure_guided_4step+model:+ family: minimax_h3+ architecture: minimax-h3-33b+ checkpoint_path: /path/to/MiniMax-H3/ref2va+ transformer_subfolder: transformer+ transformer_device_map: null+ training_mode: lora+ torch_dtype: bfloat16+ load_conditioners: false+ camera_attention_mode: fused_prope+ attention_head_dim: 128+ camera_prope_head_dim_start: 96+ camera_prope_head_dim_end: 128+ camera_translation_transform: logd4+ camera_intrinsics_mode: wan_fixed+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ rows_per_latent: 1008+ num_frames_per_block: 5+ max_prior_clean_chunks: 5+ attention_backend: flash+ adapter:+ type: lora+ target: block_qkvo_ffn+ rank: 384+ alpha: 384+ dropout: 0.0+ bias: none+ dtype: bfloat16+ expected_target_linear_modules: 312+ expected_trainable_parameters: 2075394048+data:+ input_mode: preencoded+ format: webdataset+ dataset_name: h3_preencoded_wds+ transport:+ kind: local+ root: /path/to/encoded_samples+ index_root: /path/to/encoded_samples+ train_index: index.jsonl+ test_index: index.jsonl+ silence_latents_path: assets/h3_silence_153_158_170.safetensors+ encoder_contract_path: assets/encoder_contract.json+ preencode_version: h3.158f.v1+ pixel_frames: 158+ encoded_latents: 47+ train_target_latents: 47+ height: 768+ width: 1344+ latent_channels: 24+ latent_height: 48+ latent_width: 84+ frame_sampling: contiguous+ source_fps_policy: audit_only+ random_start: false+ resolution_label: default+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ seed: 42+ num_workers: 8+ prefetch_factor: 2+ omni_root: /path/to/encoded_samples+ omni_index: /path/to/encoded_samples/train.jsonl+ omni_length_buckets: true+distributed:+ world_size: 8+ sequence_parallel_size: 2+ context_parallel_size: 1+ rank_partition: node_shard+ sp_peers_share_sample: true+ sp_peers_share_rng: true+train:+ stage: stage0p5+ causal_mode: bidirectional+ objective: dmd+ precision: bfloat16+ micro_batch_size: 1+ gradient_accumulation_steps: 1+ global_batch_size: 4+ max_steps: 800+ video_timestep_shift: 12.0+ audio_timestep_shift: 3.0+ keyframe_noise_augmentation: 0.999+ audio_loss_weight: 0.0+ optimizer:+ name: fp32_master_adamw+ learning_rate: 1.0e-05+ betas: &id001+ - 0.0+ - 0.999+ epsilon: 1.0e-08+ weight_decay: 0.0+ gradient_clip: 10.0+ warmup_steps: 0+ min_lr_ratio: 1.0+ fsdp:+ sharding_strategy: FULL_SHARD+ activation_checkpointing: true+ preserve_checkpoint_dtype: true+ param_dtype: null+ reduce_dtype: float32+ buffer_dtype: null+ saved_attention_blocks: 0+ frozen_base_shard_size: 8+ observations:+ patterns:+ shared: 1.0+ dropout:+ observation: 0.0+ objective_variant: v2+ critic_optimizer:+ name: fp32_master_adamw+ learning_rate: 2.0e-06+ betas: *id001+ epsilon: 1.0e-08+ weight_decay: 0.0+ gradient_clip: 10.0+ warmup_steps: 0+ min_lr_ratio: 1.0+ dmd:+ # the released recipe: the student's adapter starts as the public four-step LoRA and is distilled with+ # structure guidance; the control LoRA stays frozen+ mode: adapt_plug+ # the public DMAD four-step LoRA the student starts from+ init_lora: /path/to/dmad_4step_lora.safetensors+ # the DMAD LoRA's own rank; it seeds the first 128 rank slots of the rank-384 student (model.adapter), the rest+ # start at a zero update, so the released four-step LoRA is rank 384+ init_lora_rank: 128+ init_lora_alpha: 128+ forwards: 4+ critic_updates_per_student: 5+ flow_matching_weight: 1.0+ score_shift: 12.0+ control: /path/to/control_lora_checkpoint+ structure_guidance: 1.5+validation:+ sample_count: 2+ selection_seed: 42+ noise_seed: 42+ pixel_frames: 158+ latent_frames: 47+ fps: 24+ num_inference_steps: 30+ max_relative_translation: 20.0+ max_camera_absolute_value: 20.0+ validate_every_steps: 10000000+ smoke_step: 10000000+ passes:+ - live+ - ema+checkpoint:+ save_every_steps: 400+ save_optimizer: true+ ema:+ enabled: true+ dtype: float32+ sharded: true+ decay: 0.9999+ start_step: 0+ update_every_steps: 1+ initialization: {}+runtime:+ output_dir: /path/to/runs/structure_guided_4stepdiff --git a/docs/inference.md b/docs/inference.mdnew file mode 100644index 0000000..88acbea--- /dev/null+++ b/docs/inference.md@@ -0,0 +1,65 @@+# Inference++## Install++```bash+git clone https://github.com/Enderfga/SolarRenderer && cd SolarRenderer+pip install -e . && pip install flash-attn==2.7.4.post1 --no-build-isolation # Python 3.10+, CUDA; inference uses flash attention+```++flash-attn builds from source when no prebuilt wheel matches your torch and CUDA, which takes a while.++Tested with torch 2.8, diffusers 0.40.0, transformers 5.14.1, peft 0.20.0 and flash-attn 2.7.4, with+`ffmpeg` and `ffprobe` on the path. Inference runs on one GPU; the text encoder and the transformer are loaded one+after the other.++## Render a clip++**1. Download MiniMax-H3 and the weights** (about 135 GB and 8 GB). MiniMax-H3 is released under its own+[license](https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/main/LICENSE); read it before use.++```bash+python scripts/prepare_h3.py --dir ./MiniMax-H3+hf download Enderfga/SolarRenderer solarrenderer_control_lora.safetensors solarrenderer_4step_lora.safetensors \+ --local-dir ./weights+```++**2. Prepare the inputs.**++- `structure.mp4`: the structure video, 24 fps, at least 158 frames (6.6 s), landscape 1344Γ768 or portrait+ 768Γ1344 (other sizes are resized and cropped to the canvas). Any representation works; the prompt names it.+- `first.png` (optional, recommended): the first frame, in the look you want. See [prompts and first frames](prompts.md).+- `prompt.txt`: the six-part prompt. See [prompts and first frames](prompts.md).++**3. Render.**++```bash+python scripts/render.py --h3 ./MiniMax-H3 \+ --control weights/solarrenderer_control_lora.safetensors \+ --fewstep weights/solarrenderer_4step_lora.safetensors \+ --structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4 --seed 0+```++Output: 158 frames at 24 fps. Encoding takes about 4 minutes and sampling about 3.5 minutes on one GPU.++## Stream a clip++The causal pair renders the same clip as a stream: the video is generated block by block, each block of at most 5+latents (about 20 frames) seeing the prompt, the first frame, the structure of its last 6 blocks and the frames already+generated for the 5 blocks before it, never a later block. Each block is sampled in four steps and then committed to a+key/value cache, so a block can be shown as soon as it is done. Download the causal LoRAs and add `--causal`:++```bash+hf download Enderfga/SolarRenderer solarrenderer_causal_control_lora.safetensors \+ solarrenderer_causal_4step_lora.safetensors --local-dir ./weights+python scripts/render.py --h3 ./MiniMax-H3 --causal \+ --control weights/solarrenderer_causal_control_lora.safetensors \+ --fewstep weights/solarrenderer_causal_4step_lora.safetensors \+ --structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4 --seed 0+```++The inputs and the output are the same as above.++## Render the benchmark++`eval/render_benchmark.py` renders every benchmark case with each model loaded once per GPU; see [`eval/README.md`](../eval/README.md).diff --git a/docs/prompts.md b/docs/prompts.mdnew file mode 100644index 0000000..c262760--- /dev/null+++ b/docs/prompts.md@@ -0,0 +1,41 @@+# Prompts and first frames+++SolarRenderer was trained on one six-part prompt form, and a prompt in that form is the single biggest lever on+quality; a short free-form sentence works much worse. [`skills/solarrenderer-caption/`](../skills/solarrenderer-caption)+is a standard agent skill (`SKILL.md` and three helpers) that teaches a coding agent such as Codex or Claude Code to+write it:++```+subject_definitions: what each input is (fixed sentences, from conditions.py)+summary: task type + one paragraph (who does what, how the camera moves)+retention_analysis: what is taken from each input (fixed sentences)+detailed_description: 350-500 words (materials, light, weather, characters, events, in order)+overall_soundscape: ambience+non_diegetic_music: N/A+```++Give the skill folder to your agent together with the structure video (and the first frame, if you have one), and+ask it to write `prompt.txt` in the look you want. It will:++1. watch the structure video: the number of characters, their facing, the layout and the camera path;+2. pick the structure sentence: one of the six trained types in `conditions.DEFINE` word for word, or, for any+ other structure, `custom_condition(looks_like, not_shown)`, a plain description of what the clip looks like and+ what of its look must not appear;+3. write the free parts, spending the words on what the structure leaves open;+4. assemble the prompt with `conditions.assemble(free, inputs, condition)` and validate it with+ `python skills/solarrenderer-caption/check.py prompt.txt`.++**No first frame?** The skill also covers making one with an image model (Codex's image generation, or any model+that edits a reference image). Frame 0 of the structure video is the reference; the image prompt keeps its camera,+layout, characters, poses and facing, and replaces the grey surfaces with the look the prompt describes:++```bash+F=skills/solarrenderer-caption/first_frame.py+python $F frame0 structure.mp4 frame0.png # the reference image+python $F prompt prompt.txt # the image prompt (--stylised for toon, pixel art, ...)+# generate a 16:9 image from frame0.png with that prompt, then:+python $F fit generated.png first.png # crop and scale to 1344x768+```++To restyle a clip, keep the structure video, write the prompt for the new world, and make a first frame in that world.diff --git a/docs/training.md b/docs/training.mdnew file mode 100644index 0000000..5bb2237--- /dev/null+++ b/docs/training.md@@ -0,0 +1,86 @@+# Training+++Training runs in three steps on one 8-GPU node with FSDP and sequence parallelism; the configurations in `configs/` are+set for it. The released weights were trained with the same recipe and the same number of samples at a larger global+batch. Training needs about 140 GB of memory per GPU. On one such node a control step takes about 84 s and a+post-training step 3 to 8 minutes, so the full schedules take weeks and days respectively. Replace the+`/path/to/...` entries before launching. Data and checkpoints can be local paths or remote object-storage+prefixes (`<scheme>://...`); remote ones are read and written through [fsspec](https://filesystem-spec.readthedocs.io),+so install the fsspec backend for your storage.++**1. Encode the training samples.** Each sample is one line of a JSONL file: the target clip, the prompt, and the+observations, each laid over the target frame by frame (`"all"`), on given latent frames (`[0]` for a first frame), or+not aligned (`null`):++```json+{"id": "clip_0001_full", "target": "clips/clip_0001.mp4", "text": "subject_definitions:\n<Video 1> is ...",+ "observations": [{"media": "structure/clip_0001.mp4", "frames": "all"},+ {"media": "first/clip_0001.png", "frames": [0]}]}+```++To train on the released data, download the shards and let `tools/hf_samples.py` write the clips and the sample+list (each clip with the inputs and prompt training gave it; `--per-source N` keeps N clips per source for a quick test):++```bash+hf download Enderfga/SolarRenderer-Data --repo-type dataset --include "train/*" --local-dir ./SolarRenderer-Data+python tools/hf_samples.py --data ./SolarRenderer-Data --out ./samples # writes ./samples/samples.jsonl+```++Each clip is one sample. Its inputs (structure and first frame, structure only, first frame only, or text only) are+fixed by a hash of its id, and the prompt carries the matching fixed sentences. Encoding runs one worker per GPU and writes `samples/` and one index per worker:++```bash+H3_REF_ROOT=./MiniMax-H3 torchrun --nproc-per-node 8 tools/omni_preencode.py --samples ./samples/samples.jsonl --out ./encoded+cat ./encoded/index.*.jsonl | sort -u > ./encoded/train.jsonl # workers help each other, so rows can repeat+```++The [`data`](https://github.com/Enderfga/SolarRenderer/tree/data) branch holds the data processing, cleaning and+captioning pipeline that produced our training samples; the data itself is on+[Hugging Face](https://huggingface.co/datasets/Enderfga/SolarRenderer-Data).++**2. Train the control LoRA** (flow matching, 38,400 steps at global batch 4):++```bash+torchrun --standalone --nproc-per-node 8 -m solarwm train --config configs/control_lora.yaml+```++**3. Post-train the four-step LoRA with structure guidance** (800 steps at global batch 4). Convert the public DMAD LoRA first,+then point `train.dmd.init_lora` at it and `train.dmd.control` at the control run's checkpoint directory from step 2 (the+one holding `adapter.pt`):++```bash+hf download ZhengmingYu/DMAD minimax_h3/dmad_minimax_h3_4step_lora_critic.safetensors --local-dir ./dmad+python scripts/convert_dmad.py --h3 ./MiniMax-H3 \+ --src ./dmad/minimax_h3/dmad_minimax_h3_4step_lora_critic.safetensors --out ./weights/dmad_4step_lora.safetensors+torchrun --standalone --nproc-per-node 8 -m solarwm train --config configs/structure_guided_4step.yaml+```++The structure guidance is `train.dmd.structure_guidance` (1 = plain distribution matching); the released weights use+1.5; take the live adapter at the last step. To render with your own runs, export each run's LoRA with+`python scripts/export_lora.py --checkpoint <checkpoint dir> --out <file>.safetensors`.++## Causal version++The causal LoRAs follow the same two stages on top of the bidirectional ones. As above, the released causal LoRAs+saw the same number of samples as the configurations below, at a larger batch.++**4. Train the causal control LoRA** (flow matching with block-causal attention and teacher forcing, 19,200 steps at+global batch 2). It starts from the bidirectional control LoRA: point `checkpoint.initialization.student.path` at the+control run's checkpoint directory from step 2.++```bash+torchrun --standalone --nproc-per-node 8 -m solarwm train --config configs/causal_control_lora.yaml+```++**5. Post-train the causal four-step LoRA with structure guidance** (1,600 steps at global batch 1). The causal control+LoRA from step 4 (`train.dmd.student_init`) is frozen in the student's base; the student is a separate four-step LoRA+that starts as the bidirectional one (`train.dmd.accel_init`) and is distilled on its own block-by-block streams, with+the bidirectional control LoRA from step 2 (`train.dmd.control`) as the teacher:++```bash+torchrun --standalone --nproc-per-node 8 -m solarwm train --config configs/causal_structure_guided_4step.yaml+```++Export both runs with `scripts/export_lora.py` and render with `scripts/render.py --causal`.+diff --git a/eval/README.md b/eval/README.mdnew file mode 100644index 0000000..57694f8--- /dev/null+++ b/eval/README.md@@ -0,0 +1,146 @@+# Scoring the SolarRenderer benchmark++The benchmark has two splits, scored with the same metrics and the same judge. Every case gives a structure video, a+first frame and a prompt:++| Split | Cases | First frame | Ground truth |+|---|---|---|---|+| `hard` | 208 cases over 16 structure types, 11 never seen in training (74 cases, tag `unseen condition`), tagged for large motion, fast camera, stylized looks, night, multi-person and unseen condition | made by an image model from the structure's frame 0 | none |+| `heldout` | 53 held-out clips from CARLA, engine recordings with near-range depth, Magpie and UnrealZoo | the real one | the real clip |++Scoring has three parts:++1. **Automatic metrics** (`metrics.py`), in four groups:+ - quality: DOVER, LAION aesthetic and VideoScore2;+ - stability: low-pass flicker and DINOv2 adjacent-frame similarity;+ - structure: CoTracker3 motion error and edge F1 against the structure video;+ - first frame: DINOv2 similarity to the first frame over the last second.++ On the held-out split, `gt_metrics.py` adds DISTS, LPIPS, PSNR, SSIM and sharpness against the real clip.+ `summary.py` gives the mean of every reading and the **automatic score** (0β100, the four groups).+2. **Pairwise VLM judge** (`judge.py`): Gemini compares two methods on every case in both orders and answers four+ questions: visual quality, structure and motion, first frame, overall. The **net score** is wins minus losses over+ cases.+3. **Elo** (`elo.py`): a BradleyβTerry fit of every judged pair on a split gives one rating per method.++The released results (`results/` in the dataset) hold the SolarRenderer renders of both splits and every verdict+behind the paper's Elo table, so a new method only needs its own renders and its comparison with SolarRenderer.++## Setup++```bash+hf download Enderfga/SolarRenderer-Data --repo-type dataset --include "benchmark/*" --local-dir ./SolarRenderer-Data+bash eval/setup.sh # DOVER, the LAION aesthetic head, CoTracker3 (pinned versions, checksummed)+```++VideoScore2 needs its own environment, because it requires a different transformers version:++```bash+python -m venv vs2 && vs2/bin/pip install "torch==2.8.*" "torchvision==0.23.*" "transformers==4.53.2" "qwen-vl-utils==0.0.14" accelerate av decord+```++## 1. Render++Put a method's renders in one directory:++```+<renders>/hard/<case_id>.mp4+<renders>/heldout/<clip_id>.mp4 e.g. heldout/carla/cc000014066.mp4+```++Render each case from its structure video, first frame and prompt. Each clip should be 24 fps; the published renders are 1344Γ768 (768Γ1344 portrait),+158 frames. For SolarRenderer itself:++```bash+CUDA_VISIBLE_DEVICES=0 python eval/render_benchmark.py --bench ./SolarRenderer-Data --out ./renders/solarrenderer \+ --h3 ./MiniMax-H3 --control weights/solarrenderer_control_lora.safetensors \+ --fewstep weights/solarrenderer_4step_lora.safetensors --shard 0 --shards 8+```++Run one process per GPU with `--shard k --shards n`.++## 2. Automatic metrics++```bash+CUDA_VISIBLE_DEVICES=0 python eval/metrics.py --bench ./SolarRenderer-Data --renders ./renders/solarrenderer --method solarrenderer \+ --out ./scores --shard 0 --shards 8+CUDA_VISIBLE_DEVICES=0 vs2/bin/python eval/metrics.py --bench ./SolarRenderer-Data --renders ./renders/solarrenderer \+ --method solarrenderer --out ./scores --metrics videoscore2 --shard 0 --shards 8+python eval/gt_metrics.py --bench ./SolarRenderer-Data --renders ./renders/solarrenderer --method solarrenderer --out ./scores++python eval/summary.py --scores ./scores --methods solarrenderer,other --split hard+python eval/summary.py --scores ./scores --methods solarrenderer,other --split hard --tag "unseen condition"+python eval/summary.py --scores ./scores --methods solarrenderer,other --split heldout+```++On the released renders (`results/solarrenderer`) these commands give the paper's means; DOVER and VideoScore2 depend+slightly on library versions (VideoScore2 generates its answer, so expect differences of about 0.02).++Results are one JSON per case and metric, under `<out>/<metric>/<method>/hard/<case_id>.json` and+`<out>/<metric>/<method>/heldout/<clip_id>.json`. A rerun only computes what is missing.++The automatic score maps each reading linearly from a fixed range, so it does not depend on the other methods:++| Reading | Range | Group |+|---|---|---|+| DOVER overall | [0, 1] | quality |+| LAION aesthetic | [1, 10] | quality |+| VideoScore2 visual quality | [1, 5] | quality |+| low-pass flicker | [0, 10], reversed, clipped at 0 | stability |+| DINOv2 adjacent-frame similarity | [0, 1] | stability |+| motion error | [0, 0.5], reversed, clipped at 0 | structure |+| edge F1 | [0, 1] | structure |+| DINOv2 similarity to the first frame | [0, 1] | first frame |++The readings of each group are averaged, and the score is the mean of the four groups, on the cases every listed+method has. Ground-truth readings are listed but not scored.++## 3. Pairwise judge++```bash+export GEMINI_API_KEY=...+python eval/judge.py run --bench ./SolarRenderer-Data --a ./renders/solarrenderer --b ./renders/other \+ --name-a solarrenderer --name-b other --out ./judge/solarrenderer__vs__other.jsonl+python eval/judge.py report ./judge/solarrenderer__vs__other.jsonl+```++How it judges:++- **Model**: `gemini-3.8-flash` at temperature 0. The judge sees the structure video, the first frame, video A and+ video B, compressed as described at the top of `judge.py`.+- **Two orders**: every case is judged twice, with A and B swapped. A case is a win for X when both orders pick X, or+ one picks X and the other is a tie. When each order picks its own position, the case counts as a tie.+- **Net score**: wins minus losses, divided by the number of cases (from β1 to 1). `*` marks a 95% bootstrap interval+ that excludes zero. On the hard split the interval resamples whole structure videos (2,000 draws), because each+ structure video is rendered in several looks; held-out clips are resampled one by one (10,000 draws).+- **Counts**: the report says how many calls were answered and failed, and how many cases have only one order and+ are left out.+- **Report**: given for each split and for each hard-split tag.++A rerun retries calls that were rate-limited. Even at temperature 0, a few answers can change from run to run, so+compare methods within one run.++## 4. Elo++```bash+hf download Enderfga/SolarRenderer-Data --repo-type dataset --include "results/*" --local-dir ./SolarRenderer-Data+python eval/judge.py run --bench ./SolarRenderer-Data --a ./SolarRenderer-Data/results/solarrenderer --b ./renders/mine \+ --name-a solarrenderer --name-b mine --out ./judge/solarrenderer__vs__mine.jsonl+python eval/elo.py ./SolarRenderer-Data/results/judge/hard/*.jsonl ./judge/solarrenderer__vs__mine.jsonl --split hard+python eval/elo.py ./SolarRenderer-Data/results/judge/hard/*.jsonl ./judge/solarrenderer__vs__mine.jsonl --split hard \+ --tag "unseen condition"+python eval/elo.py ./SolarRenderer-Data/results/judge/heldout/*.jsonl ./judge/solarrenderer__vs__mine.jsonl --split heldout+```++`elo.py` fits a BradleyβTerry model to the overall verdicts (`--question` picks another) of every pair in the given+files, with a tie counting half a win to each side, and reports each method on the Elo scale: 400 points per factor of+ten in odds, mean of the rated methods 1000. Every method also plays one virtual tie against a fixed reference, so a+method that loses every comparison still gets a finite rating. The interval resamples the same units as the net+score, 400 draws. `--methods` rates only the listed methods, from the comparisons among them; on the released+verdicts, the nine methods of the paper's main table give its Elo columns:++```bash+python eval/elo.py ./SolarRenderer-Data/results/judge/hard/*.jsonl --split hard --methods solarrenderer,lynnreal_omni,\+control_model_50step,official_backbone_50step,pai_controlnet_union,streaming_cwm,streamrender_h3,cosmos_transfer2_5,\+wan2_2_vace_fun+```diff --git a/eval/bench.py b/eval/bench.pynew file mode 100644index 0000000..a76b0b2--- /dev/null+++ b/eval/bench.py@@ -0,0 +1,59 @@+"""The SolarRenderer benchmark as a list of cases, read from a local copy of the Hugging Face dataset.++ hf download Enderfga/SolarRenderer-Data --repo-type dataset --include "benchmark/*" --local-dir ./SolarRenderer-Data++Two splits, one case format; every case gives a structure video, a first frame and a prompt:+ hard benchmark/hard/cases.jsonl: 208 cases, the first frame made by an image model, no ground truth+ heldout benchmark/heldout/cases.jsonl: 53 held-out clips with their real first frame and the real video++A method's renders live in one directory: <renders>/hard/<case_id>.mp4 and <renders>/heldout/<clip_id>.mp4.+"""++from __future__ import annotations++import json+from dataclasses import dataclass, field+from pathlib import Path++@dataclass+class Case:+ split: str # hard | heldout+ id: str # case_id (hard) or clip_id (heldout)+ prompt: str+ condition: str # the structure video+ first_frame: str+ ground_truth: str | None # held-out clips only+ tags: list[str] = field(default_factory=list)+ kind: str | None = None # the structure type++ @property+ def key(self) -> str:+ return self.id++ def render(self, renders: str | Path) -> Path:+ return Path(renders) / self.split / f"{self.id}.mp4"+++def _rows(path: Path) -> list[dict]:+ return [json.loads(line) for line in path.read_text().splitlines() if line.strip()]+++def load(root: str | Path, split: str = "all") -> list[Case]:+ """Every case of the split (hard, heldout or all) under a local copy of the dataset."""++ root = Path(root)+ cases: list[Case] = []+ if split in ("hard", "all"):+ for r in _rows(root / "benchmark" / "hard" / "cases.jsonl"):+ files = r["files"]+ prompt = (root / files["prompt.txt"]).read_text().strip()+ cases.append(Case("hard", r["case_id"], prompt, str(root / files["condition.mp4"]),+ str(root / files["first_frame.png"]), None, list(r.get("tags") or []),+ r.get("condition")))+ if split in ("heldout", "all"):+ for r in _rows(root / "benchmark" / "heldout" / "cases.jsonl"):+ files = r["files"]+ cases.append(Case("heldout", r["clip_id"], r["prompt"], str(root / files["condition.mp4"]),+ str(root / files["first_frame.png"]), str(root / r["ground_truth"]), [],+ r.get("condition")))+ return casesdiff --git a/eval/dover_score.py b/eval/dover_score.pynew file mode 100644index 0000000..4d10cbf--- /dev/null+++ b/eval/dover_score.py@@ -0,0 +1,85 @@+"""DOVER video quality (Wu et al., ICCV 2023), as eval/metrics.py reads it.++Code: github.com/VQAssessment/DOVER at f1ddc96 (eval/setup.sh clones it to $SOLAR_EVAL_MODELS/DOVER), weights: the+v0.1.0 release DOVER.pth. Preprocessing is the repo's own: the "val-l1080p" sample_types of dover.yml (technical: 7x7+fragments of 32x32, 3 clips x 32 frames, interval 2; aesthetic: 224x224 resize, 32 frames over the clip), the+UnifiedFrameSampler set-up and per-view normalisation of evaluate_one_video.py. Each branch's raw score is the mean of+its head output. numpy (the temporal offsets) and torch (the fragment positions) are seeded with 0 before every video,+so a video always gets the same score. The published numbers seeded numpy only, so their fragment positions came from+torch's running state: a clip's published DOVER can differ from this one by about 0.01-0.02, which averages out over a+split.++Scores (all in (0, 1), higher is better), the repo's score-level fusion with each (mean, std) pair applied to the+branch it was computed from:+ t = (raw_technical + 0.08285) / 0.03774 technical = sigmoid(t)+ a = (raw_aesthetic - 0.1107) / 0.07355 aesthetic = sigmoid(a)+ overall = sigmoid(0.6104 * t + 0.3896 * a)+At f1ddc96 both repo scripts normalise the technical output with (0.1107, 0.07355) and the aesthetic output with+(-0.08285, 0.03774), but the repo's own LIVE-VQC predictions give 0.1105/0.0735 for the aesthetic head and+-0.0827/0.0377 for the technical head: the pairs are swapped there. Verbatim, technical sits at ~0.05-0.09 for every+video and aesthetic saturates near 1.+"""+import math+import os+import sys+from pathlib import Path++import numpy as np+import torch+import yaml++MODELS = Path(os.environ.get("SOLAR_EVAL_MODELS", Path(__file__).resolve().parent.parent / "eval_models"))+DOVER_REPO = os.environ.get("DOVER_REPO", str(MODELS / "DOVER"))+DOVER_CKPT = os.environ.get("DOVER_CKPT", str(MODELS / "DOVER" / "pretrained_weights" / "DOVER.pth"))+sys.path.insert(0, DOVER_REPO)+os.environ.setdefault("TORCH_HOME", str(MODELS / "torchhub")) # the aesthetic branch's ImageNet init, before DOVER.pth++MEAN = torch.FloatTensor([123.675, 116.28, 103.53])+STD = torch.FloatTensor([58.395, 57.12, 57.375])+++def sigmoid(x):+ return 1.0 / (1.0 + math.exp(-x))+++def fuse(raw):+ t = (raw["technical"] + 0.08285) / 0.03774+ a = (raw["aesthetic"] - 0.1107) / 0.07355+ return {"aesthetic": sigmoid(a), "technical": sigmoid(t), "overall": sigmoid(0.6104 * t + 0.3896 * a)}+++def build(device):+ from dover.datasets import UnifiedFrameSampler+ from dover.models import DOVER++ with open(os.path.join(DOVER_REPO, "dover.yml")) as f:+ opt = yaml.safe_load(f)+ model = DOVER(**opt["model"]["args"]).to(device)+ model.load_state_dict(torch.load(DOVER_CKPT, map_location=device))+ model.eval()+ stypes = opt["data"]["val-l1080p"]["args"]["sample_types"]+ samplers = {}+ for k, s in stypes.items(): # same as evaluate_one_video.py+ if "t_frag" not in s:+ samplers[k] = UnifiedFrameSampler(s["clip_len"], s["num_clips"], s["frame_interval"])+ else:+ samplers[k] = UnifiedFrameSampler(s["clip_len"] // s["t_frag"], s["t_frag"],+ s["frame_interval"], s["num_clips"])+ return model, stypes, samplers++[email protected]_grad()+def score(path, model, stypes, samplers, device, seed):+ from dover.datasets import spatial_temporal_view_decomposition++ np.random.seed(seed)+ torch.manual_seed(seed)+ views, _ = spatial_temporal_view_decomposition(path, stypes, samplers)+ keys = list(views)+ for k in keys:+ v = views[k]+ nc = stypes[k].get("num_clips", 1)+ views[k] = (((v.permute(1, 2, 3, 0) - MEAN) / STD).permute(3, 0, 1, 2)+ .reshape(v.shape[0], nc, -1, *v.shape[2:]).transpose(0, 1).to(device))+ outs = model(views) # one score map per view, in the order of `views`+ return {k: o.mean().item() for k, o in zip(keys, outs)}diff --git a/eval/elo.py b/eval/elo.pynew file mode 100644index 0000000..203b779--- /dev/null+++ b/eval/elo.py@@ -0,0 +1,109 @@+#!/usr/bin/env python+"""Elo ratings from the pairwise judge: one rating per method from every verdict between any two methods.++ python eval/elo.py ./judge/*.jsonl [--split hard|heldout] [--tag "unseen condition"] [--question q4_overall]++Reads the outputs of eval/judge.py run (any set of method pairs, each judged in both orders) and fits a Bradley-Terry+model to the per-case verdicts of the chosen question (both-order rule of eval/judge.py; a tie counts half a win to+each side). Strengths are reported on the Elo scale: 400 points per factor of ten in odds, mean of the rated methods+1000. Every method also plays one virtual tie against a fixed reference player, so a method that loses every+comparison still gets a finite rating (methods with the same record share one rating). The 95% interval resamples+whole structure videos on the hard split and clips on the held-out split, 400 draws.++To add a method to the published table, judge its renders against the released SolarRenderer renders (or any rated+method) on the same split and fit again with the released verdicts.+"""+from __future__ import annotations++import argparse+import collections+import json+import math+import random+import sys+from pathlib import Path++sys.path.insert(0, str(Path(__file__).resolve().parent))+from judge import Q, segment, verdict # noqa: E402+++def games(files: list[str], split: str, tag: str | None, question: str, methods: set[str] | None = None):+ """(unit, a, b, winner or 'tie') for every case of the split judged in both orders, between two of ``methods``+ (all methods if None)."""+ by = collections.defaultdict(dict)+ for f in files:+ for line in open(f):+ r = json.loads(line)+ if methods and not set(r["pair"]) <= methods:+ continue+ if "answer" in r and r["split"] == split and (not tag or tag in (r.get("tags") or [])):+ by[(r["case"], tuple(r["pair"]))][r["swap"]] = r+ return [(segment(c), p[0], p[1], verdict(rs, question)) for (c, p), rs in by.items() if len(rs) == 2]+++def fit(gs) -> dict[str, float]:+ """Bradley-Terry strengths by minorization-maximization, with one virtual tie per method against a reference of+ strength 1; returned on the Elo scale with the rated methods' mean at 1000."""+ methods = sorted({g[1] for g in gs} | {g[2] for g in gs})+ wins, n = collections.Counter(), collections.Counter()+ for _, a, b, w in gs:+ n[a, b] += 1+ if w == a:+ wins[a, b] += 1+ elif w == b:+ wins[b, a] += 1+ else:+ wins[a, b] += 0.5+ wins[b, a] += 0.5+ p = {m: 1.0 for m in methods}+ for _ in range(2000):+ new = {}+ for i in methods:+ w = sum(wins[i, j] for j in methods if j != i) + 0.5+ d = sum((n[i, j] + n[j, i]) / (p[i] + p[j]) for j in methods if j != i) + 1 / (p[i] + 1)+ new[i] = w / d+ step = max(abs(math.log(new[m] / p[m])) for m in methods)+ p = new+ if step < 1e-7:+ break+ g = math.exp(sum(math.log(v) for v in p.values()) / len(p))+ return {m: 1000 + 400 * math.log10(p[m] / g) for m in methods}+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("results", nargs="+", help="outputs of eval/judge.py run")+ ap.add_argument("--split", default="hard", choices=("hard", "heldout"))+ ap.add_argument("--tag", help='hard split only: rate on the cases with this tag, e.g. "unseen condition"')+ ap.add_argument("--question", default="q4_overall", choices=Q)+ ap.add_argument("--methods", help="comma list: rate only these methods, from the comparisons among them")+ ap.add_argument("--draws", type=int, default=400)+ ap.add_argument("--json", help="also write the ratings here")+ a = ap.parse_args()+ gs = games(a.results, a.split, a.tag, a.question, set(a.methods.split(",")) if a.methods else None)+ if not gs:+ sys.exit("no case judged in both orders on this split")+ point = fit(gs)+ by = collections.defaultdict(list)+ for g in gs:+ by[g[0]].append(g)+ units = list(by.values())+ rng, boot = random.Random(0), collections.defaultdict(list)+ for _ in range(a.draws):+ for m, v in fit([g for _ in units for g in rng.choice(units)]).items():+ boot[m].append(v)+ def interval(xs): # a method absent from a draw (judged on few structure videos) has fewer values+ xs = sorted(xs)+ return xs[int(0.025 * len(xs))], xs[max(0, int(0.975 * len(xs)) - 1)]++ rows = {m: (point[m], *interval(boot[m])) for m in point}+ label = a.split + (f", {a.tag}" if a.tag else "")+ print(f"{label}: {a.question}, {len(gs)} judged cases, {len(units)} resampling units")+ for m, (e, l, h) in sorted(rows.items(), key=lambda x: -x[1][0]):+ print(f" {m:24s} {e:6.0f} [{l:.0f}, {h:.0f}]")+ if a.json:+ Path(a.json).write_text(json.dumps({m: {"elo": e, "lo": l, "hi": h} for m, (e, l, h) in rows.items()}, indent=1))+++if __name__ == "__main__":+ main()diff --git a/eval/gt_metrics.py b/eval/gt_metrics.pynew file mode 100644index 0000000..97957f4--- /dev/null+++ b/eval/gt_metrics.py@@ -0,0 +1,179 @@+#!/usr/bin/env python+"""Pixel metrics against the ground truth, for the held-out split.++ python eval/gt_metrics.py --bench ./SolarRenderer-Data --renders ./renders/mymethod --method mymethod --out ./scores++Writes <out>/gt/<method>/heldout/<clip_id>.json. Render and ground truth are decoded at the ground truth's size+and cut to the shorter. Readings (lower is closer for DISTS / LPIPS, higher for PSNR / SSIM):+ dists_real / lpips_real DISTS and LPIPS (AlexNet), 4 fps, 672 px wide, mean over frames+ dists_hr / lpips_hr the same on the four full-size quarters of every frame (2x2 tiles of 672x384)+ psnr per-frame 10 log10(255^2 / MSE) on full-size RGB, all frames, averaged+ ssim per-pixel greyscale SSIM (11x11 Gaussian, sigma 1.5), all frames, averaged+ sharp_log / sharp_log_hr log of (mean Laplacian variance of the render / of the ground truth), grey, 8 fps,+ 672 px and full size: 0 is the ground truth's sharpness+Each reading is also given without frame 0 (suffix _nof0): frame 0 is the given first frame, so it only measures how+faithfully that frame is copied.+"""+from __future__ import annotations++import argparse+import json+import math+import subprocess+import sys+import traceback+from pathlib import Path++import numpy as np+import torch++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import bench # noqa: E402++W = 672+++def size_of(path):+ probe = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",+ "-of", "csv=p=0", path], capture_output=True, text=True).stdout.strip().split(",")+ return int(probe[0]), int(probe[1])+++def decode(path, fps, gray, size):+ """Frames at fps over the whole clip at size (w, h): uint8 [T,H,W] (grey) or [T,H,W,3]."""+ w, h = size+ chain = f"fps={fps},scale={w}:{h}:flags=area,format={'gray' if gray else 'rgb24'}"+ raw = subprocess.run(["ffmpeg", "-nostdin", "-loglevel", "error", "-i", path, "-vf", chain, "-f", "rawvideo", "-"],+ capture_output=True, check=True).stdout+ c = 1 if gray else 3+ a = np.frombuffer(raw, np.uint8)+ n = len(a) // (w * h * c)+ return a[: n * w * h * c].reshape(n, h, w) if gray else a[: n * w * h * c].reshape(n, h, w, 3)+++def sizes(real):+ w, h = size_of(real)+ return (W, int(round(h * W / w / 2) * 2)), (w, h)+++def sharpness(gray):+ f = gray.astype(np.float32)+ lap = -4 * f[:, 1:-1, 1:-1] + f[:, :-2, 1:-1] + f[:, 2:, 1:-1] + f[:, 1:-1, :-2] + f[:, 1:-1, 2:]+ return float(lap.var(axis=(1, 2)).mean())+++def to_tensor(rgb, device):+ return torch.from_numpy(rgb).to(device).permute(0, 3, 1, 2).float().div(255.0)++[email protected]_grad()+def distances(a, b, metrics, device, tiles=False):+ n = min(len(a), len(b))+ a, b = a[:n], b[:n]+ if tiles: # the four quarters of every frame, each as its own picture+ h, w = a.shape[1] // 2, a.shape[2] // 2+ cut = lambda v: np.concatenate([v[:, i * h:(i + 1) * h, j * w:(j + 1) * w] for i in range(2) for j in range(2)]) # noqa: E731+ a, b = cut(a), cut(b)+ out = {}+ for name, m in metrics.items():+ vals = []+ for i in range(0, len(a), 8):+ x, y = to_tensor(np.ascontiguousarray(a[i:i + 8]), device), to_tensor(np.ascontiguousarray(b[i:i + 8]), device)+ vals.append(m(x, y).flatten().float().cpu())+ out[name] = float(torch.cat(vals).mean())+ return out+++def real_readings(real):+ lo, hi = sizes(real)+ return {"lo": lo, "hi": hi, "sharp": sharpness(decode(real, 8, True, lo)),+ "sharp_hr": sharpness(decode(real, 8, True, hi)),+ "rgb": decode(real, 4, False, lo), "rgb_hr": decode(real, 4, False, hi)}+++def _gaussian(window: int = 11, sigma: float = 1.5) -> torch.Tensor:+ axis = torch.arange(window, dtype=torch.float32) - (window - 1) / 2+ kernel = torch.exp(-(axis**2) / (2 * sigma**2))+ kernel = kernel / kernel.sum()+ return (kernel[:, None] @ kernel[None, :])[None, None]+++def ssim(made: torch.Tensor, want: torch.Tensor) -> torch.Tensor:+ window = _gaussian().to(made.device, made.dtype)+ pad = window.shape[-1] // 2+ blur = lambda x: torch.nn.functional.conv2d(x, window, padding=pad) # noqa: E731+ mu_a, mu_b = blur(made), blur(want)+ var_a = blur(made * made) - mu_a * mu_a+ var_b = blur(want * want) - mu_b * mu_b+ covariance = blur(made * want) - mu_a * mu_b+ c1, c2 = (0.01 * 255) ** 2, (0.03 * 255) ** 2+ return ((2 * mu_a * mu_b + c1) * (2 * covariance + c2)) / ((mu_a**2 + mu_b**2 + c1) * (var_a + var_b + c2))+++def psnr(a, b):+ mse = float(((a - b) ** 2).mean())+ return 10 * np.log10(255.0 ** 2 / max(mse, 1e-6))+++def pixel(made, real, device):+ hi = size_of(real)+ a, b = decode(made, 24, False, hi), decode(real, 24, False, hi)+ n = min(len(a), len(b))+ ps, ss = [], []+ w = torch.tensor([0.299, 0.587, 0.114], device=device)+ for k in range(n):+ ps.append(float(psnr(a[k].astype(np.float32), b[k].astype(np.float32))))+ x = torch.from_numpy(a[k]).to(device).float() @ w+ y = torch.from_numpy(b[k]).to(device).float() @ w+ ss.append(float(ssim(x[None, None], y[None, None]).mean()))+ return {"psnr": float(np.mean(ps)), "psnr_nof0": float(np.mean(ps[1:])), "psnr_frame0": ps[0],+ "ssim": float(np.mean(ss)), "ssim_nof0": float(np.mean(ss[1:])), "ssim_frame0": ss[0], "frames": n}+++def quality(made, R, metrics, device):+ lo, hi = decode(made, 4, False, R["lo"]), decode(made, 4, False, R["hi"])+ d = distances(lo, R["rgb"], metrics, device)+ dh = distances(hi, R["rgb_hr"], metrics, device, tiles=True)+ d0 = distances(lo[1:], R["rgb"][1:], metrics, device)+ dh0 = distances(hi[1:], R["rgb_hr"][1:], metrics, device, tiles=True)+ return {"dists_real": d["dists"], "dists_hr": dh["dists"], "dists_real_nof0": d0["dists"],+ "dists_hr_nof0": dh0["dists"], "lpips_real": d["lpips"], "lpips_hr": dh["lpips"],+ "lpips_real_nof0": d0["lpips"], "lpips_hr_nof0": dh0["lpips"],+ "sharp_log": math.log(sharpness(decode(made, 8, True, R["lo"])) / R["sharp"]),+ "sharp_log_hr": math.log(sharpness(decode(made, 8, True, R["hi"])) / R["sharp_hr"])}+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--bench", required=True)+ ap.add_argument("--renders", required=True)+ ap.add_argument("--method", required=True)+ ap.add_argument("--out", required=True)+ ap.add_argument("--shard", type=int, default=0)+ ap.add_argument("--shards", type=int, default=1)+ ap.add_argument("--device", default="cuda:0")+ a = ap.parse_args()+ import pyiqa++ device = torch.device(a.device)+ metrics = {"dists": pyiqa.create_metric("dists", device=device, as_loss=False),+ "lpips": pyiqa.create_metric("lpips", device=device, as_loss=False)}+ for c in bench.load(a.bench, "heldout")[a.shard::a.shards]:+ out = Path(a.out) / "gt" / a.method / "heldout" / f"{c.key}.json"+ made = c.render(a.renders)+ if out.exists() or not made.exists():+ continue+ try:+ res = {**quality(str(made), real_readings(c.ground_truth), metrics, device),+ **pixel(str(made), c.ground_truth, device)}+ except Exception:+ print(f"SKIP {c.key}\n{traceback.format_exc()[-600:]}", flush=True)+ continue+ out.parent.mkdir(parents=True, exist_ok=True)+ out.write_text(json.dumps({"id": c.id, "method": a.method, **res}))+ print(f"{c.key} dists_hr={res['dists_hr']:.4f} psnr={res['psnr']:.2f}", flush=True)+++if __name__ == "__main__":+ main()diff --git a/eval/judge.py b/eval/judge.pynew file mode 100644index 0000000..a17826e--- /dev/null+++ b/eval/judge.py@@ -0,0 +1,264 @@+#!/usr/bin/env python+"""Pairwise VLM judge of the SolarRenderer benchmark: two methods, every case both have, both orders.++ export GEMINI_API_KEY=...+ python eval/judge.py run --bench ./SolarRenderer-Data --a ./renders/mine --b ./renders/other \+ --name-a mine --name-b other --out ./judge/mine_vs_other.jsonl [--split hard|heldout|all] [--threads 8]+ python eval/judge.py report ./judge/mine_vs_other.jsonl [more.jsonl ...]++The judge (gemini-3.8-flash, temperature 0, through the Gemini API) sees, in order, the structure video, the first+frame, video A and video B, and answers four questions with A, B or tie: q1 visual quality (the two videos on their+own), q2 structure and motion faithfulness, q3 first-frame appearance kept, q4 overall. Every case is judged twice+with A and B swapped; a case is a win for X when both orders pick X, or one picks X and the other ties; opposite picks+(each order choosing its own position) count as a tie. The net score is win rate minus loss rate, with a bootstrap+95% interval. eval/elo.py fits Elo ratings from the verdicts of several methods.++Videos are sent as they were for the published numbers: 672x384 (384x672 portrait), 12 fps, H.264 from the original+at CRF 30, raised in steps of 3 (up to 45) while over 0.7 MB, no audio; the first frame as a 672x384 JPEG. They are cached under <out>.media/.+Results are appended to <out> as they arrive; a rerun skips what is answered (rate-limited calls are retried then).+"""+from __future__ import annotations++import argparse+import base64+import collections+import json+import os+import random+import subprocess+import sys+import threading+import time+import urllib.request+from concurrent.futures import ThreadPoolExecutor+from pathlib import Path++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import bench # noqa: E402++MODEL = os.environ.get("JUDGE_MODEL", "gemini-3.8-flash")+URL = "https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"+Q = ["q1_visual_quality", "q2_structure_motion", "q3_first_frame", "q4_overall"]+PROMPT = """You are an expert judge of AI-generated video for film and game rendering.+Inputs, in order: (1) REFERENCE STRUCTURE - a rough proxy render (grey/flat/schematic) that fixes layout, object and character motion and camera for every frame; (2) FIRST FRAME - the image the generated video must start from and keep the look of; (3) VIDEO A; (4) VIDEO B. Both A and B were generated from the same reference structure, first frame and text prompt.+Compare A and B. Look closely at fine detail, textures, faces/hands/limbs, edges, and frame-to-frame stability (flicker, warping, smearing, melting, popping). Ignore that the reference structure itself looks crude - judge how well each video follows its layout, motion timing and camera.+The order of A and B is random; do not prefer either position. Answer each question with "A", "B" or "tie" (use tie only if you genuinely cannot tell):+q1_visual_quality: judging the two videos ON THEIR OWN (ignore the first frame and the reference here - a video that departs from the first frame's look can still win this question), which has higher visual quality: sharper fine detail and texture, fewer artifacts (blur, smearing, melting, distorted bodies/faces), better temporal stability (no flicker or warping)?+q2_structure_motion: which follows the reference structure's layout, motion and camera more faithfully over the whole clip?+q3_first_frame: which better keeps the first frame's appearance throughout - colours, lighting, contrast, materials, identities and style - without drifting?+q4_overall: which would you rather ship as the final rendered shot?+Return only JSON: {"q1_visual_quality":"A|B|tie","q2_structure_motion":"A|B|tie","q3_first_frame":"A|B|tie","q4_overall":"A|B|tie","reason":"<=40 words on the decisive differences"}"""+++# ---------------------------------------------------------------------------------------------- media++def _size(path: str) -> tuple[int, int]:+ out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",+ "-of", "csv=p=0", path], capture_output=True, text=True).stdout.strip().split(",")+ return int(out[0]), int(out[1])+++def small_video(src: str, dst: Path) -> Path:+ """672x384 (384x672 portrait), 12 fps, H.264 from the original at CRF 30, raised in steps of 3 (up to 45) while+ the file is over 0.7 MB."""+ if not dst.exists():+ dst.parent.mkdir(parents=True, exist_ok=True)+ w, h = _size(src)+ scale = "384:672" if h > w else "672:384"+ tmp = dst.with_suffix(".tmp.mp4")+ crf = 30+ while True:+ subprocess.run(["ffmpeg", "-loglevel", "error", "-y", "-i", src, "-vf", f"fps=12,scale={scale}:flags=lanczos,format=yuv420p",+ "-c:v", "libx264", "-crf", str(crf), "-an", "-movflags", "+faststart", str(tmp)], check=True)+ if tmp.stat().st_size <= 700_000 or crf >= 45:+ break+ crf += 3+ tmp.rename(dst)+ return dst+++def small_image(src: str, dst: Path) -> Path:+ if not dst.exists():+ dst.parent.mkdir(parents=True, exist_ok=True)+ from PIL import Image+ w, h = Image.open(src).size+ scale = "384:672" if h > w else "672:384"+ subprocess.run(["ffmpeg", "-loglevel", "error", "-y", "-i", src, "-vf", f"scale={scale}:flags=lanczos", "-q:v", "3", "-frames:v", "1",+ str(dst)], check=True)+ return dst+++# ---------------------------------------------------------------------------------------------- judge++def part(path: str) -> dict:+ mime = "video/mp4" if path.endswith(".mp4") else "image/jpeg"+ return {"inline_data": {"mime_type": mime, "data": base64.b64encode(open(path, "rb").read()).decode()}}+++def ask(row: dict, key: str) -> dict:+ parts = [{"text": PROMPT}, {"text": "REFERENCE STRUCTURE:"}, part(row["cond"]), {"text": "FIRST FRAME:"},+ part(row["first"])]+ parts += [{"text": "VIDEO A:"}, part(row["a"]), {"text": "VIDEO B:"}, part(row["b"])]+ body = {"contents": [{"role": "user", "parts": parts}],+ "generationConfig": {"temperature": 0, "maxOutputTokens": 4000}}+ err = ""+ for attempt in range(4):+ try:+ req = urllib.request.Request(URL.format(model=MODEL), data=json.dumps(body).encode(),+ headers={"x-goog-api-key": key, "Content-Type": "application/json"})+ d = json.load(urllib.request.urlopen(req, timeout=300))+ txt = "".join(p.get("text", "") for p in d["candidates"][0]["content"]["parts"])+ ans = json.loads(txt[txt.find("{"): txt.rfind("}") + 1])+ return {**row, "answer": ans, "usage": d.get("usageMetadata")}+ except Exception as e: # rate limits and transient errors: back off, then record the error+ err = str(e)[:200]+ time.sleep(5 * (attempt + 1))+ return {**row, "error": err}+++def pairs(a: argparse.Namespace, media: Path) -> list[dict]:+ rows = []+ for c in bench.load(a.bench, a.split):+ ra, rb = c.render(a.a), c.render(a.b)+ if not (ra.exists() and rb.exists()):+ continue+ tag = c.key.replace("/", "__")+ sa = str(small_video(str(ra), media / a.name_a / f"{tag}.mp4"))+ sb = str(small_video(str(rb), media / a.name_b / f"{tag}.mp4"))+ cond = str(small_video(c.condition, media / "condition" / f"{tag}.mp4"))+ first = str(small_image(c.first_frame, media / "first" / f"{tag}.jpg"))+ for swap in (0, 1):+ (x, px), (y, py) = ((a.name_a, sa), (a.name_b, sb)) if not swap else ((a.name_b, sb), (a.name_a, sa))+ rows.append({"key": f"{c.split}/{c.key}|{a.name_a}|{a.name_b}|{swap}", "case": f"{c.split}/{c.key}",+ "split": c.split, "tags": c.tags, "a": px, "b": py, "cond": cond,+ "first": first, "pair": [a.name_a, a.name_b], "swap": swap, "a_m": x, "b_m": y})+ return rows+++def run(a: argparse.Namespace) -> None:+ key = os.environ.get("GEMINI_API_KEY")+ if not key:+ sys.exit("set GEMINI_API_KEY")+ out = Path(a.out)+ out.parent.mkdir(parents=True, exist_ok=True)+ media = Path(str(out) + ".media")+ rows = pairs(a, media)+ done = set()+ if out.exists():+ done = {r["key"] for r in map(json.loads, out.read_text().splitlines()) if "answer" in r}+ todo = [r for r in rows if r["key"] not in done]+ print(f"{len(rows) // 2} cases, {len(todo)} calls to make", flush=True)+ lock, n = threading.Lock(), [0]++ def work(r):+ res = ask(r, key)+ with lock:+ with open(out, "a") as f:+ f.write(json.dumps(res) + "\n")+ n[0] += 1+ if n[0] % 20 == 0:+ print(f"{n[0]}/{len(todo)}", flush=True)++ with ThreadPoolExecutor(a.threads) as ex:+ list(ex.map(work, todo))+ report(argparse.Namespace(results=[str(out)]))+++# ---------------------------------------------------------------------------------------------- report++def verdict(rs: dict, q: str):+ v = [{"A": r["a_m"], "B": r["b_m"]}.get(r["answer"].get(q, "tie"), "tie") for r in rs.values()]+ if v[0] == v[1]:+ return v[0]+ if "tie" in v:+ return [t for t in v if t != "tie"][0] # one order decided, the other tied+ return "tie" # each order picked its own position: position bias+++def ci(xs, groups=None, n=2000, seed=0):+ """Mean and 95% bootstrap interval; with ``groups`` (one label per x) whole groups are resampled - on the hard+ split the cases of one structure video share its layout and motion, so they are not independent."""+ rng = random.Random(seed)+ m = sum(xs) / len(xs)+ if groups is None:+ b = sorted(sum(rng.choice(xs) for _ in xs) / len(xs) for _ in range(n))+ else:+ by = collections.defaultdict(list)+ for x, g in zip(xs, groups):+ by[g].append(x)+ units = list(by.values())+ b = []+ for _ in range(n):+ draw = [x for _ in units for x in rng.choice(units)]+ b.append(sum(draw) / len(draw))+ b.sort()+ return m, b[int(0.025 * n)], b[int(0.975 * n)]+++def segment(case: str) -> str:+ """hard/<segment>__<look> -> the structure video it was rendered from; a held-out clip is its own unit."""+ return case.split("__")[0] if case.startswith("hard/") else case+++def net_ci(cases: list[str], xs: list[int]):+ """The intervals of the paper: on the hard split whole structure videos are resampled (2,000 draws), since the+ cases of one structure video share its layout and motion; held-out clips are resampled one by one (10,000)."""+ if cases and all(c.startswith("hard/") for c in cases):+ return ci(xs, [segment(c) for c in cases], n=2000)+ return ci(xs, n=10000)+++def report(a: argparse.Namespace) -> None:+ by = collections.defaultdict(dict)+ meta, calls = {}, collections.Counter()+ for f in a.results:+ for line in open(f):+ r = json.loads(line)+ calls[(tuple(r["pair"]), "answered" if "answer" in r else "failed")] += 1+ if "answer" in r:+ by[(r["case"], tuple(r["pair"]))][r["swap"]] = r+ meta[r["case"]] = r+ for p in sorted({p for _, p in by}):+ x, y = p+ cases = [c for (c, pp) in by if pp == p and len(by[(c, pp)]) == 2]+ single = sum(1 for (c, pp) in by if pp == p and len(by[(c, pp)]) == 1)+ print(f"\n=== {x} vs {y}: {len(cases)} cases judged in both orders (net = ({x} wins - {y} wins) / cases)")+ print(f" calls answered {calls[(p, 'answered')]}, failed {calls[(p, 'failed')]} (a rerun retries them); "+ f"{single} cases with one order only are left out; hard intervals resample whole structure videos")+ groups = [(s, [c for c in cases if meta[c]["split"] == s]) for s in ("hard", "heldout")]+ tags = sorted({t for c in cases for t in meta[c].get("tags") or []})+ groups += [(f"hard, {t}", [c for c in cases if t in (meta[c].get("tags") or [])]) for t in tags]+ for label, sel in groups:+ if not sel:+ continue+ cells = []+ for q in Q:+ v = [verdict(by[(c, p)], q) for c in sel]+ w, l = v.count(x), v.count(y)+ m, lo, hi = net_ci(sel, [1 if t == x else -1 if t == y else 0 for t in v])+ star = "*" if len(v) >= 10 and (lo > 0 or hi < 0) else " " # no interval from a handful of cases+ cells.append(f"{q.split('_', 1)[1][:9]:9s} {w:3d}/{len(v) - w - l:3d}/{l:3d} {m:+.2f}{star}")+ print(f" {label:28s} n={len(sel):3d} " + " ".join(cells))+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ sub = ap.add_subparsers(dest="cmd", required=True)+ r = sub.add_parser("run")+ r.add_argument("--bench", required=True)+ r.add_argument("--a", required=True, help="renders of method A")+ r.add_argument("--b", required=True, help="renders of method B")+ r.add_argument("--name-a", required=True)+ r.add_argument("--name-b", required=True)+ r.add_argument("--out", required=True)+ r.add_argument("--split", default="all", choices=("hard", "heldout", "all"))+ r.add_argument("--threads", type=int, default=8)+ p = sub.add_parser("report")+ p.add_argument("results", nargs="+")+ a = ap.parse_args()+ run(a) if a.cmd == "run" else report(a)+++if __name__ == "__main__":+ main()diff --git a/eval/metrics.py b/eval/metrics.pynew file mode 100644index 0000000..7c8d1e1--- /dev/null+++ b/eval/metrics.py@@ -0,0 +1,494 @@+#!/usr/bin/env python+"""Automatic metrics of the SolarRenderer benchmark, the same on both splits.++ python eval/metrics.py --bench ./SolarRenderer-Data --renders ./renders/mymethod --method mymethod \+ --out ./scores [--split hard|heldout|all] [--metrics dover,laion_aes,...] [--shard k --shards n]++For every case with a render, computes the missing metrics and writes <out>/<metric>/<method>/<case key>.json, where+the case key is hard/<case_id> or heldout/<clip_id>. Run one process per GPU with --shard / --shards.+VideoScore2 needs its own environment (transformers 4.53); run it as a separate pass with --metrics videoscore2.+Models are set up by eval/setup.sh.++Readings, grouped into the four dimensions of eval/summary.py:+ quality dover DOVER (ICCV 2023), technical / aesthetic / overall; the repo's preprocessing (val-l1080p+ views), numpy seeded with 0 before every video, score-level fusion (see dover_score.py)+ laion_aes LAION aesthetic predictor v2 (sac+logos+ava1-l14-linearMSE) on CLIP ViT-L/14 image+ embeddings, 2 frames per second, mean+ videoscore2 VideoScore2 visual quality: greedy decoding at 2 fps, the expectation over levels 1-5+ stability flicker RAFT-large flow t->t+1 and t+1->t at 672x384, frame t+1 warped back to t; mean absolute+ residual (0-255) over forward-backward consistent pixels after a Gaussian low-pass+ (sigma 2 px), warp_l1_lp - lower is better. The unfiltered residual (warp_l1) rises with+ sharpness and is reported only for reference.+ dino DINOv2 ViT-S/14 CLS embeddings of all frames at 224x392: mean adjacent-frame cosine+ control motion CoTracker3 offline on the structure video and on the render, a 20x12 grid queried at+ frames 0/40/80/120; error = |render track - structure track| / frame diagonal, over the+ (point, frame) pairs where the structure track is reliable (visibility x confidence > 0.6)+ edges Canny (5x5 blur, sigma 1.2, 50/150) on both, every 4th frame, edge F1 with a 2 px tolerance+ first frame firstframe the DINOv2 embeddings against the first frame's: mean cosine over the last second+"""+from __future__ import annotations++import argparse+import json+import math+import os+import shutil+import subprocess+import sys+import time+import traceback+from pathlib import Path++import numpy as np+import torch+import torch.nn.functional as F++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+import bench # noqa: E402++MODELS = Path(os.environ.get("SOLAR_EVAL_MODELS", HERE.parent / "eval_models"))+FFMPEG = shutil.which("ffmpeg") or "ffmpeg"+ALL = ["dover", "laion_aes", "flicker", "dino", "firstframe", "motion", "edges"]+VERSION = "1"+++# ---------------------------------------------------------------------------------------------- io++def decode(path: str, size: tuple[int, int] | None = None, fps: float | None = None) -> np.ndarray:+ """uint8 [T,H,W,3] RGB; size = (w, h), area-resampled."""+ vf = []+ if fps:+ vf.append(f"fps={fps}")+ if size:+ vf.append(f"scale={size[0]}:{size[1]}:flags=area")+ vf.append("format=rgb24")+ if size is None:+ w, h = probe_size(path)+ else:+ w, h = size+ raw = subprocess.run([FFMPEG, "-nostdin", "-loglevel", "error", "-i", path, "-vf", ",".join(vf),+ "-f", "rawvideo", "-"], capture_output=True, check=True).stdout+ a = np.frombuffer(raw, np.uint8)+ n = len(a) // (w * h * 3)+ return a[: n * w * h * 3].reshape(n, h, w, 3)+++def probe_size(path: str) -> tuple[int, int]:+ out = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=width,height",+ "-of", "csv=p=0", path], capture_output=True, text=True).stdout.strip().split(",")+ return int(out[0]), int(out[1])+++def load_png(path: str) -> np.ndarray:+ from PIL import Image+ return np.asarray(Image.open(path).convert("RGB"))+++# ---------------------------------------------------------------------------------------------- models++class Models:+ """Lazily built, one per worker; weights from eval/setup.sh under SOLAR_EVAL_MODELS (default ./eval_models)."""++ def __init__(self, device):+ self.device = device+ self._m = {}++ def get(self, name):+ if name not in self._m:+ self._m[name] = getattr(self, f"_build_{name}")()+ return self._m[name]++ def _build_dover(self):+ import dover_score+ model, stypes, samplers = dover_score.build(self.device)+ return dover_score, model, stypes, samplers++ def _build_clip(self):+ from transformers import CLIPModel+ clip = CLIPModel.from_pretrained("openai/clip-vit-large-patch14", torch_dtype=torch.float32)+ clip = clip.to(self.device).eval()+ head = torch.nn.Sequential( # improved-aesthetic-predictor MLP (simple_inference.py), dropouts are no-ops+ torch.nn.Linear(768, 1024), torch.nn.Dropout(0.2), torch.nn.Linear(1024, 128), torch.nn.Dropout(0.2),+ torch.nn.Linear(128, 64), torch.nn.Dropout(0.1), torch.nn.Linear(64, 16), torch.nn.Linear(16, 1))+ sd = torch.load(MODELS / "sac+logos+ava1-l14-linearMSE.pth", map_location="cpu")+ sd = {k.replace("layers.", ""): v for k, v in sd.items()}+ head.load_state_dict(sd, strict=True)+ return clip, head.to(self.device).eval()++ def _build_dino(self):+ import timm+ m = timm.create_model("vit_small_patch14_dinov2.lvd142m", pretrained=True, num_classes=0,+ dynamic_img_size=True)+ return m.to(self.device).eval()++ def _build_raft(self):+ from torchvision.models.optical_flow import Raft_Large_Weights, raft_large+ return raft_large(weights=Raft_Large_Weights.C_T_SKHT_V2, progress=False).to(self.device).eval()++ def _build_cotracker(self):+ sys.path.insert(0, str(MODELS / "co-tracker"))+ from cotracker.models.build_cotracker import build_cotracker+ model = build_cotracker(None, offline=True, window_len=60, v2=False)+ sd = torch.load(MODELS / "cotracker3" / "baseline_offline.pth", map_location="cpu")+ if "model" in sd:+ sd = sd["model"]+ model.load_state_dict(sd, strict=True)+ return model.to(self.device).eval()++ def _build_vs2(self):+ import videoscore2+ model, processor, tokenizer = videoscore2.build()+ return videoscore2, model, processor, tokenizer+++# ---------------------------------------------------------------------------------------------- metrics+[email protected]_grad()+def m_dover(path, M):+ ds, model, stypes, samplers = M.get("dover")+ raw = ds.score(path, model, stypes, samplers, M.device, 0)+ return {"dover": ds.fuse(raw), "dover_raw": raw}+++CLIP_MEAN = torch.tensor([0.48145466, 0.4578275, 0.40821073]).view(1, 3, 1, 1)+CLIP_STD = torch.tensor([0.26862954, 0.26130258, 0.27577711]).view(1, 3, 1, 1)+++def clip_pre(frames: np.ndarray) -> torch.Tensor:+ """openai CLIP preprocessing: bicubic resize of the short side to 224 (PIL), centre crop 224."""+ from PIL import Image+ out = []+ for f in frames:+ im = Image.fromarray(f)+ w, h = im.size+ s = 224 / min(w, h)+ im = im.resize((max(224, round(w * s)), max(224, round(h * s))), Image.BICUBIC)+ w, h = im.size+ l, t = (w - 224) // 2, (h - 224) // 2+ out.append(np.asarray(im.crop((l, t, l + 224, t + 224))))+ x = torch.from_numpy(np.stack(out)).permute(0, 3, 1, 2).float() / 255.0+ return (x - CLIP_MEAN) / CLIP_STD++[email protected]_grad()+def m_laion(frames24, M):+ clip, head = M.get("clip")+ sel = frames24[::12] # 2 per second at 24 fps+ x = clip_pre(sel).to(M.device)+ emb = clip.visual_projection(clip.vision_model(pixel_values=x).pooler_output)+ emb = emb / emb.norm(dim=-1, keepdim=True)+ s = head(emb).squeeze(-1).float().cpu().numpy()+ return {"laion_aes": float(s.mean()), "laion_aes_min": float(s.min()), "frames": int(len(sel))}+++IMNET_MEAN = torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)+IMNET_STD = torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)+DINO_HW = (224, 392) # 16 x 28 patches of 14, the 1344x768 aspect (1.75)++[email protected]_grad()+def dino_embed(frames: np.ndarray, M) -> torch.Tensor:+ m = M.get("dino")+ out = []+ for i in range(0, len(frames), 64):+ x = torch.from_numpy(np.ascontiguousarray(frames[i:i + 64])).to(M.device).permute(0, 3, 1, 2).float() / 255+ x = F.interpolate(x, size=DINO_HW, mode="bicubic", align_corners=False, antialias=True).clamp(0, 1)+ x = (x - IMNET_MEAN.to(x.device)) / IMNET_STD.to(x.device)+ out.append(F.normalize(m(x).float(), dim=-1))+ return torch.cat(out)+++def m_dino(emb):+ adj = (emb[1:] * emb[:-1]).sum(-1)+ return {"adjacent_cos": float(adj.mean()), "adjacent_cos_min": float(adj.min()),+ "first_last_cos": float((emb[0] * emb[-1]).sum())}+++def m_firstframe(emb, first_emb, fps=24.0):+ cos = (emb @ first_emb).cpu().numpy().astype(np.float64)+ t = np.arange(len(cos)) / fps+ slope = float(np.polyfit(t, cos, 1)[0])+ return {"last1s_cos": float(cos[-int(fps):].mean()), "slope_per_s": slope, "frame0_cos": float(cos[0]),+ "mean_cos": float(cos.mean()), "curve_1fps": [round(float(c), 4) for c in cos[::int(fps)]]}+++FLOW_WH = (672, 384)++[email protected]_grad()+def flows(frames: np.ndarray, M, batch=12):+ """Forward (t->t+1) and backward (t+1->t) RAFT flows at FLOW_WH. frames uint8 [T,h,w,3] already at FLOW_WH."""+ raft = M.get("raft")+ x = torch.from_numpy(np.ascontiguousarray(frames)).to(M.device).permute(0, 3, 1, 2).float() / 127.5 - 1.0+ fw, bw = [], []+ for i in range(0, len(x) - 1, batch):+ a = x[i:min(i + batch, len(x) - 1)]+ b = x[i + 1:min(i + batch, len(x) - 1) + 1]+ f = raft(torch.cat([a, b]), torch.cat([b, a]))[-1]+ fw.append(f[:len(a)]); bw.append(f[len(a):])+ return x, torch.cat(fw), torch.cat(bw)+++def warp(img, flow):+ """Sample img at x + flow(x). img [N,C,H,W], flow [N,2,H,W] (pixels). Returns warped and in-bounds mask."""+ n, _, h, w = img.shape+ yy, xx = torch.meshgrid(torch.arange(h, device=img.device), torch.arange(w, device=img.device), indexing="ij")+ gx = xx[None] + flow[:, 0]+ gy = yy[None] + flow[:, 1]+ grid = torch.stack([2 * gx / (w - 1) - 1, 2 * gy / (h - 1) - 1], -1)+ out = F.grid_sample(img, grid, mode="bilinear", padding_mode="border", align_corners=True)+ inb = (gx >= 0) & (gx <= w - 1) & (gy >= 0) & (gy <= h - 1)+ return out, inb++[email protected]_grad()+def m_flicker(frames_lo: np.ndarray, M):+ x, fw, bw = flows(frames_lo, M)+ img_t, img_t1 = x[:-1], x[1:]+ warped, inb = warp(img_t1, fw)+ bw_at = warp(bw, fw)[0] # backward flow sampled where the forward flow lands+ d = ((fw + bw_at) ** 2).sum(1)+ ok = inb & (d < 0.01 * ((fw ** 2).sum(1) + (bw_at ** 2).sum(1)) + 0.5) # Sundaram et al. 2010 consistency check+ err = (img_t - warped).abs().mean(1) * 127.5 # [-1,1] -> 0-255 units+ okf = ok.float()+ per_pair = (err * okf).sum((1, 2)) / okf.sum((1, 2)).clamp(min=1)+ pp = per_pair.cpu().numpy()+ # low-pass residual: the warp residual blurred (Gaussian sigma 2 px at 672 wide, normalised convolution over the+ # consistent pixels) before |.|. Sub-pixel misalignment of fine texture is high-frequency and cancels; brightness /+ # colour flicker is low-frequency and stays. (The raw reading rises with sharpness.)+ k = torch.exp(-0.5 * (torch.arange(-6, 7, device=x.device).float() / 2.0) ** 2)+ k = (k / k.sum())++ def blur(z): # [N,C,H,W], separable+ c = z.shape[1]+ z = F.conv2d(F.pad(z, (6, 6, 0, 0), mode="replicate"), k.view(1, 1, 1, -1).repeat(c, 1, 1, 1), groups=c)+ return F.conv2d(F.pad(z, (0, 0, 6, 6), mode="replicate"), k.view(1, 1, -1, 1).repeat(c, 1, 1, 1), groups=c)+ lp_pp = []+ for i in range(0, len(img_t), 16):+ m = okf[i:i + 16, None]+ r = (img_t[i:i + 16] - warped[i:i + 16]) * m+ bm = blur(m)+ lp = (blur(r) / bm.clamp(min=1e-3)).abs().mean(1) * 127.5+ v = m[:, 0] * (bm[:, 0] > 0.5)+ lp_pp.append((lp * v).sum((1, 2)) / v.sum((1, 2)).clamp(min=1))+ lp_pp = torch.cat(lp_pp).cpu().numpy()+ g = (0.299 * x[:, 0] + 0.587 * x[:, 1] + 0.114 * x[:, 2]) * 127.5 # grey, 0-255 offsets+ lap = -4 * g[:, 1:-1, 1:-1] + g[:, :-2, 1:-1] + g[:, 2:, 1:-1] + g[:, 1:-1, :-2] + g[:, 1:-1, 2:]+ return {"warp_l1": float(pp.mean()), "warp_l1_p90": float(np.percentile(pp, 90)),+ "warp_l1_lp": float(lp_pp.mean()), "warp_l1_lp_p90": float(np.percentile(lp_pp, 90)),+ "sharpness_lapvar": float(lap.flatten(1).var(1).mean()),+ "warp_l1_max": float(pp.max()), "consistent_frac": float(okf.mean()),+ "flow_mag": float(fw.norm(dim=1).mean()), "pairs": int(len(pp))}+++GRID = (20, 12) # x, y+QUERY_FRAMES = (0, 40, 80, 120) # a fresh grid every 40 frames: under fast camera moves a frame-0 grid has left the+ # picture by frame 60 (0-22% of frame-0 points were still in frame at 100)+CT_HW = (384, 512)+REL_THR = 0.6++[email protected]_grad()+def track(frames: np.ndarray, M, query_frames=QUERY_FRAMES):+ """CoTracker3 offline; a GRID at each query frame. Returns tracks [T,N,2] (input pixels), vis, conf [T,N],+ query xy [N,2], query frame [N]."""+ model = M.get("cotracker")+ T, H, W, _ = frames.shape+ v = torch.from_numpy(frames.copy()).to(M.device).permute(0, 3, 1, 2).float()+ v = F.interpolate(v, CT_HW, mode="bilinear", align_corners=True)[None] # as CoTrackerPredictor does+ gx = (np.arange(GRID[0]) + 0.5) / GRID[0] * W+ gy = (np.arange(GRID[1]) + 0.5) / GRID[1] * H+ grid = np.array([(x, y) for y in gy for x in gx], np.float32)+ qf = [t for t in query_frames if t < T - 1]+ pts = np.concatenate([grid] * len(qf))+ qt = np.repeat(np.array(qf), len(grid))+ q = torch.zeros(1, len(pts), 3, device=M.device)+ q[0, :, 0] = torch.from_numpy(qt).to(M.device).float()+ q[0, :, 1] = torch.from_numpy(pts[:, 0]).to(M.device) * (CT_HW[1] - 1) / (W - 1)+ q[0, :, 2] = torch.from_numpy(pts[:, 1]).to(M.device) * (CT_HW[0] - 1) / (H - 1)+ tr, vis, conf, _ = model(video=v, queries=q, iters=6)+ tr = tr[0] * tr.new_tensor([(W - 1) / (CT_HW[1] - 1), (H - 1) / (CT_HW[0] - 1)])+ tr[torch.from_numpy(qt).long(), torch.arange(len(pts))] = torch.from_numpy(pts).to(tr.device) # query = query+ return tr.float().cpu().numpy(), vis[0].float().cpu().numpy(), conf[0].float().cpu().numpy(), pts, qt+++def motion_compare(tc, vc, cc, tm, vm, cm, pts, qt, W, H):+ """Per (point, frame after its query frame) where the cond track is reliable."""+ diag = math.hypot(W, H)+ n = min(len(tc), len(tm))+ t = np.arange(n)[:, None]+ after = t > qt[None]+ rel = ((vc[:n] * cc[:n]) > REL_THR) & after+ err = np.linalg.norm(tm[:n] - tc[:n], axis=-1) / diag+ still = np.linalg.norm(tc[:n] - pts[None], axis=-1) / diag # a frozen render: every track stays at its query+ out_vis = (vm[:n] * cm[:n]) > REL_THR+ # velocity step k -> k+1, valid when both ends are reliable (the query frame end is exact)+ start_ok = ((vc[:n - 1] * cc[:n - 1]) > REL_THR) & (t[:-1] > qt[None]) | (t[:-1] == qt[None])+ vrel = rel[1:] & start_ok+ verr = np.linalg.norm((tm[1:n] - tm[:n - 1]) - (tc[1:n] - tc[:n - 1]), axis=-1) / diag+ if rel.sum() == 0:+ return {"kept_frac": 0.0, "err_mean": None}+ return {"err_mean": float(err[rel].mean()), "frac_lt_2pct": float((err[rel] < 0.02).mean()),+ "err_median": float(np.median(err[rel])),+ "vel_err_mean": float(verr[vrel].mean()) if vrel.any() else None,+ "frozen_err_mean": float(still[rel].mean()), "frozen_frac_lt_2pct": float((still[rel] < 0.02).mean()),+ "kept_frac": float(rel.sum() / after.sum()), "kept_pairs": int(rel.sum()),+ "kept_frac_by_query": [round(float(rel[:, qt == q].sum() / max(after[:, qt == q].sum(), 1)), 3)+ for q in np.unique(qt)],+ "render_reliable_frac": float(out_vis[rel].mean()), "frames": int(n)}+++def canny(frames: np.ndarray) -> np.ndarray:+ import cv2+ return np.stack([cv2.Canny(cv2.GaussianBlur(cv2.cvtColor(f, cv2.COLOR_RGB2GRAY), (5, 5), 1.2), 50, 150)+ for f in frames])+++def edge_errors(made: np.ndarray, want: np.ndarray, tolerance: int = 2, level: int = 64) -> dict:+ """Edge F1: an edge counts as found when the other side has one within ``tolerance`` pixels."""++ frames = min(len(made), len(want))+ found = torch.from_numpy(np.ascontiguousarray(made[:frames]) >= level).float()[:, None]+ given = torch.from_numpy(np.ascontiguousarray(want[:frames]) >= level).float()[:, None]+ window = 2 * tolerance + 1++ def near(edges: torch.Tensor) -> torch.Tensor:+ return torch.nn.functional.max_pool2d(edges, window, stride=1, padding=tolerance)++ hits_found = float((found * near(given)).sum())+ hits_given = float((given * near(found)).sum())+ precision = hits_found / max(float(found.sum()), 1.0)+ recall = hits_given / max(float(given.sum()), 1.0)+ f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0.0+ return {"F1": f1, "precision": precision, "recall": recall, "frames": frames}+++def m_edges(render_full, cond_full):+ r, c = render_full[::4], cond_full[::4]+ n = min(len(r), len(c))+ out = edge_errors(canny(r[:n]), canny(c[:n]))+ out["cond_edge_density"] = float((canny(c[:n]) > 0).mean())+ return out++[email protected]_grad()+def m_vs2(path, prompt, M):+ vs, model, processor, tokenizer = M.get("vs2")+ r = vs.judge(model, processor, tokenizer, path, prompt)+ return {"visual_quality": r["visual_quality"], "visual_quality_expected": r["visual_quality_expected"],+ "alignment": r["alignment"], "physics": r["physics"], "reason": r["reason"][-400:]}+++# ---------------------------------------------------------------------------------------------- driver++def compute(need: list[str], render: str, cond: str | None, first: str | None, M, prompt: str = "") -> dict:+ """All metrics in need on local files; returns {metric: payload with 'sec'}."""+ res = {}++ def timed(name, fn):+ torch.cuda.synchronize(M.device)+ t0 = time.time()+ try:+ out = fn()+ except Exception:+ traceback.print_exc()+ out = {"error": traceback.format_exc()[-400:]}+ torch.cuda.synchronize(M.device)+ out["sec"] = round(time.time() - t0, 2)+ res[name] = out++ full = decode(render) if set(need) & {"laion_aes", "dino", "firstframe", "motion", "edges"} else None+ if full is not None and len(full) == 0:+ raise RuntimeError("render decodes to 0 frames")+ cv = None+ if set(need) & {"motion", "edges"}:+ cv = decode(cond)+ if cv.shape[1:3] != full.shape[1:3]:+ cv = decode(cond, size=(full.shape[2], full.shape[1]))+ if "dover" in need:+ timed("dover", lambda: m_dover(render, M))+ if "laion_aes" in need:+ timed("laion_aes", lambda: m_laion(full, M))+ if "flicker" in need:+ timed("flicker", lambda: m_flicker(decode(render, size=FLOW_WH), M))+ if "dino" in need or "firstframe" in need:+ t0 = time.time()+ emb = dino_embed(full, M)+ te = time.time() - t0+ if "dino" in need:+ timed("dino", lambda: m_dino(emb))+ res["dino"]["sec"] = round(res["dino"]["sec"] + te, 2)+ if "firstframe" in need:+ timed("firstframe", lambda: m_firstframe(emb, dino_embed(load_png(first)[None], M)[0]))+ res["firstframe"]["sec"] = round(res["firstframe"]["sec"] + te, 2)+ if "motion" in need:+ def mo():+ # the query frames both videos have: a render shorter than its structure (Wan, 121 frames) would+ # otherwise track a different number of points than the structure+ qf = tuple(t for t in QUERY_FRAMES if t < min(len(cv), len(full)) - 1)+ tc, vc, cc, pts, qt = track(cv, M, qf)+ tm, vm, cm, _, _ = track(full, M, qf)+ return motion_compare(tc, vc, cc, tm, vm, cm, pts, qt, full.shape[2], full.shape[1])+ timed("motion", mo)+ if "edges" in need:+ timed("edges", lambda: m_edges(full, cv))+ if "videoscore2" in need:+ timed("videoscore2", lambda: m_vs2(render, prompt, M))+ return res+++def result_path(out: str, metric: str, method: str, case: bench.Case) -> Path:+ """<out>/<metric>/<method>/hard/<case_id>.json or <out>/<metric>/<method>/heldout/<clip_id>.json"""+ return Path(out) / metric / method / case.split / f"{case.key}.json"+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--bench", required=True, help="local copy of the SolarRenderer-Data dataset")+ ap.add_argument("--renders", required=True, help="<renders>/hard/<case_id>.mp4, <renders>/heldout/<clip_id>.mp4")+ ap.add_argument("--method", required=True, help="the name results are filed under")+ ap.add_argument("--out", required=True)+ ap.add_argument("--split", default="all", choices=("hard", "heldout", "all"))+ ap.add_argument("--metrics", default=",".join(ALL), help=f"comma list of {ALL + ['videoscore2']}")+ ap.add_argument("--shard", type=int, default=0)+ ap.add_argument("--shards", type=int, default=1)+ ap.add_argument("--device", default="cuda:0")+ a = ap.parse_args()+ metrics = [m for m in a.metrics.split(",") if m]+ M = Models(torch.device(a.device))+ torch.cuda.set_device(M.device)+ cases = bench.load(a.bench, a.split)[a.shard::a.shards]+ done = skipped = 0+ for c in cases:+ render = c.render(a.renders)+ need = [m for m in metrics if not result_path(a.out, m, a.method, c).exists()]+ if not need:+ continue+ if not render.exists():+ skipped += 1+ continue+ try:+ res = compute(need, str(render), c.condition, c.first_frame, M, c.prompt)+ except Exception:+ traceback.print_exc()+ print(f"SKIP {c.key}", flush=True)+ continue+ for m, payload in res.items():+ if "error" in payload: # not written: the next run retries it+ print(f"{c.key} {m} ERROR {payload['error'][-200:]}", flush=True)+ continue+ p = result_path(a.out, m, a.method, c)+ p.parent.mkdir(parents=True, exist_ok=True)+ rec = {"id": c.id, "split": c.split, "method": a.method, "metric": m, "kind": c.kind,+ "tags": c.tags, "version": VERSION, **payload}+ p.write_text(json.dumps(rec, ensure_ascii=False))+ done += 1+ brief = {m: {k: (round(v, 4) if isinstance(v, float) else v) for k, v in p.items()+ if not isinstance(v, (list, dict, str))} for m, p in res.items()}+ print(f"{c.key} {json.dumps(brief)}", flush=True)+ print(f"done: {done} cases scored, {skipped} without a render", flush=True)+++if __name__ == "__main__":+ main()diff --git a/eval/render_benchmark.py b/eval/render_benchmark.pynew file mode 100644index 0000000..8e07d95--- /dev/null+++ b/eval/render_benchmark.py@@ -0,0 +1,75 @@+"""Render the SolarRenderer benchmark with SolarRenderer, each model loaded once per GPU.++ python eval/render_benchmark.py --bench ./SolarRenderer-Data --out ./renders/solarrenderer \+ --h3 ./MiniMax-H3 --control weights/solarrenderer_control_lora.safetensors \+ --fewstep weights/solarrenderer_4step_lora.safetensors [--split hard|heldout|all] [--shard k --shards n]++Writes <out>/hard/<case_id>.mp4 and <out>/heldout/<clip_id>.mp4, the layout eval/metrics.py and eval/judge.py+read. Run one process per GPU with --shard / --shards (CUDA_VISIBLE_DEVICES=k). Every case is encoded first, the text+encoder is released, then every case is sampled; renders already on disk are skipped. Every case renders from its structure video, first+frame and prompt.+"""++from __future__ import annotations++import argparse+import sys+import tempfile+import time+from pathlib import Path++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE))+sys.path.insert(0, str(HERE.parent / "scripts"))++import bench # noqa: E402+import render # noqa: E402+++def main() -> None:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--bench", required=True, help="local copy of the SolarRenderer-Data dataset")+ ap.add_argument("--out", required=True)+ ap.add_argument("--h3", required=True)+ ap.add_argument("--control", required=True)+ ap.add_argument("--fewstep", required=True)+ ap.add_argument("--split", default="all", choices=("hard", "heldout", "all"))+ ap.add_argument("--seed", type=int, default=0)+ ap.add_argument("--forwards", type=int, default=4)+ ap.add_argument("--causal", action="store_true", help="the causal LoRA pair, streamed block by block")+ ap.add_argument("--shard", type=int, default=0)+ ap.add_argument("--shards", type=int, default=1)+ ap.add_argument("--device", default="cuda:0")+ a = ap.parse_args()+ cases = [c for c in bench.load(a.bench, a.split)][a.shard::a.shards]+ todo = [c for c in cases if not c.render(a.out).exists()]+ print(f"shard {a.shard}/{a.shards}: {len(cases)} cases, {len(todo)} to render", flush=True)+ if not todo:+ return+ with tempfile.TemporaryDirectory() as work:+ encoder = render.load_encoder(a.h3, a.device)+ members = {}+ for c in todo:+ member = f"{work}/{c.key.replace('/', '__')}.safetensors"+ began = time.perf_counter()+ try:+ render.encode_sample(encoder, render.sample_of(c.id, c.prompt, c.condition, c.first_frame,+ target=c.condition), member)+ except Exception as error: # one bad input must not stop the sweep+ print(f"SKIP encode {c.key}: {error}", flush=True)+ continue+ members[c.key] = (c, member)+ print(f"encoded {c.key} in {time.perf_counter() - began:.0f} s", flush=True)+ del encoder+ render.free_cuda()+ renderer = render.load_renderer(a.h3, a.control, a.fewstep, a.device, causal=a.causal)+ for key, (c, member) in members.items():+ try:+ took = render.render_member(renderer, member, str(c.render(a.out)), a.seed, a.forwards)+ print(f"rendered {key} in {took:.0f} s", flush=True)+ except Exception as error:+ print(f"SKIP render {key}: {error}", flush=True)+++if __name__ == "__main__":+ main()diff --git a/eval/setup.sh b/eval/setup.shnew file mode 100755index 0000000..322cdef--- /dev/null+++ b/eval/setup.sh@@ -0,0 +1,37 @@+#!/usr/bin/env bash+# Set up the models eval/metrics.py reads, under SOLAR_EVAL_MODELS (default ./eval_models), at the versions the+# published numbers used. Run once from the repository root: bash eval/setup.sh+set -euo pipefail+M=${SOLAR_EVAL_MODELS:-eval_models}+mkdir -p "$M"+pip install "timm>=1.0" "pyiqa>=0.1.16" opencv-python-headless decord scikit-video++check() { # check <file> <sha256>+ echo "$2 $1" | sha256sum -c --quiet - || { echo "checksum mismatch: $1"; exit 1; }+}++# DOVER at f1ddc96 and its v0.1.0 weights+if [ ! -d "$M/DOVER" ]; then+ git clone https://github.com/VQAssessment/DOVER "$M/DOVER"+ git -C "$M/DOVER" checkout f1ddc96+fi+mkdir -p "$M/DOVER/pretrained_weights"+W="$M/DOVER/pretrained_weights/DOVER.pth"+[ -s "$W" ] || curl -L -o "$W" https://github.com/VQAssessment/DOVER/releases/download/v0.1.0/DOVER.pth+check "$W" f4a42c0bbc94c94dd7409e7f40887d44c5c30314d1d09e7edf03cc35813b4838++# LAION aesthetic predictor v2 head (CLIP ViT-L/14 comes from the Hub: openai/clip-vit-large-patch14)+W="$M/sac+logos+ava1-l14-linearMSE.pth"+[ -s "$W" ] || curl -L -o "$W" "https://github.com/christophschuhmann/improved-aesthetic-predictor/raw/main/sac+logos+ava1-l14-linearMSE.pth"+check "$W" 21dd590f3ccdc646f0d53120778b296013b096a035a2718c9cb0d511bff0f1e0++# CoTracker3 at 82e02e8, offline baseline weights+if [ ! -d "$M/co-tracker" ]; then+ git clone https://github.com/facebookresearch/co-tracker "$M/co-tracker"+ git -C "$M/co-tracker" checkout 82e02e8+fi+hf download facebook/cotracker3 baseline_offline.pth --local-dir "$M/cotracker3"+check "$M/cotracker3/baseline_offline.pth" da09bbac871f7398e5b29c4de5213652658949737bc158840b101678ba8ad1df++# DINOv2 ViT-S/14 (timm), RAFT-large (torchvision) and DISTS / LPIPS (pyiqa) download themselves on first use.+echo "eval models ready in $M"diff --git a/eval/summary.py b/eval/summary.pynew file mode 100644index 0000000..bf7c90f--- /dev/null+++ b/eval/summary.py@@ -0,0 +1,107 @@+#!/usr/bin/env python+"""Tables from eval/metrics.py results: the mean of every reading per method, and the automatic score.++ python eval/summary.py --scores ./scores --methods mine,other [--split hard|heldout] [--tag "unseen condition"]++Readings (lower is better for flicker and motion error), in four groups:+ quality DOVER overall, LAION aesthetic, VideoScore2 visual quality (expected)+ stability low-pass warp residual, DINOv2 adjacent-frame cosine+ structure CoTracker3 motion error, edge F1+ first frame DINOv2 cosine to the first frame over the last second+Held-out split: also DISTS and LPIPS against the real video (eval/gt_metrics.py), when present.++Automatic score (0-100), the score of the paper: each reading is mapped linearly from a fixed range - DOVER, edge F1,+DINOv2 adjacent-frame and first-frame similarity from [0, 1], the aesthetic score from [1, 10], VideoScore2 from+[1, 5], flicker from [0, 10] and motion error from [0, 0.5], both reversed and clipped at 0 - the readings of each+group are averaged, and the score is the mean of the four groups. It uses each method's means over the cases, so it+does not depend on the other methods. Methods are compared on the cases every listed method has.+"""+from __future__ import annotations++import argparse+import json+from pathlib import Path++READ = { # name: (metric directory, getter, group)+ "dover_overall": ("dover", lambda j: j["dover"]["overall"], "quality"),+ "laion_aes": ("laion_aes", lambda j: j["laion_aes"], "quality"),+ "vs2_visual": ("videoscore2", lambda j: j.get("visual_quality_expected"), "quality"),+ "warp_l1_lp": ("flicker", lambda j: j["warp_l1_lp"], "stability"),+ "dino_adjacent": ("dino", lambda j: j["adjacent_cos"], "stability"),+ "motion_err": ("motion", lambda j: j["err_mean"], "structure"),+ "edge_f1": ("edges", lambda j: j["F1"], "structure"),+ "first_last1s": ("firstframe", lambda j: j["last1s_cos"], "first_frame"),+ "dists_real": ("gt", lambda j: j["dists_real"], "ground_truth"),+ "lpips_real": ("gt", lambda j: j["lpips_real"], "ground_truth"),+}+SCALE = { # reading -> 0-100+ "dover_overall": lambda x: 100 * x,+ "laion_aes": lambda x: 100 * (x - 1) / 9,+ "vs2_visual": lambda x: 100 * (x - 1) / 4,+ "warp_l1_lp": lambda x: 100 * max(0.0, 1 - x / 10),+ "dino_adjacent": lambda x: 100 * x,+ "motion_err": lambda x: 100 * max(0.0, 1 - x / 0.5),+ "edge_f1": lambda x: 100 * x,+ "first_last1s": lambda x: 100 * x,+}+GROUPS = ("quality", "stability", "structure", "first_frame")+++def score(means: dict[str, float]) -> tuple[float, ...]:+ """(score, quality, stability, structure, first frame) from a method's mean readings."""+ groups = [[SCALE[n](means[n]) for n in SCALE if READ[n][2] == g] for g in GROUPS]+ groups = [sum(v) / len(v) for v in groups]+ return (sum(groups) / len(groups), *groups)+++def load(scores: Path, method: str, split: str, tag: str | None = None) -> dict[str, dict[str, float]]:+ """case id -> reading -> value"""+ out: dict[str, dict[str, float]] = {}+ for name, (metric, get, _) in READ.items():+ root = scores / metric / method / split+ for p in root.rglob("*.json") if root.exists() else []:+ try:+ j = json.loads(p.read_text())+ v = get(j)+ except (KeyError, TypeError, json.JSONDecodeError):+ continue+ if tag and metric != "gt" and tag not in (j.get("tags") or []):+ continue+ if v is not None:+ out.setdefault(str(p.relative_to(root))[:-5], {})[name] = float(v)+ if tag: # the gt readings carry no tags; keep the cases the tagged readings selected+ out = {k: v for k, v in out.items() if set(v) - {"dists_real", "lpips_real"}}+ return out+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--scores", required=True, type=Path)+ ap.add_argument("--methods", required=True, help="comma list")+ ap.add_argument("--split", default="hard", choices=("hard", "heldout"))+ ap.add_argument("--tag", help='hard split only: the cases with this tag, e.g. "unseen condition"')+ a = ap.parse_args()+ methods = [m for m in a.methods.split(",") if m]+ data = {m: load(a.scores, m, a.split, a.tag) for m in methods}+ need = [n for n, (_, _, g) in READ.items() if g != "ground_truth" or a.split == "heldout"]+ common = set.intersection(*({k for k, v in d.items() if all(n in v for n in SCALE)} for d in data.values()))+ if not common:+ raise SystemExit("no case scored for every method")+ print(f"{a.split}{', ' + a.tag if a.tag else ''}: {len(common)} cases every method has\n")+ print(f"{'method':24s} {'score':>6s} {'qual':>6s} {'stab':>6s} {'struct':>6s} {'first':>6s} "+ + " ".join(f"{n:>13s}" for n in need))+ rows = []+ for m, d in data.items():+ means = {}+ for n in need:+ v = [d[k][n] for k in common if n in d[k]]+ if v:+ means[n] = sum(v) / len(v)+ rows.append((score(means), m, means))+ for sc, m, means in sorted(rows, key=lambda r: -r[0][0]):+ print(f"{m:24s} " + " ".join(f"{x:6.1f}" for x in sc) + " "+ + " ".join(f"{means[n]:13.4f}" if n in means else f"{'-':>13s}" for n in need))+++if __name__ == "__main__":+ main()diff --git a/eval/videoscore2.py b/eval/videoscore2.pynew file mode 100644index 0000000..ffb953d--- /dev/null+++ b/eval/videoscore2.py@@ -0,0 +1,157 @@+"""VideoScore2 (TIGER-Lab/VideoScore2), as eval/metrics.py reads it: visual quality, prompt alignment and physical+plausibility on 1-5. Decoding is greedy (the released script samples at temperature 0.7; greedy decoding gives the same score+for the same video, so methods can be compared); a score is read off the distribution over the levels:+the authors' discounted level and the plain expectation (*_expected, the one the summary uses).+Needs transformers 4.53 and qwen-vl-utils 0.0.14 (its own environment, see eval/README.md).+"""+from __future__ import annotations++import os+import re++import numpy as np+import torch++MODEL = os.environ.get("VIDEOSCORE2", "TIGER-Lab/VideoScore2")+QUERY = """+ You are an expert for evaluating AI-generated videos from three dimensions:+ (1) visual quality - clarity, smoothness, artifacts;+ (2) text-to-video alignment - fidelity to the prompt;+ (3) physical/common-sense consistency - naturalness and physics plausibility.++ Video prompt: {prompt}++ Please output in this format:+ visual quality: <v_score>;+ text-to-video alignment: <t_score>,+ physical/common-sense consistency: <p_score>+ """++DIMENSIONS = (+ ("visual_quality", "visual quality:"),+ ("alignment", "text-to-video alignment:"),+ ("physics", "physical/common-sense consistency:"),+)+++def soft_score(hard: int | None, index: int, scores, tokenizer) -> tuple[float | None, float | None]:+ """Two readings of the same distribution over the levels 1-5.++ The first is the authors' own: the level it settled on, discounted by how much of the mass sat+ elsewhere. It is what their published numbers mean, so it stays. It is also discontinuous - the+ argmax hops - and two cards can disagree on it by a third of a level. The second is the plain+ expectation over the levels, which moves smoothly and is the one to read a trend off.+ """++ if hard is None or index < 0:+ return None, None+ logits = torch.log_softmax(scores[index][0], dim=-1)+ probabilities = []+ for level in range(1, 6):+ ids = tokenizer.encode(str(level), add_special_tokens=False)+ if len(ids) == 1:+ probabilities.append((level, float(np.exp(logits[ids[0]].item()))))+ if not probabilities:+ return None, None+ levels, values = zip(*probabilities)+ total = sum(values)+ best = max(values)+ expected = sum(level * value for level, value in probabilities) / total+ return round(levels[values.index(best)] * best / total, 4), round(expected, 4)+++def score_token(label: str, tokenizer, generated: list[int]) -> int:+ text = tokenizer.decode(generated, skip_special_tokens=False)+ match = re.search(r"(?:\(\d+\)\s*|\n\s*)?" + re.escape(label), text, flags=re.IGNORECASE)+ if not match:+ return -1+ digit = re.search(r"\d", text[match.end() :])+ if not digit:+ return -1+ wanted = text[: match.end() + digit.start() + 1]+ for cut in range(len(generated)):+ if tokenizer.decode(generated[: cut + 1], skip_special_tokens=False) == wanted:+ return cut+ return -1+++# Some answers leave the requested format ("Dim-1 Visual Quality: 4", "Physical Consistency: 5"): the score is then+# read where the answer last states it for that dimension.+FALLBACK = {+ "visual_quality": r"visual quality",+ "alignment": r"text-to-video alignment",+ "physics": r"physical(?:/common-sense)? consistency",+}+++def stated_score(pattern: str, tokenizer, generated: list[int]) -> tuple[int | None, int]:+ text = tokenizer.decode(generated, skip_special_tokens=False)+ hits = list(re.finditer(pattern + r"[^\d\n]{0,60}?([1-5])\b", text, flags=re.IGNORECASE))+ if not hits:+ return None, -1+ hit = hits[-1]+ wanted = text[: hit.end(1)]+ for cut in range(len(generated)):+ if tokenizer.decode(generated[: cut + 1], skip_special_tokens=False) == wanted:+ return int(hit.group(1)), cut+ return None, -1+++def build():+ from transformers import AutoModelForVision2Seq, AutoProcessor, AutoTokenizer++ model = (+ AutoModelForVision2Seq.from_pretrained(MODEL, trust_remote_code=True, torch_dtype="auto")+ .to("cuda")+ .eval()+ )+ processor = AutoProcessor.from_pretrained(MODEL, trust_remote_code=True)+ tokenizer = getattr(processor, "tokenizer", None) or AutoTokenizer.from_pretrained(+ MODEL, trust_remote_code=True, use_fast=False+ )+ return model, processor, tokenizer+++def judge(model, processor, tokenizer, video: str, prompt: str, fps: float = 2.0) -> dict:+ from qwen_vl_utils import process_vision_info++ messages = [+ {+ "role": "user",+ "content": [+ {"type": "video", "video": video, "fps": fps},+ {"type": "text", "text": QUERY.format(prompt=prompt)},+ ],+ }+ ]+ text = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)+ images, videos = process_vision_info(messages)+ inputs = processor(+ text=[text], images=images, videos=videos, fps=fps, padding=True, return_tensors="pt"+ ).to("cuda")+ with torch.no_grad():+ out = model.generate(+ **inputs,+ max_new_tokens=1024,+ output_scores=True,+ return_dict_in_generate=True,+ do_sample=False,+ )+ length = inputs["input_ids"].shape[1]+ generated = out.sequences[0, length:].tolist()+ spoken = processor.batch_decode(out.sequences[:, length:], skip_special_tokens=True)[0]+ stated = re.search(+ r"visual quality:\s*(\d+).*?text-to-video alignment:\s*(\d+)"+ r".*?physical/common-sense consistency:\s*(\d+)",+ spoken,+ re.DOTALL | re.IGNORECASE,+ )+ result = {"reason": spoken.strip()[-600:]}+ for position, (name, label) in enumerate(DIMENSIONS):+ if stated:+ hard, index = int(stated.group(position + 1)), score_token(label, tokenizer, generated)+ else:+ hard, index = stated_score(FALLBACK[name], tokenizer, generated)+ soft, expected = soft_score(hard, index, out.scores, tokenizer)+ result[name], result[f"{name}_expected"] = soft, expected+ return resultdiff --git a/examples/sem-highway_synthwave/README.md b/examples/sem-highway_synthwave/README.mdnew file mode 100644index 0000000..6c7b3b8--- /dev/null+++ b/examples/sem-highway_synthwave/README.md@@ -0,0 +1,24 @@+# Example: a synthwave highway run++A retro synthwave highway run: a neon-trimmed coupe overtakes a glowing semi-truck, flies under a neon gantry and+overpass at dusk, and slides back into its lane.++| File | What it is |+|---|---|+| `structure.mp4` | semantic segmentation, 1344Γ768, 24 fps, 158 frames |+| `first.png` | the first frame, in the synthwave look |+| `prompt.txt` | the six-part prompt |++```bash+python scripts/render.py --h3 ./MiniMax-H3 \+ --control weights/solarrenderer_control_lora.safetensors \+ --fewstep weights/solarrenderer_4step_lora.safetensors \+ --structure examples/sem-highway_synthwave/structure.mp4 \+ --first examples/sem-highway_synthwave/first.png \+ --prompt examples/sem-highway_synthwave/prompt.txt \+ --out render.mp4 --seed 0+```++<p align="center"><img src="preview.gif" width="90%"/></p>++Structure (left) and the four-step render (right).diff --git a/examples/sem-highway_synthwave/first.png b/examples/sem-highway_synthwave/first.pngnew file mode 100644index 0000000..bd9bb7fBinary files /dev/null and b/examples/sem-highway_synthwave/first.png differdiff --git a/examples/sem-highway_synthwave/preview.gif b/examples/sem-highway_synthwave/preview.gifnew file mode 100644index 0000000..436f682Binary files /dev/null and b/examples/sem-highway_synthwave/preview.gif differdiff --git a/examples/sem-highway_synthwave/prompt.txt b/examples/sem-highway_synthwave/prompt.txtnew file mode 100644index 0000000..54d3a64--- /dev/null+++ b/examples/sem-highway_synthwave/prompt.txt@@ -0,0 +1,19 @@+subject_definitions:+<Video 1> is a semantic segmentation of the scene, one flat colour per class (road, sidewalk, building, vegetation, vehicle, pedestrian, sky and so on), frame by frame, from the same camera as the target video.+<Video 2> is the first frame of [Shot 1], held still.++summary:+[keyframe completion + reference generation] The target video begins from <Video 2> and follows the frame-by-frame structure and motion of <Video 1>. A retro synthwave highway run: a neon-trimmed coupe overtakes a glowing semi-truck, flies under a neon gantry and overpass at dusk, and slides back into its lane.++retention_analysis:+<Video 1> (frame-by-frame layout of every class and camera): attribute_transfer - every region keeps its class, outline and place; materials, colours and lighting are not given by <Video 1>; the flat class colours are not shown.+<Video 2> ([Shot 1] first frame): fully_preserved - the composition, subjects, materials, colours and lighting of <Video 2> are the target video's first frame.++detailed_description:+The target video is a stylised synthwave animation in the look of an eighties retro-futuristic music video, with a magenta-to-violet sunset gradient sky, a huge striped setting sun on the horizon, neon pink and cyan edge lighting, glossy reflective surfaces and a light film grain. [Shot 1] The shot begins from <Video 2>. The camera trails a few metres behind and above a black wedge-shaped sports coupe with a glowing cyan light strip across its tail, in the right lane of a six-lane divided highway whose lane lines glow hot pink. Ahead sits the back of a chrome semi-trailer outlined in purple neon, pulled by a tractor with a glowing grille. After half a second the coupe glides left into the middle lane, the camera tilting with it, and begins to close on the truck. The trailer's mirror-like side slides by on the right, reflecting the sunset, then its glowing wheels and the cab with chrome stacks, passing within a couple of metres. Midway, a gantry with two glowing signs in retro lettering sweeps overhead. Further up the road, tail lights of an SUV and a sedan streak away in the fast lane, with a hatchback, a box truck and a van glowing ahead. Across a dark median lined with tall lamp posts topped by pink light bars, oncoming cars and a coach race past as streaks of white light. Near the end the coupe flashes under a concrete overpass trimmed with neon tubes, while a glowing SUV and a delivery truck cross above, their light trails painting streaks along the bridge railing. Clear of the truck, the coupe slides back into the right lane toward the setting sun. The coupe stays planted through every move, rocking slightly over expansion joints, and the camera follows with a slight lag so each lane change feels smooth, heavy and rhythmic, like a slow camera move in a music video. Guard rails gleam with reflected pink light, palm-like trees and stylised pines stand as black silhouettes, utility poles and their wires cut thin lines across the gradient sky, and low warehouses with glowing windows and purple hills fade toward a wireframe-like horizon.++overall_soundscape:+Smooth tyre roar and wind with a punchy engine note as the coupe overtakes, a deep whoosh past the truck and a hollow echo under the overpass.++non_diegetic_music:+N/A\ No newline at end of filediff --git a/examples/sem-highway_synthwave/structure.mp4 b/examples/sem-highway_synthwave/structure.mp4new file mode 100644index 0000000..0a3dce7Binary files /dev/null and b/examples/sem-highway_synthwave/structure.mp4 differdiff --git a/pyproject.toml b/pyproject.tomlnew file mode 100644index 0000000..41fce7f--- /dev/null+++ b/pyproject.toml@@ -0,0 +1,44 @@+[build-system]+requires = ["setuptools>=77", "wheel"]+build-backend = "setuptools.build_meta"++[project]+name = "solarrenderer"+version = "1.0.0"+description = "SolarRenderer: structure-conditioned video rendering on MiniMax-H3."+readme = "README.md"+requires-python = ">=3.10"+license = "Apache-2.0"+license-files = ["LICENSE", "NOTICE"]+authors = [{name = "SolarRenderer authors"}]+# The versions the release was tested with.+dependencies = [+ "torch>=2.8,<2.9",+ "torchvision>=0.23,<0.24",+ "diffusers==0.40.0",+ "transformers==5.14.1",+ "peft==0.20.0",+ "accelerate>=1.14",+ "safetensors>=0.8",+ "huggingface_hub>=1.25",+ "fsspec>=2024.6",+ "einops>=0.8",+ "numpy>=1.26,<2",+ "PyYAML>=6.0",+ "av>=13.1",+ "pillow>=11",+ "imageio==2.37.4",+ "imageio-ffmpeg==0.6.0",+]++[project.optional-dependencies]+dev = ["pytest>=8.0"]++[project.scripts]+solarwm = "solarwm.cli:main"++[tool.setuptools]+package-dir = {"" = "src"}++[tool.setuptools.packages.find]+where = ["src"]diff --git a/scripts/convert_dmad.py b/scripts/convert_dmad.pynew file mode 100644index 0000000..5ec38e3--- /dev/null+++ b/scripts/convert_dmad.py@@ -0,0 +1,80 @@+"""Convert the public DMAD four-step LoRA for MiniMax-H3 to the keys the training code reads.++ hf download ZhengmingYu/DMAD minimax_h3/dmad_minimax_h3_4step_lora_critic.safetensors --local-dir ./dmad+ python scripts/convert_dmad.py --h3 ./MiniMax-H3 \+ --src ./dmad/minimax_h3/dmad_minimax_h3_4step_lora_critic.safetensors --out ./weights/dmad_4step_lora.safetensors++The release names each pair ``<module>.lora.down.weight`` / ``<module>.lora.up.weight``; the training code reads+``<module>.lora_A.weight`` / ``<module>.lora_B.weight``. Every module is checked against the Ref2VA transformer: it must+exist and the pair's shape must match its weight.+"""++from __future__ import annotations++import argparse+import json+import os+from pathlib import Path+++def base_shapes(h3: Path) -> dict[str, tuple[int, ...]]:+ from safetensors import safe_open++ root = h3 / "transformer_ref"+ shards = sorted(set(json.loads((root / "diffusion_pytorch_model.safetensors.index.json").read_text())["weight_map"].values()))+ shapes = {}+ for shard in shards:+ with safe_open(str(root / shard), framework="pt") as handle:+ for key in handle.keys():+ if key.endswith(".weight"):+ shapes[key[: -len(".weight")]] = tuple(handle.get_slice(key).get_shape())+ return shapes+++def convert(src: str) -> tuple[dict, float | None]:+ from safetensors import safe_open+ from safetensors.torch import load_file++ with safe_open(src, framework="pt") as handle:+ meta = handle.metadata() or {}+ out, alphas = {}, set()+ for key, value in load_file(src).items():+ key = key.removeprefix("diffusion_model.").removeprefix("transformer.")+ if key.endswith(".alpha"):+ alphas.add(float(value))+ continue+ key = (key.replace(".default.weight", ".weight").replace(".lora.down.weight", ".lora_A.weight")+ .replace(".lora.up.weight", ".lora_B.weight"))+ if not key.endswith((".lora_A.weight", ".lora_B.weight")):+ raise ValueError(f"unexpected key {key}")+ out[key] = value.contiguous()+ named = meta.get("alpha", meta.get("lora_alpha"))+ alpha = float(named) if named is not None else (alphas.pop() if len(alphas) == 1 else None)+ return out, alpha+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--h3", required=True, type=Path)+ parser.add_argument("--src", required=True)+ parser.add_argument("--out", required=True)+ args = parser.parse_args()+ from safetensors.torch import save_file++ state, alpha = convert(args.src)+ shapes = base_shapes(args.h3.resolve())+ modules = {key.rsplit(".lora_", 1)[0] for key in state}+ for module in sorted(modules):+ a, b = state[f"{module}.lora_A.weight"], state[f"{module}.lora_B.weight"]+ if shapes.get(module) != (b.shape[0], a.shape[1]) or a.shape[0] != b.shape[1]:+ raise ValueError(f"{module}: LoRA {tuple(b.shape)} x {tuple(a.shape)} does not fit {shapes.get(module)}")+ metadata = {"rank": str(min(int(v.shape[0]) for k, v in state.items() if k.endswith("lora_A.weight")))}+ if alpha is not None:+ metadata["alpha"] = str(alpha)+ os.makedirs(os.path.dirname(os.path.abspath(args.out)), exist_ok=True)+ save_file(state, args.out, metadata=metadata)+ print(f"{len(modules)} modules, rank {metadata['rank']}, alpha {metadata.get('alpha', 'rank')} -> {args.out}")+++if __name__ == "__main__":+ main()diff --git a/scripts/export_lora.py b/scripts/export_lora.pynew file mode 100644index 0000000..2ddfb37--- /dev/null+++ b/scripts/export_lora.py@@ -0,0 +1,29 @@+"""Export a training run's LoRA (<checkpoint>/adapter.pt) as the safetensors scripts/render.py reads.++ python scripts/export_lora.py --checkpoint /path/to/runs/structure_guided_4step/checkpoint_model_000100 \+ --out weights/my_4step_lora.safetensors++Keys become <module>.lora_A.weight / <module>.lora_B.weight in bf16; the run's rank and alpha go in the metadata.+"""+import argparse++import torch+from safetensors.torch import save_file+++def main() -> None:+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--checkpoint", required=True, help="a checkpoint directory holding adapter.pt")+ ap.add_argument("--out", required=True)+ a = ap.parse_args()+ saved = torch.load(f"{a.checkpoint.rstrip('/')}/adapter.pt", map_location="cpu", weights_only=True)+ meta = saved["metadata"]+ state = {k.removeprefix("base_model.model."): v.to(torch.bfloat16).contiguous() for k, v in saved["state"].items()}+ if not all(k.endswith((".lora_A.weight", ".lora_B.weight")) for k in state):+ raise SystemExit("adapter.pt holds more than LoRA pairs")+ save_file(state, a.out, metadata={"format": "pt", "rank": str(meta["rank"]), "alpha": str(meta["alpha"])})+ print(f"{len(state) // 2} LoRA pairs, rank {meta['rank']}, alpha {meta['alpha']} -> {a.out}")+++if __name__ == "__main__":+ main()diff --git a/scripts/prepare_h3.py b/scripts/prepare_h3.pynew file mode 100644index 0000000..8f9ce00--- /dev/null+++ b/scripts/prepare_h3.py@@ -0,0 +1,60 @@+"""Download MiniMax-H3 and point its modular pipeline at the local copy.++ python scripts/prepare_h3.py --dir ./MiniMax-H3++SolarRenderer reads the released Ref2VA transformer (``transformer_ref``), the Qwen3-VL text encoder, its processor and+tokenizer, and the video VAE. Training reads them through <dir>/ref2va, links that name the Ref2VA transformer+``transformer``. The repository's ``modular_model_index.json`` names every component by its Hub id; it is+rewritten to name the local directory, so encoding runs offline.+"""++from __future__ import annotations++import argparse+import json+from pathlib import Path++REPO = "MiniMaxAI/MiniMax-H3"+NEEDED = ["transformer_ref/*", "text_encoder/*", "tokenizer/*", "processor/*", "vae/*", "audio_vae/*", "scheduler/*",+ "audio_scheduler/*", "model_index.json", "modular_model_index.json", "LICENSE"]+++def localize(directory: Path) -> None:+ index = directory / "modular_model_index.json"+ spec = json.loads(index.read_text())+ for value in spec.values():+ if isinstance(value, list) and len(value) == 3 and isinstance(value[2], dict):+ value[2]["pretrained_model_name_or_path"] = str(directory)+ index.write_text(json.dumps(spec, indent=2) + "\n")+++def ref2va_layout(directory: Path) -> Path:+ """<dir>/ref2va: the Ref2VA transformer as ``transformer`` beside the VAEs and processors - the layout training+ reads (model.checkpoint_path in configs/), made of links into the download."""++ root = directory / "ref2va"+ root.mkdir(exist_ok=True)+ for name, target in (("transformer", "transformer_ref"), ("vae", "vae"), ("audio_vae", "audio_vae"),+ ("processor", "processor"), ("tokenizer", "tokenizer"), ("scheduler", "scheduler")):+ link = root / name+ if not link.exists() and not link.is_symlink():+ link.symlink_to(Path("..") / target)+ return root+++def main() -> None:+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ parser.add_argument("--dir", required=True, type=Path)+ parser.add_argument("--skip-download", action="store_true", help="only rewrite the index of an existing copy")+ args = parser.parse_args()+ directory = args.dir.resolve()+ if not args.skip_download:+ from huggingface_hub import snapshot_download++ snapshot_download(REPO, local_dir=str(directory), allow_patterns=NEEDED)+ localize(directory)+ print(f"MiniMax-H3 ready at {directory}; training layout at {ref2va_layout(directory)}")+++if __name__ == "__main__":+ main()diff --git a/scripts/render.py b/scripts/render.pynew file mode 100644index 0000000..fa580f9--- /dev/null+++ b/scripts/render.py@@ -0,0 +1,211 @@+"""Render one clip with SolarRenderer: a structure video, an optional first frame and a six-part prompt in, a video out.++ python scripts/render.py --h3 ./MiniMax-H3 \+ --control solarrenderer_control_lora.safetensors --fewstep solarrenderer_4step_lora.safetensors \+ --structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4++ # streaming: the causal pair, rendered block by block (5 latents, about 20 frames) through a key/value cache+ python scripts/render.py --h3 ./MiniMax-H3 --causal \+ --control solarrenderer_causal_control_lora.safetensors --fewstep solarrenderer_causal_4step_lora.safetensors \+ --structure structure.mp4 --first first.png --prompt prompt.txt --out render.mp4++The inputs are encoded exactly as the training samples were (tools/omni_preencode.py): the structure video laid over+every target frame, the first frame over latent 0, the prompt through the released Qwen3-VL presentation. The+renderer is MiniMax-H3's Ref2VA transformer with the four-step LoRA merged into its weights and the control LoRA on+top, sampled in four evaluations with fresh noise between them. Output: 1344x768 (or 768x1344), 24 fps, 158 frames.+With --causal the causal pair renders block-causally: each block of at most 5 latents sees the prompt, the first frame,+the structure of the last 6 blocks and the clean frames of the 5 blocks before it - never a later block - and is+sampled in four evaluations before it is committed to the cache.++The text encoder and the transformer do not share the card: the clip is encoded first, the encoder is released, then+the transformer is loaded. One GPU runs it end to end. eval/render_benchmark.py uses the same functions to+render many clips with each model loaded once.+"""++from __future__ import annotations++import argparse+import gc+import json+import os+import sys+import tempfile+import time+from pathlib import Path++HERE = Path(__file__).resolve().parent+sys.path.insert(0, str(HERE.parent / "src"))+sys.path.insert(0, str(HERE.parent / "tools"))++FRAMES = 158+RANK = ALPHA = 384+SILENCE = HERE.parent / "assets" / "h3_silence_153_158_170.safetensors"+++def parse() -> argparse.Namespace:+ p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ p.add_argument("--h3", required=True, help="local MiniMaxAI/MiniMax-H3 (scripts/prepare_h3.py)")+ p.add_argument("--control", required=True, help="solarrenderer_control_lora.safetensors")+ p.add_argument("--fewstep", required=True, help="solarrenderer_4step_lora.safetensors")+ p.add_argument("--structure", required=True, help="the structure video, frame-aligned with the target")+ p.add_argument("--first", default="", help="the first frame (optional)")+ p.add_argument("--prompt", required=True, help="a text file holding the six-part prompt (skills/solarrenderer-caption)")+ p.add_argument("--out", required=True, help="the output .mp4")+ p.add_argument("--seed", type=int, default=0)+ p.add_argument("--forwards", type=int, default=4, help="model evaluations; the four-step LoRA was trained for 4")+ p.add_argument("--causal", action="store_true", help="the causal LoRA pair: stream block by block")+ p.add_argument("--device", default="cuda:0")+ return p.parse_args()+++# ---------------------------------------------------------------------------------------------- encoding++def load_encoder(h3: str, device: str):+ """The released ref2va front end (text encoder, processor, reference VAE encoder), as training encoded with it."""++ os.environ["H3_REF_ROOT"] = str(Path(h3).resolve())+ import omni_preencode as pre++ pre.ROOT = os.environ["H3_REF_ROOT"]+ return pre, pre.build_pipelines(device)+++def sample_of(name: str, prompt: str, structure: str | None, first: str | None, target: str | None = None) -> dict:+ """A clip's inputs as a training sample. The target only sets the canvas and the length - sampling never reads its+ latents - so it is the structure video itself, or a given clip when there is no structure video."""++ observations = []+ if structure:+ observations.append({"media": structure, "frames": "all"})+ if first:+ observations.append({"media": first, "frames": [0]})+ return {"id": name, "target": target or structure, "text": prompt, "observations": observations}+++def encode_sample(encoder, sample: dict, member: str) -> None:+ from safetensors.torch import save_file++ pre, pipes = encoder+ tensors, meta = pre.encode(sample, pipes, FRAMES)+ save_file(tensors, member, metadata={"omni": json.dumps(meta)})+++def free_cuda() -> None:+ """After the caller dropped its last reference to the encoder."""++ import torch++ gc.collect()+ torch.cuda.empty_cache()+++# ---------------------------------------------------------------------------------------------- rendering++def lora_pairs(path: str) -> tuple[dict, float]:+ from safetensors import safe_open+ from safetensors.torch import load_file++ with safe_open(path, framework="pt") as handle:+ meta = handle.metadata() or {}+ return load_file(path), float(meta.get("alpha", ALPHA))+++CAUSAL = {"mode": "teacher", "chunk": 5, "window": 6, "corrupt": 0.0} # the causal pair's training plan+++def load_renderer(h3: str, control: str, fewstep: str, device: str, causal: bool = False):+ """MiniMax-H3's Ref2VA transformer, the four-step LoRA merged in, the control LoRA on top; returns a renderer."""++ import torch++ from solarwm.backends.minimax_h3.artifacts import load_silence_latents+ from solarwm.backends.minimax_h3.lora import inject_h3_lora, merge_lora_delta+ from solarwm.backends.minimax_h3.observations import observation_profile_from_train_config+ from solarwm.backends.minimax_h3.optional import load_conditioners, load_transformer+ from solarwm.backends.minimax_h3.stage0p5 import H3Stage0p5Core++ torch.cuda.set_device(torch.device(device))+ model_cfg = {+ "checkpoint_path": str(Path(h3).resolve()),+ "transformer_subfolder": "transformer_ref",+ "torch_dtype": "bfloat16",+ "attention_backend": "flash",+ "adapter": {"rank": RANK, "alpha": ALPHA, "dropout": 0.0},+ }+ modules = load_transformer(model_cfg, device=torch.device(device))+ pairs, alpha = lora_pairs(fewstep)+ rank = int(next(v for k, v in pairs.items() if k.endswith("lora_A.weight")).shape[0])+ merged = merge_lora_delta(modules.transformer, pairs, alpha=alpha, rank=rank, scale=1.0)+ print(f"four-step LoRA merged into {merged} weights", flush=True)+ wrapped, lora = inject_h3_lora(modules.transformer, model_cfg["adapter"], base_identity={})+ pairs, _alpha = lora_pairs(control)+ lora.load_state_dict({f"base_model.model.{k}": v for k, v in pairs.items()}, broadcast=False)+ conditioners = load_conditioners(+ model_cfg, device=torch.device(device), qwen=False, video_vae=True, audio_vae=False, schedulers=False+ )+ silence, _ = load_silence_latents(str(SILENCE), pixel_frames=FRAMES)+ if causal:+ from solarwm.backends.minimax_h3.omni_causal import H3OmniCausalCore++ core = H3OmniCausalCore(wrapped, silence, torch.device(device), dict(CAUSAL), geometry=None,+ observations=observation_profile_from_train_config({}))+ else:+ core = H3Stage0p5Core(wrapped, silence, torch.device(device), geometry=None,+ observations=observation_profile_from_train_config({}), flow_map=False)+ return core, conditioners, torch.device(device)+++def render_member(renderer, member: str, out: str, seed: int = 0, forwards: int = 4) -> float:+ """Sample one encoded clip and write its mp4; returns the sampling time in seconds."""++ import torch++ from solarwm.backends.minimax_h3.inference import _mp4_bytes, decode_h3_latents+ from solarwm.backends.minimax_h3.omni_data import H3OmniStream++ core, conditioners, device = renderer+ stream = object.__new__(H3OmniStream)+ stream.root, stream.current_plan_fingerprint, stream.aligned_control = "/", "render", False+ batch = stream.read({"id": Path(out).stem, "member": member})+ began = time.perf_counter()+ with torch.no_grad():+ if hasattr(core, "stream"):+ # block by block: each block's clean estimate re-noised between evaluations, as the student was trained;+ # the structure keys and values are recomputed per evaluation so one card holds the clip+ from solarwm.backends.minimax_h3.flow import make_shifted_schedule++ torch.manual_seed(seed)+ latents, _inputs, _times, _jitters = core.stream(+ batch, generator=torch.Generator(device=device).manual_seed(seed),+ schedule=make_shifted_schedule(forwards + 1, shift=12.0), exit_index=forwards - 1,+ cache_structure=False,+ )+ else:+ # generate() counts schedule points: N points run N - 1 evaluations+ latents = core.generate(batch, noise_seed=seed, num_inference_steps=forwards + 1, renoise=True)+ took = time.perf_counter() - began+ decoded = decode_h3_latents(conditioners.video_vae, latents, device=device)+ Path(out).parent.mkdir(parents=True, exist_ok=True)+ Path(out + ".tmp").write_bytes(_mp4_bytes(decoded))+ os.replace(out + ".tmp", out)+ return took+++def main() -> None:+ args = parse()+ with tempfile.TemporaryDirectory() as work:+ began = time.perf_counter()+ encoder = load_encoder(args.h3, args.device)+ member = f"{work}/member.safetensors"+ prompt = Path(args.prompt).read_text()+ encode_sample(encoder, sample_of(Path(args.out).stem, prompt, args.structure, args.first or None), member)+ del encoder+ free_cuda()+ print(f"encoded in {time.perf_counter() - began:.0f} s", flush=True)+ renderer = load_renderer(args.h3, args.control, args.fewstep, args.device, causal=args.causal)+ took = render_member(renderer, member, args.out, args.seed, args.forwards)+ print(f"sampled {args.forwards} steps in {took:.0f} s; wrote {args.out}", flush=True)+++if __name__ == "__main__":+ main()diff --git a/skills/solarrenderer-caption/SKILL.md b/skills/solarrenderer-caption/SKILL.mdnew file mode 100644index 0000000..66f1b33--- /dev/null+++ b/skills/solarrenderer-caption/SKILL.md@@ -0,0 +1,119 @@+---+name: solarrenderer-caption+description: Write the text prompt for SolarRenderer, a neural renderer that turns a structure video (grey box, semantic map, masks and skeletons, depth, sketch, point cloud, ...) plus an optional first frame into a finished video. Use when preparing a SolarRenderer input, captioning clips for SolarRenderer training data, or fixing a prompt that renders badly.+---++# Writing SolarRenderer prompts++SolarRenderer was trained on prompts in one fixed six-part form. A prompt in that form, describing the target video+concretely, is the single biggest lever on output quality; a short free-form sentence works much worse.++## The six parts, in this order++```+subject_definitions:+<Video 1> is <what the structure clip is>.+<Video 2> is the first frame of [Shot 1], held still.++summary:+[keyframe completion + reference generation] The target video begins from <Video 2> and follows the frame-by-frame structure and motion of <Video 1>. <one short paragraph: who is where doing what, how the camera moves>++retention_analysis:+<Video 1> <what of it is followed, what it does not give, what of its look is not shown>.+<Video 2> ([Shot 1] first frame): fully_preserved - the composition, subjects, materials, colours and lighting of <Video 2> are the target video's first frame.++detailed_description:+<1-2 sentences on visual style and light> [Shot 1] The shot begins from <Video 2>. <first-frame state, then what happens, in playback order>++overall_soundscape:+<1-4 sentences of ambience and physical sounds>++non_diegetic_music:+N/A+```++Everything except the summary paragraph, the detailed description and the two sound parts is fixed text. Do not+paraphrase it: `conditions.py` holds the exact sentences and `assemble()` builds the prompt from the free parts.++## Which inputs, which labels++| Inputs given | Task type in the summary | Labels |+|---|---|---|+| structure + first frame | `[keyframe completion + reference generation]` | structure `<Video 1>`, first frame `<Video 2>` |+| structure only | `[reference generation]` | structure `<Video 1>` |+| first frame only | `[keyframe completion]` | first frame `<Video 1>` |+| text only | none: write `integrated_multimodal_description: [Shot 1] ...`, `overall_soundscape:`, `non_diegetic_music:` | none |++Define only the inputs actually given, and never use a label that is not defined.++## The structure sentence++For the six trained condition types use the sentence in `conditions.DEFINE` word for word: `scene_graybox`+(untextured grey block-out, Magpie / engine white-box), `semantic` (CARLA-style class colours), `mask_skeleton`+(flat object masks with skeletons), `near_depth` (grey shaded engine depth, black beyond ~30 m), `smpl_mesh`+(grey body mesh on white), `scene_whitebox` (untextured grey rendering of the scene and the main character).++For any other structure, say plainly what the clip looks like and what of its look must not appear, with+`conditions.custom_condition(looks_like, not_shown)`:++```python+custom_condition("a sparse 3D point cloud scan of the scene and its characters, white points on black",+ "the dotted point look")+custom_condition("a black pencil line sketch of the scene on white paper, outlines only, like an animated storyboard",+ "the pencil lines and white paper")+```++## Writing the free parts++- **summary**: one short paragraph after the fixed lead: who is where doing what, and how the camera moves.+- **detailed_description**: 350-500 words (aim for about 420).+ - Open with one or two sentences on the visual style and light ("The target video is a realistic 3D CG game render+ in bright midday sun with crisp shadows."), then `[Shot 1]`.+ - With a first frame, `[Shot 1]` opens with `The shot begins from <Video 2>.` (assemble() adds it).+ - Describe the first frame's composition, subjects and scene, then what happens in playback order.+ - Spend the words on what the structure does not carry: materials and surfaces with colours and wear; light+ (direction, colour, hardness, time of day, shadows, reflections, emissive lights); weather and atmosphere;+ effects (sparks, dust, smoke, particles, glows); characters' full appearance (each outfit piece with colour and+ material, gear, hair, skin); small objects; events and state changes; camera motion (type, amplitude, speed).+ - Keep what the structure fixes consistent with it: same layout, same number of characters, the same facing+ (if we see someone's back, say so), the same camera path.+ - Time events in words ("midway", "near the end"). A timestamp appears only at a hard cut:+ `[Shot 2] At 00:03.500, the shot cuts to ...`. Most clips are one shot.+- **overall_soundscape**: ambience and physical sounds; **non_diegetic_music**: `N/A` unless it is a scored trailer.+- English only; on-screen text is quoted exactly in its original language.+- No labels such as `<Video 1>` inside the free parts you write (assemble() places them), no guesses about story or+ names, no mood words standing in for visible detail.++To restyle a clip (same structure, another world), write the description of the world you want, keeping what the+structure fixes. The first frame, if given, must already be in that world.++## The first frame++A first frame is optional, but it is the strongest control over the look: the render starts from it and keeps its+materials, colours and light. If the user has none, make one with an image model (Codex's image generation, or any+model that edits a reference image), after the prompt is written, so that the two agree:++1. `python first_frame.py frame0 structure.mp4 frame0.png` - frame 0 of the structure clip, the reference image.+2. `python first_frame.py prompt prompt.txt` (add `--stylised` for pixel art, toon, clay, painterly and other+ non-photoreal looks) - the image prompt: keep the camera, layout, characters, poses and facing of the reference+ exactly; replace the grey or flat surfaces with the look of the detailed description's opening sentences.+3. Generate a 16:9 image from `frame0.png` with that prompt (9:16 for a portrait clip).+4. `python first_frame.py fit generated.png first.png` - crop and scale to 1344x768 (`768x1344` for portrait).+5. Check it against `frame0.png`: same number of characters, same facing, same horizon and layout. An image that+ moved things will fight the structure; regenerate it. If you have a frame of the real scene with the same layout+ (a photo, a game capture), use it as the reference instead of the structure frame - image models restyle a real+ frame more faithfully than a grey one.++The prompt must then describe that image: its `[Shot 1]` opens with what the first frame shows.++## Steps++1. Look at the structure clip (and the first frame, if any): count the characters, note their facing and the camera+ path, the layout and what the structure leaves open.+2. Write the free parts as JSON `{summary, detailed_description, overall_soundscape, non_diegetic_music}`.+3. `assemble(free, inputs, condition)` from `conditions.py`.+4. `python check.py prompt.txt` - sections in order, labels defined, task type present, first-frame opening, length,+ English, shot markers. Fix and re-check until it prints OK.++For many clips, `free_caption.py` (in the repository's `data` branch) does steps 1-2 with a vision-language model (it sees the finished video, for+training data) and checks every answer.diff --git a/skills/solarrenderer-caption/check.py b/skills/solarrenderer-caption/check.pynew file mode 100644index 0000000..e654cec--- /dev/null+++ b/skills/solarrenderer-caption/check.py@@ -0,0 +1,125 @@+"""check.py: checks for SolarRenderer prompts.++ python check.py prompt.txt [more.txt ...] check finished prompts, one per file (exit 1 if any fails)++check_free(answer, duration) cleans and checks the free parts a captioning model wrote (used by free_caption.py);+check_prompt(text) checks a finished six-part prompt, written by assemble() or by hand.+"""+import re, sys++FIELDS = ["summary", "detailed_description", "overall_soundscape", "non_diegetic_music"]+SECTIONS = ["subject_definitions", "summary", "retention_analysis", "detailed_description", "overall_soundscape",+ "non_diegetic_music"]+CJK = re.compile(r"[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uac00-\ud7af\uff00-\uffef\u3000-\u303f]")+QUOTED = re.compile(r"\"[^\"]*\"|β[^β]*β|γ[^γ]*γ")+LABEL = re.compile(r"<\s*(Subject|Picture|Video|Audio)\s*\d+\s*>")+++def normalize_shots(d):+ """Two slips captioning models keep making, fixed in code: "[Shot 1] At 00:00.000, the clip opens ..." (the first+ shot carries no timestamp) and a cut written as "At 00:03.500, the shot cuts to ..." without "[Shot N]"."""+ d = re.sub(r"\[Shot 1\]\s*At 00:00(?:\.0+)?,?\s*(\w)", lambda m: "[Shot 1] " + m.group(1).upper(), d)+ d = re.sub(r"(?<!\] )(?<!\]\s)\bAt (\d{2}:\d{2}\.\d{3}), the (shot|camera|scene) (cuts|switches|changes|transitions)",+ r"[Shot N] At \1, the \2 \3", d)+ n = iter(range(1, 1000))+ return re.sub(r"\[Shot (?:\d+|N)\]", lambda m: f"[Shot {next(n)}]", d)+++def shots_ok(d, duration):+ """[Shot 1] first and without a timestamp; each later shot opens with "At MM:SS.mmm", strictly increasing inside the+ clip; no other timestamp anywhere."""+ shots = [(m.start(), int(m.group(1))) for m in re.finditer(r"\[Shot (\d+)\]", d)]+ ok, prev = True, 0.0+ if not shots or shots[0][1] != 1 or re.match(r"\[Shot 1\]\s*At \d", d[shots[0][0]:]):+ ok = False+ for i, (pos, n) in enumerate(shots[1:], start=2):+ m = re.match(r"\[Shot \d+\]\s*At (\d{2}):(\d{2})\.(\d{3})", d[pos:])+ if n != i or not m:+ ok = False+ continue+ t = int(m.group(1)) * 60 + int(m.group(2)) + int(m.group(3)) / 1000+ ok &= prev < t < duration+ prev = t+ stray = len(re.findall(r"\d{1,2}:\d{2}(?:\.\d+)?", d)) - len(shots[1:])+ return ok and stray == 0, len(shots), stray+++def check_free(ans, duration):+ f = {k: str(ans.get(k) or "").strip() for k in FIELDS}+ f["summary"] = re.sub(r"^(\[[^\]]*\]\s*)+[.:;,\s]*", "", f["summary"])+ f["detailed_description"] = normalize_shots(f["detailed_description"])+ s = ans.get("subject")+ f["subject"] = s.strip().rstrip(".") if isinstance(s, str) and s.strip() and \+ s.strip().lower() not in ("null", "none", "n/a") else None+ c = {"fields": all(f[k] for k in FIELDS)}+ c["words"] = len(f["detailed_description"].split())+ c["length"] = 350 <= c["words"] <= 500+ text = " ".join(f[k] for k in FIELDS) + " " + (f["subject"] or "")+ c["english"] = not CJK.search(QUOTED.sub("", text))+ c["no_labels"] = not LABEL.search(text)+ c["cuts"], c["shots"], c["stray_timestamps"] = shots_ok(f["detailed_description"], duration)+ c["pass"] = all(c[k] for k in ("fields", "length", "english", "no_labels", "cuts"))+ return f, c+++def problems(c):+ p = []+ if not c.get("fields"):+ p.append("every field must be filled")+ if not c.get("length"):+ p.append(f"detailed_description has {c.get('words')} words; it must have 350-500 (aim for about 420)"+ + ("; add concrete visible detail" if c.get("words", 0) < 350 else "; tighten it"))+ if not c.get("english"):+ p.append("only English is allowed outside double-quoted on-screen text")+ if not c.get("no_labels"):+ p.append("remove every reference label such as <Picture 1> or <Video 1>")+ if not c.get("cuts"):+ p.append("[Shot 1] has no timestamp; a later shot starts '[Shot N] At MM:SS.mmm, the shot cuts to', times "+ "strictly increasing inside the clip; no other timestamps anywhere (describe timing inside a shot in "+ "words: 'midway', 'near the end')")+ return p+++def check_prompt(text, duration=6.583):+ """Problems of a finished prompt (an empty list when it is fine)."""+ p = []+ if text.startswith("integrated_multimodal_description:"): # text only+ body = text.split("\n\noverall_soundscape:")[0]+ if LABEL.search(text):+ p.append("a text-only prompt names no inputs")+ words = len(body.split()) - 1+ else:+ heads = re.findall(r"^([a-z_]+):\s*$", text, flags=re.M)+ if heads != SECTIONS:+ p.append(f"sections must be {SECTIONS} in this order, found {heads}")+ return p+ part = dict(zip(SECTIONS, re.split(r"^[a-z_]+:\s*$", text, flags=re.M)[1:]))+ defined = set(re.findall(r"^(<Video \d+>) is ", part["subject_definitions"], flags=re.M))+ used = set(m.group(0) for m in LABEL.finditer(text))+ if used - defined:+ p.append(f"labels used but not defined: {sorted(used - defined)}")+ tasks = re.match(r"\s*\[([a-z +]+)\]", part["summary"])+ if not tasks:+ p.append("summary must open with the task type, e.g. [keyframe completion + reference generation]")+ first = re.search(r"^(<Video \d+>) is the first frame of \[Shot 1\]", part["subject_definitions"], flags=re.M)+ if first and f"[Shot 1] The shot begins from {first.group(1)}." not in part["detailed_description"]:+ p.append(f"with a first frame, [Shot 1] must open with 'The shot begins from {first.group(1)}.'")+ body = part["detailed_description"]+ words = len(body.split())+ if not 350 <= words <= 520: # the fixed opening sentence adds a few words to the free 350-500+ p.append(f"detailed_description has {words} words; it should have about 350-500")+ if CJK.search(QUOTED.sub("", text)):+ p.append("only English outside double-quoted on-screen text")+ ok, _, stray = shots_ok(body, duration)+ if not ok:+ p.append("shot markers: [Shot 1] without a timestamp, later shots open with 'At MM:SS.mmm', no other timestamps")+ return p+++if __name__ == "__main__":+ bad = 0+ for path in sys.argv[1:]:+ issues = check_prompt(open(path).read())+ print(path, "OK" if not issues else "; ".join(issues))+ bad += bool(issues)+ sys.exit(1 if bad else 0)diff --git a/skills/solarrenderer-caption/conditions.py b/skills/solarrenderer-caption/conditions.pynew file mode 100644index 0000000..a2bf423--- /dev/null+++ b/skills/solarrenderer-caption/conditions.py@@ -0,0 +1,110 @@+"""Fixed sentences per condition type, and the assembly of a SolarRenderer prompt.++A prompt has six parts, in this order:+ subject_definitions, summary, retention_analysis, detailed_description, overall_soundscape, non_diegetic_music+The parts that say what each input is and how far it is followed are fixed per condition type (this file). The free+parts - what the finished video looks like - are written per clip (free_caption.py, or by hand with SKILL.md).++The sentences of the six trained condition types are the ones training used, word for word. Any other structure+(a sketch, a point cloud, a wireframe ...) is described with `custom_condition`, in the same form.+"""+import re++# condition type -> (what <V> is, what of it is followed / not given / not shown); {v} is the label+DEFINE = {+ "scene_graybox": ("an untextured grey block-out of the scene, rendered from the same camera as the target video, "+ "giving the layout, the large geometry, the characters' bodies, their motion and the camera of every frame",+ "(frame-by-frame layout, geometry, motion and camera): attribute_transfer - the layout, large shapes, "+ "character bodies, motion and camera are followed exactly; small objects, materials, colours and "+ "lighting are not given by {v} and are the target video's own; the grey look of {v} is not shown"),+ "semantic": ("a semantic segmentation of the scene, one flat colour per class (road, sidewalk, building, vegetation, "+ "vehicle, pedestrian, sky and so on), frame by frame, from the same camera as the target video",+ "(frame-by-frame layout of every class and camera): attribute_transfer - every region keeps its class, "+ "outline and place; materials, colours and lighting are not given by {v}; the flat class colours are "+ "not shown"),+ "scene_whitebox": ("an untextured grey rendering of the scene and the main character, from the same camera "+ "as the target video, giving the layout of the scene, the character's pose and motion, and the "+ "camera of every frame",+ "(frame-by-frame layout, character pose and motion, camera): attribute_transfer - the layout, the "+ "character's pose and motion and the camera are followed; materials, colours, lighting and any "+ "added detail are the target video's own; the grey look of {v} is not shown"),+ "mask_skeleton": ("the objects of the scene as flat-coloured masks, one colour per object, with the skeletons of the "+ "characters drawn on top, frame by frame, from the same camera as the target video",+ "(frame-by-frame object outlines, character poses and camera): attribute_transfer - every object "+ "keeps its outline and place, every character keeps its pose and motion; materials, colours and "+ "lighting are not given by {v}; the flat mask colours and skeleton lines are not shown"),+ "smpl_mesh": ("a grey 3D body mesh of the main character on a plain white background, rendered from the same camera "+ "as the target video; it gives the character's position, size, body pose and motion in every frame, "+ "and nothing of the surroundings",+ "(frame-by-frame body pose and motion of the main character): attribute_transfer - the main character's "+ "position, size, pose and motion are followed; the character's clothing, face and hair, the surroundings, "+ "the camera's view of the scene, materials, colours and lighting are not given by {v} and are the target "+ "video's own; the grey mesh and the white background of {v} are not shown"),+ "near_depth": ("a grey shaded rendering of the game engine's depth within about 30 metres of the camera, from the "+ "same camera as the target video; everything farther, and the sky, is black. It gives the exact "+ "shapes, positions and motion of the nearby scene, characters and vehicles, and the camera of every "+ "frame",+ "(frame-by-frame near geometry, motion and camera): attribute_transfer - the shapes, positions and "+ "motion of everything within the near range and the camera are followed; the far scene, materials, "+ "colours and lighting are not given by {v} and are the target video's own; the grey shading and the "+ "black background of {v} are not shown"),+}++INPUTS = ("full", "structure", "first", "text") # structure + first frame / structure only / first frame only / text only+++def custom_condition(looks_like: str, not_shown: str):+ """Fixed sentences for a structure the model was not trained on, in the form the benchmark uses, e.g.+ custom_condition("a black pencil line sketch of the scene on white paper, outlines only, like an animated storyboard",+ "the pencil lines and white paper")"""+ return (f"{looks_like}, from the same camera as the target video",+ "(frame-by-frame layout, shapes, character poses and motion, camera): attribute_transfer - the layout, "+ "shapes, poses, motion and camera are followed exactly; materials, colours and lighting are not given by "+ "{v} and are the target video's own; " + f"{not_shown} of {{v}} is not shown")+++def assemble(free: dict, inputs: str = "full", condition="scene_graybox") -> str:+ """The full prompt from the free parts (summary, detailed_description, overall_soundscape, non_diegetic_music).++ inputs: which inputs the model is given - "full" (structure + first frame), "structure", "first" or "text".+ condition: a key of DEFINE, or a (what, kept) pair from custom_condition. Labels follow the order of the inputs:+ the structure clip is <Video 1>; the first frame is <Video 2>, or <Video 1> without a structure clip.+ """+ assert inputs in INPUTS, inputs+ detail = free["detailed_description"].strip()+ sound = (free.get("overall_soundscape") or "N/A").strip() or "N/A"+ music = (free.get("non_diegetic_music") or "N/A").strip() or "N/A"+ summary = free["summary"].strip()+ if inputs == "text": # text only: the three core fields, the style inside [Shot 1]+ body = "[Shot 1] " + re.sub(r"\[Shot 1\]\s*", "", detail, count=1)+ return f"integrated_multimodal_description: {body}\n\noverall_soundscape: {sound}\n\nnon_diegetic_music: {music}"+ what, kept = DEFINE[condition] if isinstance(condition, str) else condition+ lines, keep, tasks = [], [], []+ structure = inputs in ("full", "structure")+ first = inputs in ("full", "first")+ s_lab = "<Video 1>" if structure else None+ f_lab = ("<Video 2>" if structure else "<Video 1>") if first else None+ if structure:+ lines.append(f"{s_lab} is {what}.")+ keep.append(f"{s_lab} {kept.format(v=s_lab)}.")+ tasks.append("reference generation")+ if first:+ lines.append(f"{f_lab} is the first frame of [Shot 1], held still.")+ keep.append(f"{f_lab} ([Shot 1] first frame): fully_preserved - the composition, subjects, materials, colours "+ f"and lighting of {f_lab} are the target video's first frame.")+ tasks.insert(0, "keyframe completion")+ lead = []+ if first:+ lead.append(f"The target video begins from {f_lab}")+ if structure:+ lead.append(f"follows the frame-by-frame structure and motion of {s_lab}" if first else+ f"The target video follows the frame-by-frame structure and motion of {s_lab}")+ head = (" and ".join(lead) + ". ") if lead else ""+ if first: # the opening shot starts from the first frame+ detail = re.sub(r"\[Shot 1\]\s*", f"[Shot 1] The shot begins from {f_lab}. ", detail, count=1) \+ if "[Shot 1]" in detail else f"[Shot 1] The shot begins from {f_lab}. " + detail+ return ("subject_definitions:\n" + "\n".join(lines) + "\n\n"+ f"summary:\n[{' + '.join(tasks)}] {head}{summary}\n\n"+ "retention_analysis:\n" + "\n".join(keep) + "\n\n"+ f"detailed_description:\n{detail}\n\n"+ f"overall_soundscape:\n{sound}\n\nnon_diegetic_music:\n{music}")diff --git a/skills/solarrenderer-caption/first_frame.py b/skills/solarrenderer-caption/first_frame.pynew file mode 100644index 0000000..3f60bf2--- /dev/null+++ b/skills/solarrenderer-caption/first_frame.py@@ -0,0 +1,56 @@+"""first_frame.py: the image prompt for a SolarRenderer first frame, and the crop that fits it to the canvas.++ python first_frame.py frame0 structure.mp4 frame0.png # frame 0 of the structure clip, the reference image+ python first_frame.py prompt prompt.txt [--stylised] # the image prompt, from a finished six-part prompt+ python first_frame.py fit generated.png first.png [768x1344] # crop and scale to the canvas (default 1344x768)++The first frame is made by an image model that edits frame 0 of the structure clip: same camera, same layout, same+characters in the same pose and facing, with the look the prompt describes. Any image model that takes a reference+image works (Codex's image generation, gpt-image, Nano Banana, ...).+"""+import re+import subprocess+import sys++KEEP = ("Re-render the reference image with IDENTICAL camera, framing, perspective and composition: every object, "+ "building, prop, terrain shape and character stays exactly where it is, at the same size and with the same "+ "outline and pose, pixel-aligned with the reference. Every character keeps the same facing direction as in "+ "the reference (if we see their back, we still see their back), the same pose and the same size. Only replace "+ "the untextured grey or flat-coloured surfaces with final materials, lighting and detail. ")+QUALITY = (" Masterful cinematic frame from a next-generation AAA game or film: physically based materials, rich "+ "micro-detail, volumetric light and atmosphere, deliberate colour grading, sharp focus on the subject. "+ "No text, no logos, no UI, no watermark, no borders.")+# Stylised looks (pixel art, toon, painterly, clay, ...) must not be pulled back to photoreal by the line above.+QUALITY_STYLISED = (" A polished, faithful frame rendered strictly in the art style described above - do not make it "+ "photorealistic, keep every hallmark of that style (its geometry, texture resolution, palette, "+ "shading and rendering artefacts). No text, no logos, no UI, no watermark, no borders.")+++def image_prompt(prompt: str, stylised: bool = False) -> str:+ """KEEP + the style sentences of the detailed description + the first three sentences of [Shot 1] + quality."""++ detail = prompt.split("detailed_description:", 1)[-1].split("overall_soundscape:", 1)[0]+ style, _, shot = detail.partition("[Shot 1]")+ shot = re.sub(r"^\s*The shot begins from <Video \d>\.\s*", "", shot.strip())+ opening = " ".join(re.split(r"(?<=[.!?])\s+", shot)[:3])+ return KEEP + style.strip() + " The first moment of the shot: " + opening + (QUALITY_STYLISED if stylised else QUALITY)+++def main() -> None:+ if len(sys.argv) < 2:+ raise SystemExit(__doc__)+ cmd, *rest = sys.argv[1:]+ if cmd == "frame0":+ subprocess.run(["ffmpeg", "-loglevel", "error", "-y", "-i", rest[0], "-frames:v", "1", rest[1]], check=True)+ elif cmd == "prompt":+ print(image_prompt(open(rest[0]).read(), stylised="--stylised" in rest))+ elif cmd == "fit":+ w, h = (rest[2] if len(rest) > 2 else "1344x768").split("x")+ subprocess.run(["ffmpeg", "-loglevel", "error", "-y", "-i", rest[0], "-vf",+ f"scale={w}:{h}:force_original_aspect_ratio=increase,crop={w}:{h}", rest[1]], check=True)+ else:+ raise SystemExit(__doc__)+++if __name__ == "__main__":+ main()diff --git a/src/solarwm/__init__.py b/src/solarwm/__init__.pynew file mode 100644index 0000000..8d9bad9--- /dev/null+++ b/src/solarwm/__init__.py@@ -0,0 +1,5 @@+"""SolarWM public package."""++from .version import __version__++__all__ = ["__version__"]diff --git a/src/solarwm/__main__.py b/src/solarwm/__main__.pynew file mode 100644index 0000000..e8de1ad--- /dev/null+++ b/src/solarwm/__main__.py@@ -0,0 +1,8 @@+import os++# a long video sequence fragments the CUDA caching allocator; expandable segments keep training on 8 GPUs in memory+os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")++from .cli import main # noqa: E402++raise SystemExit(main())diff --git a/src/solarwm/backends/__init__.py b/src/solarwm/backends/__init__.pynew file mode 100644index 0000000..c341700--- /dev/null+++ b/src/solarwm/backends/__init__.py@@ -0,0 +1,5 @@+"""Lazy model-family backend registry."""++from .registry import Backend, BackendSpec, load_backend++__all__ = ["Backend", "BackendSpec", "load_backend"]diff --git a/src/solarwm/backends/minimax_h3/__init__.py b/src/solarwm/backends/minimax_h3/__init__.pynew file mode 100644index 0000000..c6a5206--- /dev/null+++ b/src/solarwm/backends/minimax_h3/__init__.py@@ -0,0 +1,9 @@+"""MiniMax-H3 backend plugin.++Importing this package is intentionally safe on CPU-only installations. The+33B model runtime is not imported or constructed at plugin discovery time.+"""++from .backend import MiniMaxH3Backend, create_backend++__all__ = ["MiniMaxH3Backend", "create_backend"]diff --git a/src/solarwm/backends/minimax_h3/anyflow_conditioning.py b/src/solarwm/backends/minimax_h3/anyflow_conditioning.pynew file mode 100644index 0000000..c0e2ad8--- /dev/null+++ b/src/solarwm/backends/minimax_h3/anyflow_conditioning.py@@ -0,0 +1,97 @@+"""Opt-in target-time conditioning for MiniMax-H3 AnyFlow.++Both time inputs use H3's native data-ward coordinate, ``time = 1 - sigma``.+The second MLP is cloned from the loaded base model; enabling this extension+does not initialise random parameters or alter the ordinary FM state dict.+"""++from __future__ import annotations++import copy+import math+from collections.abc import Callable++import torch+++class H3AnyFlowConditioningMixin:+ """Add AnyFlow only after an upstream H3 checkpoint has strict-loaded.++ The owning model supplies ``time_proj`` and ``time_embedder``. Call+ ``enable_anyflow`` before PEFT/FSDP wrapping so the additional FP32 MLP+ participates in the same freezing and sharding rules as the original.+ """++ @property+ def uses_anyflow(self) -> bool:+ return hasattr(self, "delta_embedding")++ def enable_anyflow(self, gate: float = 0.25) -> bool:+ """Clone the loaded time MLP once, preserving RNG and base parameters.++ Return whether the extension was created. Repeating the call with the+ same gate is harmless and never resets a learned delta embedding.+ The gate is configuration, not trainable or persistent model state.+ """++ gate = float(gate)+ if not math.isfinite(gate) or not 0.0 <= gate <= 1.0:+ raise ValueError("H3 AnyFlow gate must be finite and in [0, 1]")+ if self.uses_anyflow:+ if float(self.anyflow_gate_buffer.item()) != float(+ torch.tensor(gate, dtype=self.anyflow_gate_buffer.dtype).item()+ ):+ raise ValueError("H3 AnyFlow is already enabled with a different gate")+ return False+ self.delta_embedding = copy.deepcopy(self.time_embedder)+ self.register_buffer(+ "anyflow_gate_buffer",+ torch.tensor(gate, dtype=torch.float32),+ persistent=False,+ )+ return True++ def initialize_anyflow_delta_from_time(self) -> bool:+ """Reset delta from the currently loaded time MLP before FSDP wrapping."""++ if not self.uses_anyflow:+ return False+ self.delta_embedding.load_state_dict(self.time_embedder.state_dict(), strict=True)+ return True++ def _time_condition(+ self,+ timestep: torch.Tensor,+ r_timestep: torch.Tensor | None = None,+ *,+ parameter_dtype: Callable[[torch.nn.Module], torch.dtype],+ ) -> torch.Tensor:+ """Embed native ``(t, r)`` and mix before block and output modulation.++ The adapter supplies its FSDP-aware dtype lookup. Keeping this lookup+ at the boundary lets the conditioning also run in CPU-only math tests+ without importing the full Diffusers H3 model.+ """++ if self.uses_anyflow:+ if r_timestep is None:+ raise ValueError("H3 AnyFlow model forward requires r_timestep")+ if r_timestep.shape != timestep.shape:+ raise ValueError(+ f"r_timestep shape {tuple(r_timestep.shape)} != "+ f"timestep shape {tuple(timestep.shape)}"+ )+ elif r_timestep is not None:+ raise ValueError("r_timestep requires H3 AnyFlow conditioning")++ temb = self.time_proj(timestep)+ temb = self.time_embedder(temb.to(parameter_dtype(self.time_embedder)))+ if not self.uses_anyflow:+ return temb+ remb = self.time_proj(r_timestep)+ remb = self.delta_embedding(remb.to(parameter_dtype(self.delta_embedding)))+ gate = self.anyflow_gate_buffer.to(device=temb.device, dtype=temb.dtype)+ return (1.0 - gate) * temb + gate * remb+++__all__ = ["H3AnyFlowConditioningMixin"]diff --git a/src/solarwm/backends/minimax_h3/anyflow_loss.py b/src/solarwm/backends/minimax_h3/anyflow_loss.pynew file mode 100644index 0000000..59045ee--- /dev/null+++ b/src/solarwm/backends/minimax_h3/anyflow_loss.py@@ -0,0 +1,81 @@+"""H3 TF AnyFlow v1.5 in the shared noise-ward flow-map coordinate.++The model callback accepts native H3 data-ward times and returns data-ward+velocity. Conditions (clean history, anchor, prompt and silence) are captured+by the caller and remain identical across the four model evaluations.+"""++from __future__ import annotations++from collections.abc import Callable++import torch++from solarwm.training.anyflow import (+ adaptive_rescale_non_diffusion_losses,+ bounded_difference_timesteps,+ central_difference_derivative,+ gaussian_timestep_weights,+)+++def h3_anyflow_v15_loss(+ model_velocity: Callable[[torch.Tensor, torch.Tensor, torch.Tensor], torch.Tensor],+ clean: torch.Tensor,+ noise: torch.Tensor,+ sigma_t: torch.Tensor,+ sigma_r: torch.Tensor,+ is_diffusion: torch.Tensor,+ *,+ epsilon: float,+ shift: float,+ num_train_timesteps: int = 1000,+ process_group=None,+) -> torch.Tensor:+ """Evaluate detached targets before retaining the prediction's gradient graph.++ The callback must be deterministic for fixed inputs (H3 AnyFlow requires+ zero adapter dropout). Every rank executes diagonal, plus, minus, then+ prediction, preserving the same four evaluations and loss expression.+ """+ if clean.shape != noise.shape or clean.shape[0] != sigma_t.numel():+ raise ValueError("H3 AnyFlow clean/noise/time batch shapes disagree")+ shape = (clean.shape[0],) + (1,) * (clean.ndim - 1)+ raw_t = sigma_t.float() * num_train_timesteps+ raw_r = sigma_r.float() * num_train_timesteps+ noisy = (1 - sigma_t.view(shape)) * clean + sigma_t.view(shape) * noise++ def noise_velocity(sample, t, r):+ # H3 predicts x0-noise and embeds data-ward time, unlike Wan's sigma.+ return -model_velocity(+ sample, 1 - t / num_train_timesteps, 1 - r / num_train_timesteps+ ).float()++ with torch.no_grad():+ diagonal = noise_velocity(noisy, raw_t, raw_t)+ plus_t, minus_t = bounded_difference_timesteps(+ raw_t,+ raw_r,+ epsilon=epsilon,+ num_train_timesteps=num_train_timesteps,+ )+ plus = noisy + diagonal * ((plus_t - raw_t) / num_train_timesteps).view(shape)+ minus = noisy - diagonal * ((raw_t - minus_t) / num_train_timesteps).view(shape)+ plus_v = noise_velocity(plus, plus_t, raw_r)+ minus_v = noise_velocity(minus, minus_t, raw_r)+ derivative = central_difference_derivative(plus_v, minus_v, plus_t, minus_t)+ del diagonal, plus, minus, plus_v, minus_v+ prediction = noise_velocity(noisy, raw_t, raw_r)+ residual = prediction + (raw_t - raw_r).view(shape) * derivative - (noise - clean)+ per_sample = residual.square().flatten(1).mean(1)+ per_sample = adaptive_rescale_non_diffusion_losses(+ per_sample,+ is_diffusion,+ process_group=process_group,+ )+ weights = gaussian_timestep_weights(+ raw_t,+ shift=shift,+ num_train_timesteps=num_train_timesteps,+ )+ return (per_sample * weights).mean()diff --git a/src/solarwm/backends/minimax_h3/artifacts.py b/src/solarwm/backends/minimax_h3/artifacts.pynew file mode 100644index 0000000..527b166--- /dev/null+++ b/src/solarwm/backends/minimax_h3/artifacts.py@@ -0,0 +1,932 @@+"""Concrete H3 preencoded WebDataset reader and immutable artifact contracts."""++from __future__ import annotations++import hashlib+import json+from collections.abc import Iterator, Mapping, Sequence+from dataclasses import dataclass, replace+from pathlib import Path+from typing import Any++import numpy as np++from solarwm.data import (+ CanonicalSampler,+ IndexRow,+ ReaderIdentity,+ SamplingConfig,+ TarShardReader,+ read_index,+ resolver_from_config,+ select_index_rows,+)+from solarwm.data.sampling import SamplePlan, plan_fingerprint+from solarwm.errors import DataContractError+from solarwm.preencode import EncoderContract, TensorSpec+from solarwm.runtime import Topology++from .camera import validate_absolute_c2w, validate_normalized_intrinsics+from .codec import H3_PREENCODE_VERSION+from .geometry import STABLE_STAGE0P5_GEOMETRY, latent_aligned_pixel_indices+++class H3CameraFilterError(DataContractError):+ """A valid H3 sample rejected by the camera magnitude filter."""+++H3_GEOMETRY_LATENT_SHAPE = (24, 47, 48, 84)+++H3_GEOMETRY_ENCODING_PROCESS = {+ "schema": "solarwm.minimax-h3-geometry-encoding.v1",+ "source": "uint16_npz_rgb_xyz",+ "resize": "cover_center_crop_nearest",+ "background": "all_channels_zero_to_unit_midpoint",+ "pixel_range": "unit",+ "pixel_normalization": "imagenet",+ "video_vae_posterior": "mode",+ "video_vae_normalization": "config.latents_mean_std",+}+++def h3_geometry_encoding_profile(profile: Any) -> dict[str, Any]:+ """How geometry renders became latents, and which renders were encoded."""++ return {+ **H3_GEOMETRY_ENCODING_PROCESS,+ "modalities": list(profile.modalities),+ "wide_modalities": list(profile.wide_modalities),+ "wide_fov_scale": profile.wide_fov_scale,+ }+++def h3_encoder_contract(+ *,+ encoder_identity: str = "official-minimax-h3",+ silence_artifact_profile: Mapping[str, Any] | None = None,+ geometry: Any = None,+) -> EncoderContract:+ """Readable encoder contract; ``geometry`` adds the ControlNet's render latents."""++ frame_geometry = STABLE_STAGE0P5_GEOMETRY+ extras = {+ "encoder_identity": encoder_identity,+ "qwen_hidden_state": 50,+ "qwen_presentation": "<Picture 1> joint image+caption",+ "video_vae_posterior": "sample(seed=42)-fp16-round",+ "video_vae_normalization": "config.latents_mean_std",+ "source_fps_policy": "audit_only",+ "frame_sampling": "exact_contiguous",+ }+ if silence_artifact_profile is not None:+ silence_profile = json.loads(json.dumps(dict(silence_artifact_profile), sort_keys=True))+ if silence_profile != h3_silence_profile():+ raise DataContractError("H3 silence artifact profile is unsupported")+ extras["silence_artifact_profile"] = silence_profile+ geometry_tensors: tuple[TensorSpec, ...] = ()+ if geometry is not None:+ extras["geometry_encoding"] = h3_geometry_encoding_profile(geometry)+ geometry_tensors = tuple(+ TensorSpec(key, H3_GEOMETRY_LATENT_SHAPE, "bfloat16") for key in geometry.tensor_keys()+ )+ return EncoderContract(+ schema="solarwm.encoder.v1",+ family="minimax_h3",+ format_version=H3_PREENCODE_VERSION,+ pixel_frames=frame_geometry.pixel_frames,+ latent_frames=frame_geometry.encoded_latents,+ height=frame_geometry.height,+ width=frame_geometry.width,+ camera_convention="absolute_c2w+normalized_K",+ tensors=(+ TensorSpec("target_latents", (24, 47, 48, 84), "bfloat16"),+ TensorSpec("anchor_latents", (24, 1, 48, 84), "bfloat16"),+ TensorSpec("prompt_embeds", (None, 5120), "bfloat16"),+ TensorSpec("text_token_tags", (None,), "int64"),+ TensorSpec("source_frame_indices", (158,), "int64"),+ TensorSpec("camera_c2w", (158, 4, 4), "float32"),+ TensorSpec("camera_K", (158, 3, 3), "float32"),+ *geometry_tensors,+ ),+ extras=extras,+ )+++def require_geometry_encoder_profile(encoder_profile: Mapping[str, Any], geometry: Any) -> None:+ """Fail before training when the latent release does not carry the profile's renders."""++ if geometry is None:+ return+ specs = {+ str(spec.get("name")): spec+ for spec in encoder_profile.get("tensors", ())+ if isinstance(spec, Mapping)+ }+ for key in geometry.tensor_keys():+ spec = specs.get(key)+ if spec is None:+ raise DataContractError(f"H3 encoder contract lacks the geometry tensor {key!r}")+ if tuple(spec.get("shape") or ()) != H3_GEOMETRY_LATENT_SHAPE or (+ spec.get("dtype") != "bfloat16"+ ):+ raise DataContractError(f"H3 encoder contract geometry tensor {key!r} differs")+ extras = encoder_profile.get("extras", {})+ encoding = extras.get("geometry_encoding") if isinstance(extras, Mapping) else None+ if not isinstance(encoding, Mapping) or any(+ encoding.get(key) != value for key, value in H3_GEOMETRY_ENCODING_PROCESS.items()+ ):+ raise DataContractError("H3 encoder contract geometry encoding differs from the run")+ if geometry.wide_enabled:+ encoded = encoding.get("wide_fov_scale")+ if (+ not isinstance(encoded, (int, float))+ or abs(float(encoded) - float(geometry.wide_fov_scale)) > 1e-9+ ):+ # The ControlNet places wide rows by dividing positions by k; a render+ # made at another k would be trained at the wrong radius.+ raise DataContractError(+ f"H3 wide renders were encoded at fov_scale={encoded}, the run uses "+ f"{geometry.wide_fov_scale}"+ )+++def h3_silence_profile(*, pixel_frames: int = 158) -> dict[str, Any]:+ """Readable identity for the supported AudioVAE-encoded silence condition."""++ lengths = {153: 255, 158: 263, 170: 283}+ if pixel_frames not in lengths:+ raise DataContractError("unsupported H3 silence duration")+ return {+ "schema": "solarwm.minimax-h3-silence-profile.v1",+ "artifact": "official-audiovae-encoded-stereo-zero-waveform",+ "format": H3_PREENCODE_VERSION,+ "waveform": "real_stereo_zero_waveform",+ "audio_sampling_rate": 32_000,+ "audio_hop_length": 800,+ "normalization": "audio_vae.config.latents_mean_std",+ "posterior": "mode",+ "tensor": {+ "key": f"silence_{pixel_frames}f",+ "shape": [2, 32, lengths[pixel_frames]],+ "dtype": "bfloat16",+ },+ }+++def _encoder_profile(payload: Mapping[str, Any]) -> dict[str, Any]:+ schema = str(payload.get("schema") or "")+ if schema == "solarwm.encoder.v1":+ try:+ contract = EncoderContract(+ schema=schema,+ family=str(payload["family"]),+ format_version=str(payload["format_version"]),+ pixel_frames=int(payload["pixel_frames"]),+ latent_frames=int(payload["latent_frames"]),+ height=int(payload["height"]),+ width=int(payload["width"]),+ camera_convention=str(payload["camera_convention"]),+ tensors=tuple(TensorSpec(**value) for value in payload["tensors"]),+ extras=dict(payload.get("extras", {})),+ )+ except (KeyError, TypeError, ValueError) as exc:+ raise DataContractError("invalid H3 encoder contract") from exc+ if contract.family != "minimax_h3":+ raise DataContractError("H3 encoder contract selected a different family")+ return json.loads(json.dumps(contract.as_dict(), sort_keys=True))+ if schema == "solarwm_minimax_h3_encoder_contract_v1":+ if not isinstance(payload.get("silence"), Mapping):+ raise DataContractError("H3 provider encoder contract lacks its silence receipt")+ # This provider control contains transaction metadata only. The+ # backend's supported, readable encoder semantics are authoritative.+ return json.loads(+ json.dumps(+ h3_encoder_contract(encoder_identity="official-minimax-h3-codec").as_dict(),+ sort_keys=True,+ )+ )+ raise DataContractError(f"unsupported H3 encoder contract schema {schema!r}")+++def read_encoder_contract(path: str | Path) -> tuple[Mapping[str, Any], Mapping[str, Any]]:+ """Read provider metadata and return the normalized semantic encoder profile."""++ source = Path(path)+ try:+ payload = json.loads(source.read_bytes())+ except (OSError, UnicodeError, json.JSONDecodeError) as exc:+ raise DataContractError(f"cannot read H3 encoder contract {source}: {exc}") from exc+ if not isinstance(payload, Mapping):+ raise DataContractError("H3 encoder contract JSON must contain an object")+ return dict(payload), _encoder_profile(payload)+++@dataclass(frozen=True)+class H3VisualObservation:+ """One untyped visual input: ``[24,f,h,w]`` latents of any length and size.++ ``frames`` lists the target latent frames it is aligned with (a depth, pose or+ edge video, the source of an edit, a keyframe, a low-resolution preview);+ ``None`` means it is not a view of the clip (a subject, a scene, a style, a+ motion clip). ``degree`` is how far it is trusted, the condition level by default.+ """++ latents: Any+ frames: tuple[int, ...] | None = None+ degree: float = 0.999+ # Where an aligned observation lies on the target canvas: ``(x0, y0, x1, y1)`` in+ # [0, 1], one per frame or one for all; the whole canvas by default.+ boxes: tuple[tuple[float, float, float, float], ...] | None = None+++@dataclass(frozen=True)+class H3ArtifactBatch:+ sample_id: str+ start_frame: int+ plan_fingerprint: str+ target_latents: Any+ anchor_latents: Any+ prompt_embeds: Any+ text_token_tags: Any+ source_frame_indices: Any+ camera_viewmats: Any+ camera_K: Any+ source_fps: float | None+ validation_slot: int | None = None+ validation_noise_seed: int | None = None+ rollout_camera_viewmats: Any = None+ rollout_camera_K: Any = None+ dataset_source: str = ""+ # Geometry ControlNet renders, keyed by artifact tensor name; None otherwise.+ geometry_latents: Mapping[str, Any] | None = None+ # The caption encoded without the first frame. The joint prompt shows Qwen the first+ # frame, so a sample that drops the anchor must use these or the frame leaks back in.+ text_prompt_embeds: Any = None+ text_prompt_tags: Any = None+ # Untyped visual inputs beyond the first-frame anchor, in any number.+ visual_observations: tuple[H3VisualObservation, ...] | None = None+ # Known degree of every target row given by the data (an edit or inpainting mask,+ # a confidence map): ``[47*1008]`` in [0, 1], zero where the row is to be generated.+ known_degrees: Any = None+++@dataclass+class _H3WorkerCursor:+ identity: ReaderIdentity+ epoch: int+ cursor: int+ plans: tuple[SamplePlan, ...]+++class H3PlanMultiplexer:+ """Replay torch DataLoader's ordered round-robin iterable-worker schedule."""++ def __init__(+ self,+ rows: Sequence[IndexRow],+ sampling: SamplingConfig,+ topology: Topology,+ *,+ num_workers: int,+ state_schema: str,+ ) -> None:+ self.rows = tuple(rows)+ self.sampling = sampling+ self.topology = topology+ self.num_workers = int(num_workers)+ self.state_schema = str(state_schema)+ if self.num_workers < 1:+ raise DataContractError("H3 reader num_workers must be positive")+ self.workers: list[_H3WorkerCursor] = []+ for worker_id in range(self.num_workers):+ identity = ReaderIdentity.from_topology(+ topology,+ worker_id=worker_id,+ num_workers=self.num_workers,+ )+ self.workers.append(+ _H3WorkerCursor(+ identity=identity,+ epoch=0,+ cursor=0,+ plans=self._plans_for_epoch(identity, 0),+ )+ )+ self.next_worker = 0++ def _plans_for_epoch(+ self,+ identity: ReaderIdentity,+ epoch: int,+ ) -> tuple[SamplePlan, ...]:+ # Replay prior finite epochs from the worker-specific fixed seed. This+ # keeps compact epoch/cursor state exactly resumable.+ sampler = CanonicalSampler(self.rows, self.sampling, identity)+ plans: tuple[SamplePlan, ...] = ()+ for current in range(epoch + 1):+ plans = tuple(sampler.iter_epoch(current))+ if not plans:+ raise DataContractError(+ f"H3 logical reader worker {identity.worker_id} owns no samples"+ )+ return plans++ @property+ def current_plan_fingerprint(self) -> str:+ if self.num_workers == 1:+ return plan_fingerprint(self.workers[0].plans)+ digest = hashlib.blake2s(b"solarwm.minimax-h3-worker-plan.v1\n")+ for worker in self.workers:+ digest.update(+ f"{worker.identity.worker_id}\t{plan_fingerprint(worker.plans)}\n".encode()+ )+ return digest.hexdigest()++ def state_dict(self, *, encoder_profile: Mapping[str, Any]) -> dict[str, Any]:+ return {+ "schema": self.state_schema,+ "num_workers": self.num_workers,+ "next_worker": self.next_worker,+ "workers": [+ {+ "worker_id": worker.identity.worker_id,+ "epoch": worker.epoch,+ "cursor": worker.cursor,+ "plan_fingerprint": plan_fingerprint(worker.plans),+ }+ for worker in self.workers+ ],+ "plan_fingerprint": self.current_plan_fingerprint,+ "encoder_profile": json.loads(json.dumps(dict(encoder_profile), sort_keys=True)),+ }++ def load_state_dict(+ self,+ values: Mapping[str, Any],+ *,+ encoder_profile: Mapping[str, Any],+ ) -> None:+ if values.get("schema") != self.state_schema:+ raise DataContractError("unsupported H3 reader checkpoint schema")+ expected_encoder = json.loads(json.dumps(dict(encoder_profile), sort_keys=True))+ if values.get("encoder_profile") != expected_encoder:+ raise DataContractError("H3 reader encoder profile changed at resume")+ if int(values.get("num_workers", 0)) != self.num_workers:+ raise DataContractError("H3 reader worker count changed at resume")+ raw_workers = values.get("workers")+ if not isinstance(raw_workers, list) or len(raw_workers) != self.num_workers:+ raise DataContractError("H3 reader checkpoint worker states differ")+ restored: list[_H3WorkerCursor] = []+ for expected_id, raw in enumerate(raw_workers):+ if not isinstance(raw, Mapping) or int(raw.get("worker_id", -1)) != expected_id:+ raise DataContractError("H3 reader checkpoint worker order differs")+ epoch, cursor = int(raw["epoch"]), int(raw["cursor"])+ if epoch < 0:+ raise DataContractError("H3 reader resume epoch is negative")+ identity = self.workers[expected_id].identity+ plans = self._plans_for_epoch(identity, epoch)+ if raw.get("plan_fingerprint") != plan_fingerprint(plans):+ raise DataContractError("H3 reader worker plan changed at resume")+ if not 0 <= cursor <= len(plans):+ raise DataContractError("H3 reader resume cursor is outside its epoch")+ restored.append(_H3WorkerCursor(identity, epoch, cursor, plans))+ next_worker = int(values.get("next_worker", -1))+ if not 0 <= next_worker < self.num_workers:+ raise DataContractError("H3 reader next worker is outside its worker count")+ self.workers = restored+ self.next_worker = next_worker+ if values.get("plan_fingerprint") != self.current_plan_fingerprint:+ raise DataContractError("H3 reader aggregate plan changed at resume")++ def next_plan(self) -> SamplePlan:+ worker = self.workers[self.next_worker]+ if worker.cursor == len(worker.plans):+ worker.epoch += 1+ worker.cursor = 0+ worker.plans = self._plans_for_epoch(worker.identity, worker.epoch)+ plan = worker.plans[worker.cursor]+ worker.cursor += 1+ self.next_worker = (self.next_worker + 1) % self.num_workers+ return plan+++def validate_fixed_validation_rows(+ rows: tuple[IndexRow, ...],+ *,+ logical_world_size: int,+ sample_count: int | None = None,+ selection_seed: int = 0,+ noise_seed: int = 0,+ require_preencoded_identity: bool = False,+) -> tuple[IndexRow, ...]:+ """Select deterministic complete waves from a recipe test index."""++ logical_world = int(logical_world_size)+ slots = logical_world if sample_count is None else int(sample_count)+ if logical_world <= 0 or slots <= 0 or slots % logical_world:+ raise DataContractError(+ "H3 fixed validation sample count must form complete logical-DP waves: "+ f"sample_count={slots} logical_world_size={logical_world_size}"+ )+ if len(rows) < slots:+ raise DataContractError(+ "recipe test index has fewer rows than validation.sample_count: "+ f"rows={len(rows)} sample_count={slots}"+ )+ # Camera guards require inspecting the encoded artifact. Build one seeded+ # candidate order from the recipe test index; the stream skips rejected+ # candidates and assigns slots only to successfully materialized cases.+ selected = select_index_rows(rows, sample_count=len(rows), seed=selection_seed)+ if int(noise_seed) < 0 or int(noise_seed) + slots > 2**63:+ raise DataContractError("H3 validation noise seed is outside [0, 2**63)")+ normalized = []+ for candidate, row in enumerate(selected):+ values = dict(row.values)+ try:+ start = int(values["start_frame"])+ shard_size = int(values["shard_size"])+ except (KeyError, TypeError, ValueError) as exc:+ raise DataContractError(+ f"H3 recipe test row {row.sample_id!r} lacks start/shard identity"+ ) from exc+ if start < 0 or row.epoch_repeats != 1 or shard_size <= 0:+ raise DataContractError(f"H3 recipe test row {row.sample_id!r} is invalid")+ if require_preencoded_identity:+ required = {+ "h3_preencoded_member",+ "h3_provenance_member",+ "manifest_member",+ "shard_generation",+ }+ missing = sorted(name for name in required if values.get(name) in (None, ""))+ if missing:+ raise DataContractError(f"H3 recipe test row {row.sample_id!r} lacks {missing}")+ generation = str(values["shard_generation"]).strip()+ if not generation:+ raise DataContractError(+ f"H3 recipe test row {row.sample_id!r} lacks provider generation"+ )+ normalized.append(IndexRow.from_mapping(candidate, values))+ return tuple(normalized)+++def _member_for(row: Any) -> str:+ values = row.values+ members = values.get("members")+ if isinstance(members, Mapping):+ for suffix in ("tensors.safetensors", "h3.safetensors", "preencoded.safetensors"):+ if suffix in members:+ return str(members[suffix])+ candidates = [+ str(value) for key, value in members.items() if str(key).endswith(".safetensors")+ ]+ if len(candidates) == 1:+ return candidates[0]+ for key in ("h3_preencoded_member", "preencoded_member"):+ if values.get(key):+ return str(values[key])+ raise DataContractError(f"H3 index row {row.sample_id!r} lacks a tensor member")+++def invert_se3_torch(matrices: Any) -> Any:+ """Invert a stack of rigid transforms, on the Torch side of the house."""++ import torch++ rotation = matrices[..., :3, :3]+ rotation_inverse = rotation.transpose(-1, -2)+ output = torch.zeros_like(matrices)+ output[..., :3, :3] = rotation_inverse+ output[..., :3, 3] = -torch.einsum("...ij,...j->...i", rotation_inverse, matrices[..., :3, 3])+ output[..., 3, 3] = 1.0+ return output+++def views_from_c2w(c2w: Any) -> Any:+ """World-to-camera views in the model's frame: the clip's first camera is the origin.++ The Torch arithmetic order is preserved exactly; the NumPy equivalent differs by a few FP32+ ULPs, and that changes the fused-PRoPE loss.+ """++ import torch++ first_w2c = invert_se3_torch(c2w[:1])[0]+ relative_c2w = torch.matmul(first_w2c.unsqueeze(0), c2w)+ relative_c2w[0] = torch.eye(4, dtype=relative_c2w.dtype)+ return invert_se3_torch(relative_c2w).contiguous()+++def align_h3_camera(tensors: Mapping[str, Any], *, audit_latents: int = 47) -> tuple[Any, Any]:+ """Validate and convert encoded absolute cameras to H3 model rows."""+ import torch++ if audit_latents not in {45, 47}:+ raise DataContractError("H3 camera audit must cover 45 Stage1 or 47 physical latents")++ if "camera_token_viewmats" in tensors or "camera_token_K" in tensors:+ if "camera_token_viewmats" not in tensors or "camera_token_K" not in tensors:+ raise DataContractError("token camera artifact is incomplete")+ raw_views = tensors["camera_token_viewmats"]+ raw_intrinsics = tensors["camera_token_K"]+ if raw_views.dtype != torch.float32 or raw_intrinsics.dtype != torch.float32:+ raise DataContractError("token camera artifacts must be FP32")+ views = raw_views.contiguous()+ intrinsics = raw_intrinsics.contiguous()+ expected = 47 * 1008+ if tuple(views.shape) != (expected, 4, 4) or tuple(intrinsics.shape) != (+ expected,+ 3,+ 3,+ ):+ raise DataContractError("token camera artifact has the wrong packed shape")+ if not bool(torch.isfinite(views).all()) or not bool(torch.isfinite(intrinsics).all()):+ raise DataContractError("token camera artifact contains non-finite values")+ validate_normalized_intrinsics(intrinsics.cpu().numpy())+ if float(views[: audit_latents * 1008].abs().max().item()) > 20.0:+ raise H3CameraFilterError("token camera artifact exceeds the magnitude guard")+ return views, intrinsics+ if "camera_c2w" not in tensors or "camera_K" not in tensors:+ raise DataContractError("H3 tensors require camera_c2w and camera_K")+ if tensors["camera_c2w"].dtype != torch.float32 or tensors["camera_K"].dtype != torch.float32:+ raise DataContractError("H3 structured camera artifacts must be FP32")+ c2w = tensors["camera_c2w"].detach().cpu()+ K = tensors["camera_K"].detach().cpu()+ indices = torch.from_numpy(latent_aligned_pixel_indices(158)).long()+ if c2w.shape[0] == 158:+ c2w = c2w.index_select(0, indices)+ elif c2w.shape[0] != 47:+ raise DataContractError("camera_c2w must have 158 pixel or 47 latent rows")+ if K.shape == (3, 3):+ K = K.unsqueeze(0).expand(47, -1, -1).clone()+ elif K.shape[0] == 158:+ K = K.index_select(0, indices)+ elif K.shape[0] != 47:+ raise DataContractError("camera_K must have 158 pixel, 47 latent, or one static row")+ validate_absolute_c2w(c2w.numpy())+ validate_normalized_intrinsics(K.numpy())+ views = views_from_c2w(c2w)+ translation = torch.linalg.vector_norm(views[:audit_latents, :3, 3], dim=-1)+ if float(translation.max()) > 20.0 or float(views[:audit_latents].abs().max()) > 20.0:+ raise H3CameraFilterError("H3 camera exceeds the magnitude guards")+ return views, K.contiguous()+++class H3PreencodedStream:+ """Deterministic logical-DP stream over H3 WDS artifacts."""++ def __init__(+ self,+ *,+ root: str,+ index: str,+ topology: Topology,+ seed: int,+ encoder_contract_path: str,+ cache_dir: str | None = None,+ cache_max_gib: float = 256.0,+ shuffle_buffer: int = 4096,+ num_workers: int = 1,+ fixed_validation: bool = False,+ fixed_validation_sample_count: int | None = None,+ fixed_validation_selection_seed: int = 0,+ fixed_validation_noise_seed: int = 0,+ fixed_validation_sample_ids: Sequence[str] | None = None,+ camera_audit_latents: int = 47,+ geometry: Any = None,+ ) -> None:+ self.camera_audit_latents = int(camera_audit_latents)+ self.geometry = geometry+ rows = read_index(index)+ normalized: list[IndexRow] = []+ for row in rows:+ values = dict(row.values)+ indices = values.get("source_frame_indices")+ if values.get("num_frames") is None and isinstance(indices, (list, tuple)):+ try:+ values["num_frames"] = max(int(value) for value in indices) + 1+ except (TypeError, ValueError) as exc:+ raise DataContractError(+ f"H3 sample {row.sample_id!r} has invalid source_frame_indices"+ ) from exc+ if values.get("num_frames") is None and values.get("start_frame") is not None:+ # Finalized H3 indexes keep only the+ # frozen occurrence start and immutable member identities.+ # The encoded tensor carries the authoritative 158 contiguous+ # source indices, which are checked against this reconstructed+ # minimum span after materialization.+ try:+ start_frame = int(values["start_frame"])+ except (TypeError, ValueError) as exc:+ raise DataContractError(+ f"H3 sample {row.sample_id!r} has invalid start_frame"+ ) from exc+ if start_frame < 0:+ raise DataContractError(f"H3 sample {row.sample_id!r} has negative start_frame")+ values["num_frames"] = start_frame + 158+ metadata = values.get("metadata", {})+ if (+ values.get("fps") is None+ and isinstance(metadata, Mapping)+ and metadata.get("source_fps") is not None+ ):+ values["fps"] = metadata["source_fps"]+ normalized.append(IndexRow.from_mapping(row.ordinal, values))+ self.rows = tuple(normalized)+ self.topology = topology+ self.fixed_validation = bool(fixed_validation)+ self.fixed_validation_sample_count = (+ int(fixed_validation_sample_count) if self.fixed_validation else 0+ )+ self.fixed_validation_noise_seed = int(fixed_validation_noise_seed)+ self.fixed_validation_successes = 0+ if self.fixed_validation and int(num_workers) != 1:+ raise DataContractError("H3 fixed validation requires one deterministic worker")+ if self.fixed_validation:+ frozen_ids = (+ tuple(str(value) for value in fixed_validation_sample_ids)+ if fixed_validation_sample_ids is not None+ else None+ )+ if frozen_ids is None:+ self.rows = validate_fixed_validation_rows(+ self.rows,+ logical_world_size=topology.dp_world_size,+ sample_count=fixed_validation_sample_count,+ selection_seed=fixed_validation_selection_seed,+ noise_seed=fixed_validation_noise_seed,+ require_preencoded_identity=True,+ )+ else:+ expected = int(fixed_validation_sample_count or 0)+ if len(frozen_ids) != expected or expected % int(topology.dp_world_size):+ raise DataContractError(+ "H3 frozen validation sample IDs do not form complete waves"+ )+ by_id = {row.sample_id: row for row in self.rows}+ if len(by_id) != len(self.rows):+ raise DataContractError("H3 test index contains duplicate sample IDs")+ restored = []+ for slot, sample_id in enumerate(frozen_ids):+ try:+ row = by_id[sample_id]+ except KeyError as exc:+ raise DataContractError(+ f"H3 frozen validation sample {sample_id!r} left the test index"+ ) from exc+ validated = validate_fixed_validation_rows(+ (row,),+ logical_world_size=1,+ sample_count=1,+ noise_seed=0,+ require_preencoded_identity=True,+ )[0]+ restored.append(IndexRow.from_mapping(slot, dict(validated.values)))+ self.rows = tuple(restored)+ self.sampling = SamplingConfig(+ seed=int(seed),+ pixel_frames=158,+ random_start=False,+ fixed_start_from_index=True,+ shuffle_buffer=1 if self.fixed_validation else int(shuffle_buffer),+ partition_mode="global_occurrence" if self.fixed_validation else "node_shard",+ )+ self.encoder_contract, self.encoder_profile = read_encoder_contract(encoder_contract_path)+ require_geometry_encoder_profile(self.encoder_profile, self.geometry)+ resolver = resolver_from_config(root, cache_dir=cache_dir, max_gib=cache_max_gib)+ self.shards = TarShardReader(resolver, max_open=4)+ self.plan = H3PlanMultiplexer(+ self.rows,+ self.sampling,+ topology,+ num_workers=int(num_workers),+ state_schema="solarwm.minimax-h3-reader.v3",+ )+ self.identity = self.plan.workers[0].identity++ @property+ def epoch(self) -> int:+ return self.plan.workers[0].epoch++ @property+ def cursor(self) -> int:+ return self.plan.workers[0].cursor++ @property+ def _plans(self) -> tuple[SamplePlan, ...]:+ return self.plan.workers[0].plans++ @property+ def current_plan_fingerprint(self) -> str:+ return self.plan.current_plan_fingerprint++ def state_dict(self) -> dict[str, Any]:+ return self.plan.state_dict(+ encoder_profile=self.encoder_profile,+ )++ def load_state_dict(self, values: Mapping[str, Any]) -> None:+ self.plan.load_state_dict(+ values,+ encoder_profile=self.encoder_profile,+ )++ def close(self) -> None:+ self.shards.close()++ def __iter__(self) -> Iterator[H3ArtifactBatch]:+ while True:+ yield self.next()++ def next(self) -> H3ArtifactBatch:+ rejected = 0+ while True:+ # One DataLoader worker owns its pending output slot until it yields.+ current_worker = self.plan.next_worker+ try:+ batch = self._next_once()+ except H3CameraFilterError as exc:+ self.plan.next_worker = current_worker+ rejected += 1+ if self.fixed_validation and rejected >= len(self.rows):+ raise H3CameraFilterError(+ "H3 recipe test index has no camera-safe validation candidate"+ ) from exc+ continue+ except Exception:+ self.plan.next_worker = current_worker+ raise+ if not self.fixed_validation:+ return batch+ slot = self.fixed_validation_successes * int(self.topology.dp_world_size) + int(+ self.topology.dp_rank+ )+ if slot >= self.fixed_validation_sample_count:+ raise DataContractError("H3 fixed validation requested more cases than configured")+ self.fixed_validation_successes += 1+ return replace(+ batch,+ validation_slot=slot,+ validation_noise_seed=self.fixed_validation_noise_seed + slot,+ )++ def _next_once(self) -> H3ArtifactBatch:+ plan = self.plan.next_plan()+ row = self.rows[plan.row_ordinal]+ return self.read_artifact(row, plan)++ def read_artifact(self, row: IndexRow, plan: SamplePlan) -> H3ArtifactBatch:+ """Materialize a frozen occurrence without advancing the training reader."""++ row_profile = row.values.get("encoder_profile")+ if row_profile is not None and row_profile != self.encoder_profile:+ raise DataContractError(f"H3 sample {row.sample_id!r} encoder profile differs")+ payload = self.shards.read(row, _member_for(row))+ try:+ import torch+ from safetensors.torch import load++ tensors = load(payload)+ except (ImportError, ValueError) as exc:+ raise DataContractError("H3 preencoded member is not valid safetensors") from exc+ required = {+ "target_latents",+ "anchor_latents",+ "prompt_embeds",+ "text_token_tags",+ "source_frame_indices",+ }+ if not required <= set(tensors):+ raise DataContractError(f"H3 preencoded member lacks {sorted(required - set(tensors))}")+ target = tensors["target_latents"]+ anchor = tensors["anchor_latents"]+ prompt = tensors["prompt_embeds"]+ tags = tensors["text_token_tags"]+ indices = tensors["source_frame_indices"]+ if tuple(target.shape) != (24, 47, 48, 84) or tuple(anchor.shape) != (+ 24,+ 1,+ 48,+ 84,+ ):+ raise DataContractError("H3 VisualVAE tensor geometry differs")+ target_dtype = str(target.dtype).removeprefix("torch.")+ anchor_dtype = str(anchor.dtype).removeprefix("torch.")+ if target_dtype != "bfloat16" or anchor_dtype != "bfloat16":+ raise DataContractError("H3 VisualVAE artifacts must be BF16")+ if prompt.dtype != torch.bfloat16:+ raise DataContractError("H3 Qwen prompt artifacts must be BF16")+ if tags.dtype != torch.int64:+ raise DataContractError("H3 text token tags must be INT64")+ if indices.dtype != torch.int64:+ raise DataContractError("H3 source-frame indices must be INT64")+ if prompt.ndim != 2 or prompt.shape[-1] != 5120 or tags.shape != prompt.shape[:1]:+ raise DataContractError("H3 Qwen prompt artifacts differ")+ for name, tensor in (+ ("target_latents", target),+ ("anchor_latents", anchor),+ ("prompt_embeds", prompt),+ ):+ if not bool(torch.isfinite(tensor.float()).all()):+ raise DataContractError(f"H3 {name} contains non-finite values")+ if tuple(indices.shape) != (158,) or not np.all(np.diff(indices.numpy()) == 1):+ raise DataContractError("H3 source indices must be 158 exact contiguous frames")+ if tuple(int(value) for value in indices.tolist()) != tuple(plan.source_frame_indices):+ raise DataContractError("H3 tensor source indices differ from the sample plan")+ views, K = align_h3_camera(tensors, audit_latents=self.camera_audit_latents)+ geometry_latents = read_geometry_latents(tensors, getattr(self, "geometry", None))+ metadata = row.values.get("metadata", {})+ source_fps = row.values.get("fps")+ if source_fps is None and isinstance(metadata, Mapping):+ source_fps = metadata.get("source_fps")+ if source_fps is not None:+ source_fps = float(source_fps)+ if not np.isfinite(source_fps) or source_fps <= 0:+ raise DataContractError("H3 source FPS must be positive when present")+ return H3ArtifactBatch(+ sample_id=row.sample_id,+ start_frame=plan.start_frame,+ plan_fingerprint=self.current_plan_fingerprint,+ target_latents=target.contiguous(),+ anchor_latents=anchor.contiguous(),+ prompt_embeds=prompt.contiguous(),+ text_token_tags=tags.contiguous(),+ source_frame_indices=indices.contiguous(),+ camera_viewmats=views,+ camera_K=K,+ source_fps=source_fps,+ validation_slot=None,+ validation_noise_seed=None,+ dataset_source=row.sample_id.split("/")[0],+ geometry_latents=geometry_latents,+ )+++def read_geometry_latents(tensors: Mapping[str, Any], geometry: Any) -> dict[str, Any] | None:+ """Validate the renders a geometry profile reads; ignore them for other runs."""++ if geometry is None:+ return None+ import torch++ values = {}+ for key in geometry.tensor_keys():+ if key not in tensors:+ raise DataContractError(f"H3 preencoded member lacks geometry tensor {key!r}")+ value = tensors[key]+ if tuple(value.shape) != H3_GEOMETRY_LATENT_SHAPE or value.dtype != torch.bfloat16:+ raise DataContractError(f"H3 geometry tensor {key!r} must be BF16 [24,47,48,84]")+ if not bool(torch.isfinite(value.float()).all()):+ raise DataContractError(f"H3 geometry tensor {key!r} contains non-finite values")+ values[key] = value.contiguous()+ return values+++def load_silence_latents(+ path: str | Path, *, pixel_frames: int = 158+) -> tuple[Any, Mapping[str, Any]]:+ """Load the official AudioVAE-encoded 158f silence tensor ``[2,32,263]``."""++ source = Path(path)+ if not source.is_file():+ raise DataContractError(f"H3 silence artifact is missing: {source}")+ try:+ from safetensors.torch import load_file++ tensors = load_file(str(source), device="cpu")+ except (ImportError, ValueError) as exc:+ raise DataContractError("H3 silence artifact is not valid safetensors") from exc+ lengths = {153: 255, 158: 263, 170: 283}+ if pixel_frames not in lengths:+ raise DataContractError("H3 silence supports the encoded 153/158/170-frame profiles")+ key = f"silence_{pixel_frames}f"+ if key not in tensors:+ raise DataContractError(f"H3 silence artifact lacks the required {key!r} tensor")+ value = tensors[key].contiguous()+ if (+ tuple(value.shape) != (2, 32, lengths[pixel_frames])+ or str(value.dtype).removeprefix("torch.") != "bfloat16"+ ):+ raise DataContractError(f"H3 silence must be BF16 [2,32,{lengths[pixel_frames]}]")+ if not bool(value.isfinite().all()):+ raise DataContractError("H3 silence contains non-finite values")+ return value, h3_silence_profile(pixel_frames=pixel_frames)+++__all__ = [+ "H3_GEOMETRY_ENCODING_PROCESS",+ "H3_GEOMETRY_LATENT_SHAPE",+ "H3ArtifactBatch",+ "H3PlanMultiplexer",+ "H3PreencodedStream",+ "align_h3_camera",+ "h3_encoder_contract",+ "h3_geometry_encoding_profile",+ "h3_silence_profile",+ "load_silence_latents",+ "read_encoder_contract",+ "read_geometry_latents",+ "require_geometry_encoder_profile",+ "validate_fixed_validation_rows",+]diff --git a/src/solarwm/backends/minimax_h3/backend.py b/src/solarwm/backends/minimax_h3/backend.pynew file mode 100644index 0000000..3ee9867--- /dev/null+++ b/src/solarwm/backends/minimax_h3/backend.py@@ -0,0 +1,50 @@+"""Lazy MiniMax-H3 backend boundary."""++from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++from solarwm.errors import BackendContractError++from .config import validate_h3_config+++@dataclass(frozen=True)+class MiniMaxH3Backend:+ """Public plugin; heavy H3 dependencies load only after an action starts."""++ family: str = "minimax_h3"++ def validate_config(self, config: Mapping[str, Any]) -> None:+ validate_h3_config(config)++ def train(self, config: Mapping[str, Any]) -> int:+ self.validate_config(config)+ from .runtime import run_training++ return int(run_training(config))++ def infer(self, config: Mapping[str, Any]) -> int:+ self.validate_config(config)+ from .runtime import run_inference++ return int(run_inference(config))++ def preencode(self, config: Mapping[str, Any]) -> int:+ self.validate_config(config)+ from .preencode_runner import run_preencode++ return int(run_preencode(config))+++def create_backend(*, family: str = "minimax_h3") -> MiniMaxH3Backend:+ """Factory used by the global lazy backend registry."""++ if family != "minimax_h3":+ raise BackendContractError(f"MiniMax-H3 backend cannot serve family {family!r}")+ return MiniMaxH3Backend(family=family)+++__all__ = ["MiniMaxH3Backend", "create_backend"]diff --git a/src/solarwm/backends/minimax_h3/camera.py b/src/solarwm/backends/minimax_h3/camera.pynew file mode 100644index 0000000..5120b8d--- /dev/null+++ b/src/solarwm/backends/minimax_h3/camera.py@@ -0,0 +1,327 @@+"""MiniMax-H3 camera normalization and fused-PRoPE suffix arithmetic.++The native MM-RoPE transform is owned by the transformer and must run first.+This module only transforms head dimensions ``[96:128)`` and preserves the+native prefix ``[0:96)`` exactly.+"""++from __future__ import annotations++from collections.abc import Callable+from dataclasses import dataclass++import numpy as np++H3_ATTENTION_HEAD_DIM = 128+H3_NATIVE_MM_ROPE_DIM = 96+H3_CAMERA_PROPE_DIM_START = 96+H3_CAMERA_PROPE_DIM_END = 128+H3_CAMERA_TRANSLATION_TRANSFORM = "logd4"++# H3 uses the shared normalized Wan intrinsics.+WAN_FIXED_FX = 969.6969696969696 / (960.0 * 2.0)+WAN_FIXED_FY = 969.6969696969696 / (540.0 * 2.0)+WAN_FIXED_CX = 0.5+WAN_FIXED_CY = 0.5+++@dataclass(frozen=True)+class H3CameraSuffixMatrices:+ """Token-aligned projective transforms for Q, K/V, and attention output."""++ query: np.ndarray+ key_value: np.ndarray+ output: np.ndarray+++def h3_fused_prope_contract(intrinsics_mode: str = "wan_fixed") -> dict[str, object]:+ """Return the checkpoint-visible H3 fused-PRoPE fingerprint.++ ``intrinsics_mode`` says whether the runtime pins every clip to one field of view or reads the+ focal length off the sample, which is what a clip with a changing focal length needs.+ """++ return {+ "order": "native_mm_rope_then_camera_prope",+ "native_mm_rope_head_slice": [0, H3_NATIVE_MM_ROPE_DIM],+ "camera_prope_head_slice": [+ H3_CAMERA_PROPE_DIM_START,+ H3_CAMERA_PROPE_DIM_END,+ ],+ "relative_translation_transform": H3_CAMERA_TRANSLATION_TRANSFORM,+ "relative_pose": "first_frame_relative_w2c",+ "input_pose": "absolute_c2w",+ "input_intrinsics": "normalized_K",+ "runtime_intrinsics": (+ "wan_fixed_focal_only" if intrinsics_mode == "wan_fixed" else "sample_focal_only"+ ),+ }+++def _floating_array(value: object, *, name: str) -> np.ndarray:+ result = np.asarray(value)+ if not np.issubdtype(result.dtype, np.floating):+ raise TypeError(f"{name} must use a floating dtype, got {result.dtype}")+ if not np.all(np.isfinite(result)):+ raise ValueError(f"{name} contains non-finite values")+ return result+++def validate_absolute_c2w(c2w: object, *, atol: float = 1e-4) -> np.ndarray:+ """Validate authoritative absolute camera-to-world matrices."""++ poses = _floating_array(c2w, name="c2w")+ if poses.ndim not in (3, 4) or poses.shape[-2:] != (4, 4):+ raise ValueError(f"c2w must be [T,4,4] or [B,T,4,4], got {poses.shape}")+ expected_bottom = np.zeros_like(poses[..., 3, :])+ expected_bottom[..., 3] = 1+ if not np.allclose(poses[..., 3, :], expected_bottom, atol=atol, rtol=0):+ raise ValueError("c2w homogeneous bottom row must be [0,0,0,1]")+ rotation = poses[..., :3, :3]+ identity = np.eye(3, dtype=rotation.dtype)+ gram = np.swapaxes(rotation, -1, -2) @ rotation+ if not np.allclose(gram, identity, atol=atol, rtol=0):+ raise ValueError("c2w rotations must be orthonormal")+ determinant = np.linalg.det(rotation)+ if not np.allclose(determinant, 1.0, atol=atol, rtol=0):+ raise ValueError("c2w rotations must be proper rotations with determinant +1")+ return poses+++def validate_normalized_intrinsics(Ks: object, *, atol: float = 1e-5) -> np.ndarray:+ """Validate normalized, no-skew pinhole intrinsics."""++ intrinsics = _floating_array(Ks, name="K")+ if intrinsics.ndim not in (3, 4) or intrinsics.shape[-2:] != (3, 3):+ raise ValueError(f"K must be [T,3,3] or [B,T,3,3], got {intrinsics.shape}")+ focal = intrinsics[..., (0, 1), (0, 1)]+ if np.any(focal <= 0) or np.any(focal > 4):+ raise ValueError("normalized K requires 0 < fx,fy <= 4")+ if not (+ np.allclose(intrinsics[..., 0, 1], 0, atol=atol, rtol=0)+ and np.allclose(intrinsics[..., 1, 0], 0, atol=atol, rtol=0)+ and np.allclose(intrinsics[..., 2, :2], 0, atol=atol, rtol=0)+ and np.allclose(intrinsics[..., 2, 2], 1, atol=atol, rtol=0)+ ):+ raise ValueError("normalized K must be no-skew with bottom row [0,0,1]")+ return intrinsics+++def invert_se3(transforms: object) -> np.ndarray:+ """Invert rigid transforms without a generic matrix inverse."""++ values = _floating_array(transforms, name="transforms")+ if values.shape[-2:] != (4, 4):+ raise ValueError(f"transforms must end in [4,4], got {values.shape}")+ output = np.zeros_like(values)+ rotation_inv = np.swapaxes(values[..., :3, :3], -1, -2)+ output[..., :3, :3] = rotation_inv+ output[..., :3, 3] = -np.einsum("...ij,...j->...i", rotation_inv, values[..., :3, 3])+ output[..., 3, 3] = 1+ return output+++def first_frame_relative_w2c(c2w: object) -> np.ndarray:+ """Convert absolute C2W to first-frame-relative W2C.++ For each frame this computes ``inverse(c2w[t]) @ c2w[0]``. The first+ result is identity, and there is no axis flip.+ """++ poses = validate_absolute_c2w(c2w)+ if poses.ndim == 3:+ result = invert_se3(poses) @ poses[0]+ result[0] = np.eye(4, dtype=result.dtype)+ return result+ result = invert_se3(poses) @ poses[:, :1]+ result[:, 0] = np.eye(4, dtype=result.dtype)+ return result+++def logd4_relative_viewmats(viewmats: object) -> np.ndarray:+ """Apply zero-safe ``t*log1p(||t||)/(4*||t||)`` in FP32 or FP64."""++ values = _floating_array(viewmats, name="viewmats")+ if values.shape[-2:] != (4, 4):+ raise ValueError(f"viewmats must end in [4,4], got {values.shape}")+ output = values.copy()+ compute_dtype = np.float64 if values.dtype == np.float64 else np.float32+ translation = values[..., :3, 3].astype(compute_dtype, copy=False)+ radius = np.linalg.norm(translation, axis=-1, keepdims=True)+ scale = np.zeros_like(radius)+ np.divide(np.log1p(radius), 4.0 * radius, out=scale, where=radius != 0)+ compressed = translation * scale+ output[..., :3, 3] = compressed.astype(values.dtype, copy=False)+ return output+++def wan_fixed_intrinsics_like(Ks: object) -> np.ndarray:+ """Return fixed normalized K with the same shape and dtype."""++ reference = _floating_array(Ks, name="K")+ if reference.shape[-2:] != (3, 3):+ raise ValueError(f"K must end in [3,3], got {reference.shape}")+ output = np.zeros_like(reference)+ output[..., 0, 0] = WAN_FIXED_FX+ output[..., 1, 1] = WAN_FIXED_FY+ output[..., 0, 2] = WAN_FIXED_CX+ output[..., 1, 2] = WAN_FIXED_CY+ output[..., 2, 2] = 1+ return output+++def _lift_K(Ks: np.ndarray) -> np.ndarray:+ output = np.zeros((*Ks.shape[:-2], 4, 4), dtype=Ks.dtype)+ output[..., :3, :3] = Ks+ output[..., 3, 3] = 1+ return output+++def _invert_K(Ks: np.ndarray) -> np.ndarray:+ output = np.zeros_like(Ks)+ output[..., 0, 0] = 1.0 / Ks[..., 0, 0]+ output[..., 1, 1] = 1.0 / Ks[..., 1, 1]+ output[..., 0, 2] = -Ks[..., 0, 2] / Ks[..., 0, 0]+ output[..., 1, 2] = -Ks[..., 1, 2] / Ks[..., 1, 1]+ output[..., 2, 2] = 1+ return output+++def prepare_camera_suffix_matrices(+ viewmats: object,+ Ks: object,+) -> H3CameraSuffixMatrices:+ """Prepare H3 logd4/fixed-K projective matrices."""++ views = logd4_relative_viewmats(viewmats)+ intrinsics = validate_normalized_intrinsics(Ks)+ if views.ndim != 4 or views.shape[-2:] != (4, 4):+ raise ValueError(f"viewmats must be token-aligned [B,S,4,4], got {views.shape}")+ if intrinsics.shape != (*views.shape[:2], 3, 3):+ raise ValueError(+ f"K must be token-aligned [B,S,3,3], got {intrinsics.shape} for viewmats {views.shape}"+ )+ fixed = wan_fixed_intrinsics_like(intrinsics)+ # H3 consumes only fx/fy; cx/cy are cleared here.+ focal_only = np.zeros_like(fixed)+ focal_only[..., 0, 0] = fixed[..., 0, 0]+ focal_only[..., 1, 1] = fixed[..., 1, 1]+ focal_only[..., 2, 2] = 1+ projection = _lift_K(focal_only) @ views+ projection_inv = invert_se3(views) @ _lift_K(_invert_K(focal_only))+ return H3CameraSuffixMatrices(+ query=np.swapaxes(projection, -1, -2).astype(views.dtype, copy=False),+ key_value=projection_inv.astype(views.dtype, copy=False),+ output=projection.astype(views.dtype, copy=False),+ )+++def _apply_token_projection(features: np.ndarray, matrix: np.ndarray) -> np.ndarray:+ batch, heads, sequence, feature_dim = features.shape+ projective_dim = int(matrix.shape[-1])+ if feature_dim % projective_dim:+ raise ValueError(f"camera PRoPE suffix {feature_dim} is not divisible by {projective_dim}")+ if matrix.shape != (batch, sequence, projective_dim, projective_dim):+ raise ValueError("camera PRoPE projection matrix must be token-aligned")+ tiled = features.reshape(batch, heads, sequence, feature_dim // projective_dim, projective_dim)+ output = np.einsum("bsij,bhspj->bhspi", matrix, tiled)+ return output.reshape(features.shape)+++def _apply_camera_suffix(features: object, matrix: np.ndarray) -> np.ndarray:+ values = _floating_array(features, name="attention features")+ if values.ndim != 4 or values.shape[-1] != H3_ATTENTION_HEAD_DIM:+ raise ValueError("H3 fused PRoPE requires [B,H,S,128] attention features")+ native = values[..., :H3_CAMERA_PROPE_DIM_START]+ camera = values[..., H3_CAMERA_PROPE_DIM_START:H3_CAMERA_PROPE_DIM_END]+ return np.concatenate((native, _apply_token_projection(camera, matrix)), axis=-1)+++def prope_qkv(+ q: object,+ k: object,+ v: object,+ *,+ viewmats: object,+ Ks: object,+) -> tuple[np.ndarray, np.ndarray, np.ndarray, Callable[[object], np.ndarray]]:+ """Apply camera PRoPE to suffix ``[96:128)`` after native MM-RoPE."""++ query = _floating_array(q, name="q")+ key = _floating_array(k, name="k")+ value = _floating_array(v, name="v")+ if query.ndim != 4 or query.shape != key.shape or query.shape != value.shape:+ raise ValueError("H3 fused PRoPE requires equal [B,H,S,D] q/k/v tensors")+ if query.shape[-1] != H3_ATTENTION_HEAD_DIM:+ raise ValueError(f"H3 fused PRoPE requires attention head_dim=128, got {query.shape[-1]}")+ matrices = prepare_camera_suffix_matrices(viewmats, Ks)+ if matrices.query.shape[:2] != (query.shape[0], query.shape[2]):+ raise ValueError("viewmats/K must align to the q/k/v batch and sequence")++ def apply_output(features: object) -> np.ndarray:+ return _apply_camera_suffix(features, matrices.output)++ return (+ _apply_camera_suffix(query, matrices.query),+ _apply_camera_suffix(key, matrices.key_value),+ _apply_camera_suffix(value, matrices.key_value),+ apply_output,+ )+++def select_camera_rows(+ relative_viewmats: object,+ intrinsics: object,+ camera_frame_ids: object,+) -> tuple[np.ndarray, np.ndarray]:+ """Expand per-latent cameras to explicit anchor/target VisualVAE rows."""++ views = _floating_array(relative_viewmats, name="relative_viewmats")+ Ks = _floating_array(intrinsics, name="K")+ frame_ids = np.asarray(camera_frame_ids, dtype=np.int64)+ if views.ndim == 3:+ views = views[None]+ if Ks.ndim == 3:+ Ks = Ks[None]+ if views.ndim != 4 or views.shape[-2:] != (4, 4):+ raise ValueError("relative_viewmats must be [T,4,4] or [B,T,4,4]")+ if Ks.shape != (*views.shape[:2], 3, 3):+ raise ValueError("K and relative_viewmats must have identical batch/time axes")+ if frame_ids.ndim != 1 or np.any(frame_ids < 0) or np.any(frame_ids >= views.shape[1]):+ raise ValueError("camera_frame_ids are outside the latent camera track")+ return views[:, frame_ids], Ks[:, frame_ids]+++__all__ = [+ "H3_ATTENTION_HEAD_DIM",+ "H3_CAMERA_PROPE_DIM_END",+ "H3_CAMERA_PROPE_DIM_START",+ "H3_CAMERA_TRANSLATION_TRANSFORM",+ "H3_NATIVE_MM_ROPE_DIM",+ "WAN_FIXED_CX",+ "WAN_FIXED_CY",+ "WAN_FIXED_FX",+ "WAN_FIXED_FY",+ "H3CameraSuffixMatrices",+ "first_frame_relative_w2c",+ "h3_fused_prope_contract",+ "invert_se3",+ "logd4_relative_viewmats",+ "prepare_camera_suffix_matrices",+ "prope_qkv",+ "select_camera_rows",+ "validate_absolute_c2w",+ "validate_normalized_intrinsics",+ "wan_fixed_intrinsics_like",+]+++def fixed_intrinsics_like(reference: object) -> object:+ """Build the released camera intrinsics on a tensor's device and dtype."""+ output = reference.new_zeros(reference.shape)+ output[..., 0, 0] = WAN_FIXED_FX+ output[..., 1, 1] = WAN_FIXED_FY+ output[..., 0, 2] = WAN_FIXED_CX+ output[..., 1, 2] = WAN_FIXED_CY+ output[..., 2, 2] = 1.0+ return outputdiff --git a/src/solarwm/backends/minimax_h3/codec.py b/src/solarwm/backends/minimax_h3/codec.pynew file mode 100644index 0000000..926d236--- /dev/null+++ b/src/solarwm/backends/minimax_h3/codec.py@@ -0,0 +1,384 @@+"""Dependency-light contracts for MiniMax-H3 raw and encoded model inputs.++The concrete lazy runtime lives in :mod:`official_codec`; keeping its protocol+and geometry checks here lets CPU-only installations audit the data boundary.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Protocol, runtime_checkable++import numpy as np++from .camera import (+ first_frame_relative_w2c,+ validate_absolute_c2w,+ validate_normalized_intrinsics,+)+from .geometry import (+ STABLE_STAGE0P5_GEOMETRY,+ audio_latents_for_video,+ latent_aligned_pixel_indices,+)+from .layout import H3PackedLayout, build_stage0p5_layout++H3_PREENCODE_VERSION = "h3.158f.v1"+H3_TEXT_HIDDEN_DIM = 5120+H3_AUDIO_HIDDEN_DIM = 32+++@dataclass(frozen=True)+class H3RawSample:+ """One exact contiguous source sample before model-specific encoding."""++ sample_id: str+ frames: object+ caption: str+ source_frame_indices: object+ camera_c2w: object+ camera_intrinsics: object+ height: int = STABLE_STAGE0P5_GEOMETRY.height+ width: int = STABLE_STAGE0P5_GEOMETRY.width+ source_fps: float | None = None+++@dataclass(frozen=True)+class H3EncodedPayload:+ """Artifacts the official H3 preencoder must produce."""++ target_latents: object+ anchor_latent: object+ prompt_embeddings: object+ text_token_tags: object+ audio_latents: object+++@dataclass(frozen=True)+class H3PreencodedSample:+ """Per-sample ``h3.158f.v1`` artifact; silence is stored separately."""++ sample_id: str+ encoder_identity: str+ target_latents: object+ anchor_latent: object+ prompt_embeddings: object+ text_token_tags: object+ source_frame_indices: object+ camera_c2w: object+ camera_intrinsics: object+ preencode_version: str = H3_PREENCODE_VERSION+ source_fps: float | None = None+++@dataclass(frozen=True)+class H3ModelInputs:+ """Validated single-sample inputs ready for the H3 transformer adapter."""++ sample_id: str+ codec_identity: str+ preencode_version: str+ target_latents: object+ anchor_latent: object+ prompt_embeddings: object+ text_token_tags: object+ audio_latents: object+ layout: H3PackedLayout+ camera_viewmats: np.ndarray+ camera_intrinsics: np.ndarray+ source_frame_indices: np.ndarray+ source_fps: float | None+++@runtime_checkable+class H3Codec(Protocol):+ """Required codec implementation used by offline preencoding.++ The methods must invoke the matching official H3 encoders. Replacing the+ prompt, image anchor, or encoded-silence branches with zeros is not a valid+ implementation of this protocol.+ """++ identity: str++ def encode_target_video(self, sample: H3RawSample) -> object: ...++ def encode_visual_anchor(self, sample: H3RawSample) -> object: ...++ def encode_joint_prompt(self, sample: H3RawSample) -> tuple[object, object]: ...++ def encode_silence_audio(self, sample: H3RawSample) -> object: ...+++def _shape(value: object, *, name: str) -> tuple[int, ...]:+ shape = getattr(value, "shape", None)+ if shape is None:+ raise TypeError(f"{name} must expose a tensor-like shape")+ try:+ return tuple(int(size) for size in shape)+ except (TypeError, ValueError) as exc:+ raise TypeError(f"{name} has an invalid shape {shape!r}") from exc+++def _dtype_name(value: object, *, name: str) -> str:+ dtype = getattr(value, "dtype", None)+ if dtype is None:+ raise TypeError(f"{name} must expose a tensor-like dtype")+ result = str(dtype).lower().replace("torch.", "").replace("numpy.", "")+ return result+++def _require_shape(value: object, expected: tuple[int, ...], *, name: str) -> None:+ observed = _shape(value, name=name)+ if observed != expected:+ raise ValueError(f"{name} shape {observed} != {expected}")+++def _require_dtype(value: object, expected: str, *, name: str) -> None:+ observed = _dtype_name(value, name=name)+ aliases = {+ "bfloat16": {"bfloat16", "bf16"},+ "float32": {"float32", "float"},+ "int64": {"int64", "long"},+ }+ if observed not in aliases[expected]:+ raise TypeError(f"{name} dtype {observed!r} is not {expected}")+++def validate_raw_sample(sample: H3RawSample) -> None:+ """Validate exact 158-frame selection and authoritative camera tracks."""++ if not isinstance(sample, H3RawSample):+ raise TypeError("sample must be H3RawSample")+ if not sample.sample_id.strip():+ raise ValueError("sample_id must be non-empty")+ if not isinstance(sample.caption, str):+ raise TypeError("caption must be a string")+ profile = STABLE_STAGE0P5_GEOMETRY+ if (sample.height, sample.width) != (profile.height, profile.width):+ raise ValueError(+ f"raw sample canvas must be {(profile.height, profile.width)}, "+ f"got {(sample.height, sample.width)}"+ )+ try:+ frame_count = len(sample.frames) # type: ignore[arg-type]+ except TypeError as exc:+ raise TypeError("frames must expose its temporal length") from exc+ if frame_count != profile.pixel_frames:+ raise ValueError(+ f"raw sample must contain {profile.pixel_frames} frames, got {frame_count}"+ )+ source_indices = np.asarray(sample.source_frame_indices)+ if source_indices.dtype != np.int64 or source_indices.shape != (profile.pixel_frames,):+ raise TypeError(f"source_frame_indices must be int64 [{profile.pixel_frames}]")+ if not np.all(np.diff(source_indices) == 1):+ raise ValueError("source_frame_indices must be exact contiguous frames")+ c2w = validate_absolute_c2w(sample.camera_c2w)+ K = validate_normalized_intrinsics(sample.camera_intrinsics)+ if c2w.shape != (profile.pixel_frames, 4, 4):+ raise ValueError(f"camera_c2w must be [{profile.pixel_frames},4,4]")+ if K.shape != (profile.pixel_frames, 3, 3):+ raise ValueError(f"camera_intrinsics must be [{profile.pixel_frames},3,3]")+ if c2w.dtype != np.float32 or K.dtype != np.float32:+ raise TypeError("camera_c2w and camera_intrinsics must be float32 artifacts")+ if sample.source_fps is not None and (+ not np.isfinite(sample.source_fps) or sample.source_fps <= 0+ ):+ raise ValueError("source_fps, when present for audit, must be positive")+++def _validate_video_text_tensors(+ target_latents: object,+ anchor_latent: object,+ prompt_embeddings: object,+ text_token_tags: object,+) -> None:+ profile = STABLE_STAGE0P5_GEOMETRY+ _require_shape(+ target_latents,+ (+ profile.latent_channels,+ profile.encoded_latents,+ profile.latent_height,+ profile.latent_width,+ ),+ name="target_latents",+ )+ _require_shape(+ anchor_latent,+ (+ profile.latent_channels,+ 1,+ profile.latent_height,+ profile.latent_width,+ ),+ name="anchor_latent",+ )+ prompt_shape = _shape(prompt_embeddings, name="prompt_embeddings")+ if len(prompt_shape) != 2 or prompt_shape[0] < 1 or prompt_shape[1] != H3_TEXT_HIDDEN_DIM:+ raise ValueError("prompt_embeddings must have shape [L,5120] with L >= 1")+ _require_shape(+ text_token_tags,+ (prompt_shape[0],),+ name="text_token_tags",+ )+ for name, value in (+ ("target_latents", target_latents),+ ("anchor_latent", anchor_latent),+ ("prompt_embeddings", prompt_embeddings),+ ):+ _require_dtype(value, "bfloat16", name=name)+ _require_dtype(text_token_tags, "int64", name="text_token_tags")+++def validate_encoded_payload(payload: H3EncodedPayload) -> None:+ """Validate all tensor outputs of an injected preencoding codec."""++ if not isinstance(payload, H3EncodedPayload):+ raise TypeError("codec output must be H3EncodedPayload")+ _validate_video_text_tensors(+ payload.target_latents,+ payload.anchor_latent,+ payload.prompt_embeddings,+ payload.text_token_tags,+ )+ validate_silence_latents(payload.audio_latents)+++def validate_preencoded_sample(sample: H3PreencodedSample) -> None:+ """Validate one immutable per-sample ``h3.158f.v1`` artifact."""++ if not isinstance(sample, H3PreencodedSample):+ raise TypeError("sample must be H3PreencodedSample")+ if not sample.sample_id.strip() or not sample.encoder_identity.strip():+ raise ValueError("sample_id and encoder_identity must be non-empty")+ if sample.preencode_version != H3_PREENCODE_VERSION:+ raise ValueError(+ f"preencode_version must be {H3_PREENCODE_VERSION!r}, got {sample.preencode_version!r}"+ )+ profile = STABLE_STAGE0P5_GEOMETRY+ _validate_video_text_tensors(+ sample.target_latents,+ sample.anchor_latent,+ sample.prompt_embeddings,+ sample.text_token_tags,+ )+ source_indices = np.asarray(sample.source_frame_indices)+ if source_indices.dtype != np.int64 or source_indices.shape != (profile.pixel_frames,):+ raise TypeError(f"source_frame_indices must be int64 [{profile.pixel_frames}]")+ if not np.all(np.diff(source_indices) == 1):+ raise ValueError("source_frame_indices must be exact contiguous frames")+ c2w = validate_absolute_c2w(sample.camera_c2w)+ intrinsics = validate_normalized_intrinsics(sample.camera_intrinsics)+ if c2w.shape != (profile.pixel_frames, 4, 4):+ raise ValueError(f"camera_c2w must be [{profile.pixel_frames},4,4]")+ if intrinsics.shape != (profile.pixel_frames, 3, 3):+ raise ValueError(f"camera_intrinsics must be [{profile.pixel_frames},3,3]")+ if c2w.dtype != np.float32 or intrinsics.dtype != np.float32:+ raise TypeError("camera_c2w and camera_intrinsics must be float32 artifacts")+ if sample.source_fps is not None and (+ not np.isfinite(sample.source_fps) or sample.source_fps <= 0+ ):+ raise ValueError("source_fps, when present for audit, must be positive")+++def validate_silence_latents(silence_latents: object) -> None:+ """Validate the separately versioned official encoded-silence artifact."""++ profile = STABLE_STAGE0P5_GEOMETRY+ _require_shape(+ silence_latents,+ (2, H3_AUDIO_HIDDEN_DIM, audio_latents_for_video(profile.pixel_frames)),+ name="audio_latents",+ )+ _require_dtype(silence_latents, "bfloat16", name="audio_latents")+++def assemble_model_inputs(+ sample: H3PreencodedSample,+ silence_latents: object,+) -> H3ModelInputs:+ """Convert validated sample/global artifacts into packed model inputs."""++ validate_preencoded_sample(sample)+ validate_silence_latents(silence_latents)+ profile = STABLE_STAGE0P5_GEOMETRY+ alignment = latent_aligned_pixel_indices(profile.pixel_frames)+ c2w = np.asarray(sample.camera_c2w, dtype=np.float32)[alignment]+ latent_K = np.asarray(sample.camera_intrinsics, dtype=np.float32)[alignment]+ relative_w2c = first_frame_relative_w2c(c2w)+ tags_array = np.asarray(sample.text_token_tags, dtype=np.int64)+ layout = build_stage0p5_layout(+ tags_array,+ profile.encoded_latents,+ profile.latent_height,+ profile.latent_width,+ audio_latents_for_video(profile.pixel_frames),+ )+ return H3ModelInputs(+ sample_id=sample.sample_id,+ codec_identity=sample.encoder_identity,+ preencode_version=sample.preencode_version,+ target_latents=sample.target_latents,+ anchor_latent=sample.anchor_latent,+ prompt_embeddings=sample.prompt_embeddings,+ text_token_tags=sample.text_token_tags,+ audio_latents=silence_latents,+ layout=layout,+ camera_viewmats=relative_w2c[layout.camera_frame_ids],+ camera_intrinsics=latent_K[layout.camera_frame_ids],+ source_frame_indices=np.asarray(sample.source_frame_indices, dtype=np.int64),+ source_fps=sample.source_fps,+ )+++def encode_raw_sample(sample: H3RawSample, codec: H3Codec) -> H3ModelInputs:+ """Run an official-compatible codec and assemble validated model inputs."""++ validate_raw_sample(sample)+ if not isinstance(codec, H3Codec):+ raise TypeError("codec does not implement the complete H3Codec protocol")+ identity = str(codec.identity).strip()+ if not identity:+ raise ValueError("codec.identity must be a non-empty provenance identifier")+ prompt, tags = codec.encode_joint_prompt(sample)+ payload = H3EncodedPayload(+ target_latents=codec.encode_target_video(sample),+ anchor_latent=codec.encode_visual_anchor(sample),+ prompt_embeddings=prompt,+ text_token_tags=tags,+ audio_latents=codec.encode_silence_audio(sample),+ )+ validate_encoded_payload(payload)++ preencoded = H3PreencodedSample(+ sample_id=sample.sample_id,+ encoder_identity=identity,+ target_latents=payload.target_latents,+ anchor_latent=payload.anchor_latent,+ prompt_embeddings=payload.prompt_embeddings,+ text_token_tags=payload.text_token_tags,+ source_frame_indices=sample.source_frame_indices,+ camera_c2w=sample.camera_c2w,+ camera_intrinsics=sample.camera_intrinsics,+ source_fps=sample.source_fps,+ )+ return assemble_model_inputs(preencoded, payload.audio_latents)+++__all__ = [+ "H3_AUDIO_HIDDEN_DIM",+ "H3_PREENCODE_VERSION",+ "H3_TEXT_HIDDEN_DIM",+ "H3Codec",+ "H3EncodedPayload",+ "H3ModelInputs",+ "H3PreencodedSample",+ "H3RawSample",+ "assemble_model_inputs",+ "encode_raw_sample",+ "validate_encoded_payload",+ "validate_preencoded_sample",+ "validate_raw_sample",+ "validate_silence_latents",+]diff --git a/src/solarwm/backends/minimax_h3/config.py b/src/solarwm/backends/minimax_h3/config.pynew file mode 100644index 0000000..ae97bbc--- /dev/null+++ b/src/solarwm/backends/minimax_h3/config.py@@ -0,0 +1,775 @@+"""Strict validation for the released H3 Stage0.5, AnyFlow and SGF profiles."""++from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++from solarwm.data import resolve_index_path+from solarwm.errors import ConfigurationError++from .camera import h3_fused_prope_contract+from .codec import H3_PREENCODE_VERSION+from .controlnet_contract import (+ geometry_profile_from_model_config,+ geometry_renders_from_preencode_config,+ validate_geometry_parallelism,+)+from .geometry import validate_stage0p5_geometry+++@dataclass(frozen=True)+class H3RunContract:+ """Resolved fields that identify the supported H3 configuration."""++ action: str+ stage: str+ pixel_frames: int+ encoded_latents: int+ sequence_parallel_size: int+ adapter_rank: int+ camera_translation_transform: str+ data_input_mode: str+++def _mapping(config: Mapping[str, Any], key: str) -> Mapping[str, Any]:+ value = config.get(key)+ if not isinstance(value, Mapping):+ raise ConfigurationError(f"{key} must be a mapping")+ return value+++def _required(mapping: Mapping[str, Any], key: str, path: str) -> Any:+ if key not in mapping:+ raise ConfigurationError(f"{path}.{key} is required")+ return mapping[key]+++def _equal(mapping: Mapping[str, Any], key: str, expected: Any, path: str) -> None:+ observed = _required(mapping, key, path)+ if isinstance(expected, bool):+ matches = observed is expected+ elif isinstance(expected, int):+ matches = (+ isinstance(observed, int) and not isinstance(observed, bool) and observed == expected+ )+ elif isinstance(expected, float):+ matches = (+ isinstance(observed, (int, float))+ and not isinstance(observed, bool)+ and float(observed) == expected+ )+ else:+ matches = observed == expected+ if not matches:+ raise ConfigurationError(f"{path}.{key} must be {expected!r}, got {observed!r}")+++def _positive_int(mapping: Mapping[str, Any], key: str, path: str) -> int:+ value = _required(mapping, key, path)+ if isinstance(value, bool) or not isinstance(value, int) or value <= 0:+ raise ConfigurationError(f"{path}.{key} must be a positive integer")+ return value+++def _number(mapping: Mapping[str, Any], key: str, path: str) -> float:+ value = _required(mapping, key, path)+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ raise ConfigurationError(f"{path}.{key} must be numeric")+ return float(value)+++def _close(mapping: Mapping[str, Any], key: str, expected: float, path: str) -> None:+ observed = _number(mapping, key, path)+ if abs(observed - expected) > 1e-12:+ raise ConfigurationError(f"{path}.{key} must be {expected}, got {observed}")+++def _nonempty_path(mapping: Mapping[str, Any], key: str, path: str) -> str:+ value = _required(mapping, key, path)+ if not isinstance(value, str) or not value.strip():+ raise ConfigurationError(f"{path}.{key} must be a non-empty path")+ return value+++def _validate_model(+ model: Mapping[str, Any], *, action: str, input_mode: str, stage: str = "stage0p5"+) -> None:+ if "checkpoint_digest" in model:+ raise ConfigurationError(+ "model does not support removed content-digest fields: ['checkpoint_digest']"+ )+ family = str(_required(model, "family", "model")).strip().lower()+ if family not in {"minimax_h3", "minimax-h3"}:+ raise ConfigurationError(f"model.family must select MiniMax-H3, got {family!r}")+ _nonempty_path(model, "checkpoint_path", "model")+ if action == "preencode":+ codec_identity = str(_required(model, "codec_identity", "model")).strip()+ if not codec_identity:+ raise ConfigurationError("model.codec_identity must be non-empty")+ for key, expected in (+ ("architecture", "minimax-h3-33b"),+ ("torch_dtype", "bfloat16"),+ ("camera_attention_mode", "fused_prope"),+ ("camera_translation_transform", "logd4"),+ ):+ _equal(model, key, expected, "model")+ intrinsics_mode = str(model.get("camera_intrinsics_mode", "wan_fixed"))+ if intrinsics_mode not in {"wan_fixed", "sample"}:+ raise ConfigurationError("model.camera_intrinsics_mode must be wan_fixed or sample")+ for key, expected in (+ ("attention_head_dim", 128),+ ("camera_prope_head_dim_start", 96),+ ("camera_prope_head_dim_end", 128),+ ("latent_channels", 24),+ ("latent_height", 48),+ ("latent_width", 84),+ ("rows_per_latent", 1008),+ ("num_frames_per_block", 5),+ ("max_prior_clean_chunks", 5),+ ):+ _equal(model, key, expected, "model")+ if action == "preencode":+ return+ for key, expected in (+ ("training_mode", "lora"),+ ("transformer_subfolder", "transformer"),+ ("transformer_device_map", None),+ ("load_conditioners", False),+ ):+ _equal(model, key, expected, "model")+ # Stage1/2 use the compiled block-mask path. Stage0.5's unmasked attention may also run on cuDNN,+ # which is the faster kernel on recent GPUs at these sequence lengths (same maths, bf16 rounding apart).+ backends = ("flex",) if stage in {"stage1", "stage2"} else ("flash", "_native_cudnn")+ if model.get("attention_backend") not in backends:+ raise ConfigurationError(+ f"model.attention_backend must be one of {backends!r}, got {model.get('attention_backend')!r}"+ )+ adapter = _mapping(model, "adapter")+ for key, expected in (+ ("type", "lora"),+ ("target", "block_qkvo_ffn"),+ ("rank", 384),+ ("alpha", 384),+ ("dropout", 0.0),+ ("bias", "none"),+ ("dtype", "bfloat16"),+ ("expected_target_linear_modules", 312),+ ("expected_trainable_parameters", 2_075_394_048),+ ):+ _equal(adapter, key, expected, "model.adapter")+++def _validate_data(data: Mapping[str, Any], *, action: str, stage: str = "stage0p5") -> str:+ input_mode = str(_required(data, "input_mode", "data")).strip().lower()+ allowed = {"raw"} if action == "preencode" else {"preencoded"}+ if input_mode not in allowed:+ raise ConfigurationError(+ f"MiniMax-H3 {action} data.input_mode must be one of {sorted(allowed)!r}; "+ f"got {input_mode!r}"+ )+ transport = _mapping(data, "transport")+ kind = str(_required(transport, "kind", "data.transport")).strip().lower()+ if kind != "local":+ raise ConfigurationError("data.transport.kind must be local")+ root = _nonempty_path(transport, "root", "data.transport")+ if not root.startswith("/"):+ raise ConfigurationError("local data.transport.root must be absolute")+ if data.get("index_root") is not None:+ index_root = _nonempty_path(data, "index_root", "data")+ if not index_root.startswith("/"):+ raise ConfigurationError("data.index_root must be absolute")+ index_fields = ("index",) if action == "preencode" else ("train_index", "test_index")+ for field in index_fields:+ _nonempty_path(data, field, "data")+ try:+ resolve_index_path(data, field)+ except Exception as exc:+ raise ConfigurationError(f"cannot resolve data.{field}: {exc}") from exc+ for key, expected in (+ ("pixel_frames", 158),+ ("encoded_latents", 47),+ ("train_target_latents", 45 if stage == "stage1" else 47),+ ("height", 768),+ ("width", 1344),+ ("latent_channels", 24),+ ("latent_height", 48),+ ("latent_width", 84),+ ("frame_sampling", "contiguous"),+ ("source_fps_policy", "audit_only"),+ ("random_start", False),+ ("resolution_label", "default"),+ ):+ _equal(data, key, expected, "data")+ _close(data, "max_relative_translation", 20.0, "data")+ _close(data, "max_camera_absolute_value", 20.0, "data")+ validate_stage0p5_geometry(+ pixel_frames=int(data["pixel_frames"]),+ encoded_latents=int(data["encoded_latents"]),+ height=int(data["height"]),+ width=int(data["width"]),+ latent_channels=int(data["latent_channels"]),+ latent_height=int(data["latent_height"]),+ latent_width=int(data["latent_width"]),+ )+ if action == "train":+ workers = data.get("num_workers", 1)+ if isinstance(workers, bool) or not isinstance(workers, int) or workers < 1:+ raise ConfigurationError("data.num_workers must be a positive integer")+ if data.get("prefetch_factor") is not None:+ prefetch = data["prefetch_factor"]+ if isinstance(prefetch, bool) or not isinstance(prefetch, int) or prefetch < 1:+ raise ConfigurationError("data.prefetch_factor must be a positive integer")+ if "omni_index" in data or "omni_root" in data:+ # Omni samples (tools/omni_preencode.py): any 5n+2 clip length, untyped observations.+ if action != "train" or stage != "stage0p5":+ raise ConfigurationError("data.omni_index applies to Stage0.5 training")+ for key in ("omni_index", "omni_root"):+ location = _nonempty_path(data, key, "data")+ if not (location.startswith("/") or "://" in location):+ raise ConfigurationError(f"data.{key} must be absolute or a remote storage location")+ if not isinstance(data.get("omni_length_buckets", False), bool):+ raise ConfigurationError("data.omni_length_buckets must be true or false")+ if input_mode == "preencoded":+ _equal(data, "preencode_version", H3_PREENCODE_VERSION, "data")+ _equal(data, "dataset_name", "h3_preencoded_wds", "data")+ _nonempty_path(data, "silence_latents_path", "data")+ _nonempty_path(data, "encoder_contract_path", "data")+ else:+ _equal(data, "fixed_start_from_index", True, "data")+ return input_mode+++def _validate_geometry(config: Mapping[str, Any], *, action: str, stage: str) -> None:+ """Validate the optional geometry ControlNet block and everything it implies."""++ if action == "preencode":+ if _mapping(config, "model").get("geometry_controlnet") is not None:+ raise ConfigurationError(+ "model.geometry_controlnet does not apply to preencode; select renders with "+ "preencode.geometry"+ )+ geometry_renders_from_preencode_config(_mapping(config, "preencode"))+ return+ profile = geometry_profile_from_model_config(_mapping(config, "model"))+ if profile is not None and profile.reads_observation:+ if not _mapping(config, "data").get("omni_index"):+ raise ConfigurationError(+ "model.geometry_controlnet.modalities [observation] reads each sample's aligned "+ "observation and needs omni samples (data.omni_index)"+ )+ initialization = config.get("checkpoint", {}).get("initialization") or {}+ if isinstance(initialization, Mapping) and "geometry_controlnet_weights" in initialization:+ if profile is None or stage != "stage0p5":+ raise ConfigurationError(+ "checkpoint.initialization.geometry_controlnet_weights seeds a Stage0.5 run "+ "with model.geometry_controlnet.enabled"+ )+ _nonempty_path(initialization, "geometry_controlnet_weights", "checkpoint.initialization")+ if isinstance(initialization, Mapping):+ for role, item in initialization.items():+ if isinstance(item, Mapping) and "geometry_controlnet" in item:+ if profile is None:+ raise ConfigurationError(+ f"checkpoint.initialization.{role}.geometry_controlnet requires "+ "model.geometry_controlnet.enabled"+ )+ if item["geometry_controlnet"] not in {"required", "fresh"}:+ raise ConfigurationError(+ f"checkpoint.initialization.{role}.geometry_controlnet must be "+ "required or fresh"+ )+ if profile is None:+ for key in ("geometry_controlnet_optimizer", "critic_geometry_controlnet_optimizer"):+ if key in config.get("train", {}):+ raise ConfigurationError(f"train.{key} requires model.geometry_controlnet.enabled")+ return+ validate_geometry_parallelism(+ profile, int(_mapping(config, "distributed")["sequence_parallel_size"])+ )+ if action != "train":+ return+ train = _mapping(config, "train")+ for key in (+ ("geometry_controlnet_optimizer", "critic_geometry_controlnet_optimizer")+ if stage == "stage2"+ else ("geometry_controlnet_optimizer",)+ ):+ optimizer = _mapping(train, key)+ unknown = sorted(set(optimizer) - {"learning_rate", "weight_decay"})+ if unknown:+ raise ConfigurationError(f"train.{key} has unsupported keys {unknown}")+ if _number(optimizer, "learning_rate", f"train.{key}") <= 0:+ raise ConfigurationError(f"train.{key}.learning_rate must be positive")+ if _number(optimizer, "weight_decay", f"train.{key}") < 0:+ raise ConfigurationError(f"train.{key}.weight_decay must be non-negative")+ if stage == "stage0p5" and isinstance(initialization, Mapping):+ item = initialization.get("student")+ if item is not None:+ if not isinstance(item, Mapping):+ raise ConfigurationError("checkpoint.initialization.student must be a mapping")+ _nonempty_path(item, "path", "checkpoint.initialization.student")+ _equal(item, "stage", "stage0p5", "checkpoint.initialization.student")+ if item.get("weight_source") not in {"live", "ema"}:+ raise ConfigurationError(+ "checkpoint.initialization.student.weight_source must be live or ema"+ )+++def _validate_observations(train: Mapping[str, Any], *, action: str, stage: str) -> None:+ """The optional known-degree training distribution; only Stage0.5 trains with it."""++ if "observations" not in train:+ return+ if action != "train" or stage != "stage0p5":+ raise ConfigurationError("train.observations applies to Stage0.5 training")+ from .observations import observation_profile_from_train_config++ observation_profile_from_train_config(train)+++def _validate_flow_map(config: Mapping[str, Any]) -> None:+ """A Stage0.5 flow-map run distils a trained omni run: it starts from that run's weights."""++ item = (config.get("checkpoint", {}).get("initialization") or {}).get("student")+ if not isinstance(item, Mapping):+ raise ConfigurationError(+ "a Stage0.5 flow-map run starts from a trained omni checkpoint "+ "(checkpoint.initialization.student)"+ )+ _nonempty_path(item, "path", "checkpoint.initialization.student")+ _equal(item, "stage", "stage0p5", "checkpoint.initialization.student")+ if item.get("weight_source") not in {"live", "ema"}:+ raise ConfigurationError("checkpoint.initialization.student.weight_source must be live or ema")+ if not _mapping(config, "data").get("omni_index"):+ raise ConfigurationError("a Stage0.5 flow-map run distils omni samples (data.omni_index)")+++def _validate_route(train: Mapping[str, Any]) -> None:+ from solarwm.config.routes import Route, supported_routes++ route = Route(+ "minimax_h3",+ str(train.get("stage", "")),+ str(train.get("causal_mode", "")),+ str(train.get("objective", "")),+ str(train.get("objective_variant", "")),+ )+ if route not in supported_routes():+ raise ConfigurationError(f"unsupported H3 training route {route.key}")+ if route.objective == "anyflow_forward_map":+ for key, expected in (+ ("anyflow_gate", 0.25),+ ("deltatime_type", "r"),+ ("finite_difference_epsilon", 5.0),+ ("diffusion_ratio", 0.5),+ ("consistency_ratio", 0.25),+ ("weight_type", "gaussian"),+ ):+ _equal(train, key, expected, "train")+ elif route.stage == "stage2":+ for key, expected in (+ ("critic_updates_per_student", 5),+ ("num_denoising_steps", 4),+ ("per_rank_exit_step", True),+ ("last_step_only", False),+ ("match_context", True),+ ("cache_mode", "exit"),+ ("tail_camera_policy", "repeat_last_latent"),+ ("audio_condition_policy", "fixed_noised_silence_per_rollout"),+ ("score_min_sigma", 0.02),+ ("score_max_sigma", 0.98),+ ):+ _equal(train, key, expected, "train")+++_DMD_RECIPES = {+ # The LongLive-Plug paper's Wan2.2 DMD optimisation: the critic at 2e-6 with the student's betas.+ "wan22": (+ {"learning_rate": 1e-5, "betas": [0.0, 0.999], "weight_decay": 0.0, "gradient_clip": 10.0},+ {"learning_rate": 2e-6, "betas": [0.0, 0.999], "weight_decay": 0.0, "gradient_clip": 10.0},+ ),+ # What LongLive-Plug's own H3 release trained with (its training_config.json): a 25x faster critic+ # with its own betas, weight decay on the student, clip 1.+ "plug_h3": (+ {"learning_rate": 2e-5, "betas": [0.0, 0.9], "weight_decay": 0.01, "gradient_clip": 1.0},+ {"learning_rate": 5e-5, "betas": [0.9, 0.999], "weight_decay": 0.0, "gradient_clip": 1.0},+ ),+}+++# train.dmd keys of the released four-step LoRA the student starts from (DMAD in the release); the older+# plug* names are still read.+_DMD_ALIASES = (("init_lora", "plug"), ("init_lora_rank", "plug_rank"), ("init_lora_alpha", "plug_alpha"),+ ("seed_init_lora", "seed_plug"))+++def dmd_settings(dmd: Mapping[str, Any]) -> dict[str, Any]:+ """train.dmd with each init_lora* key also under its plug* name, which the runtime reads."""++ out = dict(dmd)+ for new, old in _DMD_ALIASES:+ if new in out:+ if old in out and out[old] != out[new]:+ raise ConfigurationError(f"train.dmd.{new} and train.dmd.{old} name the same setting; give one")+ out[old] = out[new]+ return out+++def _validate_dmd(train: Mapping[str, Any], optimizer: Mapping[str, Any]) -> None:+ """Structure-guided DMD2 from a released four-step LoRA; train.dmd.recipe names its optimisation (wan22 by+ default)."""++ recipe = str(_mapping(train, "dmd").get("recipe", "wan22"))+ if recipe not in _DMD_RECIPES:+ raise ConfigurationError(f"train.dmd.recipe must be one of {sorted(_DMD_RECIPES)}")+ student, critic_values = _DMD_RECIPES[recipe]+ shared = (("name", "fp32_master_adamw"), ("warmup_steps", 0), ("min_lr_ratio", 1.0))+ for key, expected in (*shared, *student.items()):+ _equal(optimizer, key, expected, "train.optimizer")+ critic = _mapping(train, "critic_optimizer")+ for key, expected in (*shared, *critic_values.items()):+ _equal(critic, key, expected, "train.critic_optimizer")+ if critic.get("epsilon") != optimizer.get("epsilon"):+ raise ConfigurationError("train.critic_optimizer.epsilon must equal train.optimizer.epsilon")+ dmd = dmd_settings(_mapping(train, "dmd"))+ if dmd.get("mode") not in {"adapt_plug", "self_distill", "joint", "causal"}:+ raise ConfigurationError("train.dmd.mode must be adapt_plug, self_distill, joint or causal")+ if dmd["mode"] == "causal":+ # A block-causal student, streamed chunk by chunk and replayed: the causal control LoRA (student_init) frozen in+ # its base under a separate acceleration LoRA (decoupled) that starts as the released four-step LoRA of+ # accel_init; the bidirectional control LoRA (control) teaches.+ if not dmd.get("student_init") or not dmd.get("control"):+ raise ConfigurationError("a causal DMD names its causal control (student_init) and its teacher (control)")+ if dmd.get("decoupled") is not True:+ raise ConfigurationError("a causal DMD trains a separate acceleration LoRA: train.dmd.decoupled must be true")+ stream = dmd.get("causal_stream")+ if not isinstance(stream, Mapping) or set(stream) - {"chunk", "window"}:+ raise ConfigurationError("train.dmd.causal_stream holds chunk and window")+ from .omni_causal import validate_causal_omni++ validate_causal_omni(dict(stream))+ elif not dmd.get("plug"):+ raise ConfigurationError("train.dmd.init_lora names the released four-step LoRA the student starts from")+ if str(dmd.get("teacher", "fixed")) not in {"fixed", "follow"}:+ raise ConfigurationError("train.dmd.teacher must be fixed or follow")+ # seed_init_lora false: an adapt_plug student from a zero update instead of the released LoRA+ if not isinstance(dmd.get("seed_plug", True), bool):+ raise ConfigurationError("train.dmd.seed_init_lora must be true or false")+ # PDMD projection and the real-data anchor+ if not isinstance(dmd.get("pdmd", False), bool):+ raise ConfigurationError("train.dmd.pdmd must be true or false")+ if float(dmd.get("anchor_weight", 0.0)) < 0:+ raise ConfigurationError("train.dmd.anchor_weight must not be negative")+ for kind, value in dict(dmd.get("anchor_condition_weights", {})).items():+ if kind not in {"full", "structure", "first", "text"} or float(value) < 0:+ raise ConfigurationError(+ "train.dmd.anchor_condition_weights maps full / structure / first / text to weights >= 0"+ )+ # structure-guided distribution matching: the teacher's answer pushed along the structure direction+ guidance = dmd.get("structure_guidance", 1.0)+ if isinstance(guidance, bool) or not isinstance(guidance, (int, float)) or float(guidance) < 1.0:+ raise ConfigurationError("train.dmd.structure_guidance must be a number >= 1 (1 = the plain teacher)")+ warmup = dmd.get("structure_guidance_warmup", 0)+ if isinstance(warmup, bool) or not isinstance(warmup, int) or warmup < 0:+ raise ConfigurationError("train.dmd.structure_guidance_warmup must be a whole number of student steps >= 0")+ # guidance only where the re-noise level is at least this - the structure is laid down at high noise+ cutoff = dmd.get("structure_guidance_sigma_min", 0.0)+ if isinstance(cutoff, bool) or not isinstance(cutoff, (int, float)) or not 0.0 <= float(cutoff) < 1.0:+ raise ConfigurationError("train.dmd.structure_guidance_sigma_min must be a noise level in [0, 1)")+ # quality-guided distribution matching: away from the teacher's answer under a low-quality prompt+ quality = dmd.get("quality_guidance", 0.0)+ if isinstance(quality, bool) or not isinstance(quality, (int, float)) or float(quality) < 0.0:+ raise ConfigurationError("train.dmd.quality_guidance must be a number >= 0 (0 = off)")+ if float(quality) > 0.0 and not isinstance(dmd.get("quality_negative"), str):+ raise ConfigurationError("train.dmd.quality_negative names the low-quality prompt's safetensors")+ ceiling = dmd.get("quality_guidance_sigma_max", 1.0)+ if isinstance(ceiling, bool) or not isinstance(ceiling, (int, float)) or not 0.0 < float(ceiling) <= 1.0:+ raise ConfigurationError("train.dmd.quality_guidance_sigma_max must be a noise level in (0, 1]")+ # joint may start from the base: its control slots then train from zero.+ if dmd["mode"] == "adapt_plug" and not dmd.get("control"):+ raise ConfigurationError("train.dmd.control names the frozen control checkpoint")+++def _validate_training(+ train: Mapping[str, Any],+ distributed: Mapping[str, Any],+) -> None:+ stage = str(train["stage"])+ sgf = stage == "stage2"+ for key, expected in (+ ("precision", "bfloat16"),+ ("micro_batch_size", 1),+ ("gradient_accumulation_steps", 1),+ ("video_timestep_shift", 12.0),+ ("audio_timestep_shift", 3.0),+ ("keyframe_noise_augmentation", 0.999),+ ("audio_loss_weight", 0.0),+ ):+ _equal(train, key, expected, "train")+ optimizer = _mapping(train, "optimizer")+ # A flow map trains at the flow map's step size, whichever stage its inputs come from.+ schedule = "stage1" if str(train.get("objective")) == "anyflow_forward_map" else stage+ if str(train.get("objective")) == "dmd":+ _validate_dmd(train, optimizer)+ elif train.get("causal_omni") is not None:+ # block-causal omni: Stage1's teacher-forcing step size, or the 1e-4 the control LoRA itself was+ # trained at, after a short warmup from that LoRA+ for key, expected in (("name", "fp32_master_adamw"), ("warmup_steps", 50)):+ _equal(optimizer, key, expected, "train.optimizer")+ if _required(optimizer, "learning_rate", "train.optimizer") not in (3e-5, 1e-4):+ raise ConfigurationError("train.optimizer.learning_rate of a block-causal run must be 3e-05 or 0.0001")+ else:+ for key, expected in (+ ("name", "fp32_master_adamw"),+ ("learning_rate", {"stage0p5": 1e-4, "stage1": 3e-5, "stage2": 2e-6}[schedule]),+ ):+ _equal(optimizer, key, expected, "train.optimizer")+ # The warmup is counted in optimizer steps, so it scales with the global batch (500 at the paper's batch of+ # 128 is 16,000 at a batch of 4); any non-negative count is accepted.+ warmup = _required(optimizer, "warmup_steps", "train.optimizer")+ if not isinstance(warmup, int) or isinstance(warmup, bool) or warmup < 0:+ raise ConfigurationError(f"train.optimizer.warmup_steps must be a non-negative integer, got {warmup!r}")+ if stage != "stage0p5":+ for key, expected in (+ ("betas", [0.0, 0.999] if sgf else [0.9, 0.95]),+ ("epsilon", 1e-8),+ ("weight_decay", 0.0 if sgf else 0.01),+ ("gradient_clip", 10.0 if sgf else 1.0),+ ("min_lr_ratio", 1.0 if sgf else 0.1),+ ):+ _equal(optimizer, key, expected, "train.optimizer")+ if sgf:+ critic = _mapping(train, "critic_optimizer")+ for key, value in optimizer.items():+ _equal(critic, key, 4e-7 if key == "learning_rate" else value, "train.critic_optimizer")+ fsdp = _mapping(train, "fsdp")+ for key, expected in (+ ("sharding_strategy", "HYBRID_SHARD" if sgf else "FULL_SHARD"),+ ("activation_checkpointing", True),+ ("preserve_checkpoint_dtype", True),+ ("param_dtype", None),+ ("reduce_dtype", "float32"),+ ("buffer_dtype", None),+ ):+ _equal(fsdp, key, expected, "train.fsdp")+ saved = fsdp.get("saved_attention_blocks", 0)+ if isinstance(saved, bool) or not isinstance(saved, int) or not 0 <= saved <= 50:+ raise ConfigurationError(+ f"train.fsdp.saved_attention_blocks must be an integer in [0, 50], got {saved!r}"+ )++ world_size = _positive_int(distributed, "world_size", "distributed")+ sequence_parallel = _positive_int(distributed, "sequence_parallel_size", "distributed")+ _equal(distributed, "rank_partition", "node_shard", "distributed")+ _equal(distributed, "context_parallel_size", 1, "distributed")+ _equal(distributed, "sp_peers_share_sample", True, "distributed")+ _equal(distributed, "sp_peers_share_rng", True, "distributed")+ if "observations" in train:+ # Visual observations lengthen the packed document; the run sizes SP for its data.+ if sequence_parallel not in (2, 4, 8):+ raise ConfigurationError("H3 observation runs need sequence_parallel_size 2, 4 or 8")+ elif sequence_parallel != (4 if sgf else 2):+ raise ConfigurationError(f"H3 {stage} requires sequence_parallel_size={4 if sgf else 2}")+ if sgf:+ _equal(fsdp, "frozen_base_shard_size", 8, "train.fsdp")+ if world_size % sequence_parallel:+ raise ConfigurationError("distributed.world_size must be divisible by SP size")+ # Each stage fixes SP, micro batch and accumulation. Changing the world size+ # also changes the global batch and optimization trajectory.+ computed_global_batch = (+ world_size+ // sequence_parallel+ * int(train["micro_batch_size"])+ * int(train["gradient_accumulation_steps"])+ )+ if computed_global_batch != int(train["global_batch_size"]):+ raise ConfigurationError(+ "global batch mismatch: (world_size / SP) * micro_batch * grad_accum "+ f"is {computed_global_batch}, configured {train['global_batch_size']}"+ )+++def _validate_validation(+ validation: Mapping[str, Any], *, action: str, stage: str = "stage0p5"+) -> None:+ del action+ _positive_int(validation, "sample_count", "validation")+ for name in ("selection_seed", "noise_seed"):+ value = _required(validation, name, "validation")+ if isinstance(value, bool) or not isinstance(value, int) or value < 0:+ raise ConfigurationError(f"validation.{name} must be a non-negative integer")+ for key, expected in (+ ("pixel_frames", 158),+ ("latent_frames", 47),+ ("fps", 24),+ ("num_inference_steps", 30 if stage == "stage0p5" else 4),+ ("passes", ["live", "ema"]),+ ):+ _equal(validation, key, expected, "validation")+ smoke_step = _required(validation, "smoke_step", "validation")+ if isinstance(smoke_step, bool) or not isinstance(smoke_step, int) or smoke_step < 0:+ raise ConfigurationError("validation.smoke_step must be a non-negative integer")+ _close(validation, "max_relative_translation", 20.0, "validation")+ _close(validation, "max_camera_absolute_value", 20.0, "validation")+++def _validate_checkpoint(checkpoint: Mapping[str, Any], *, stage: str = "stage0p5") -> None:+ _equal(checkpoint, "save_optimizer", True, "checkpoint")+ ema = _mapping(checkpoint, "ema")+ for key, expected in (+ ("enabled", True),+ ("dtype", "float32"),+ ("sharded", True),+ ("decay", {"stage0p5": 0.9999, "stage1": 0.999, "stage2": 0.99}[stage]),+ ("start_step", 39 if stage == "stage2" else 0),+ ("update_every_steps", 1),+ ):+ _equal(ema, key, expected, "checkpoint.ema")+++def _validate_inference_distributed(+ distributed: Mapping[str, Any], *, stage: str = "stage0p5", full_length: bool = False+) -> None:+ world_size = _positive_int(distributed, "world_size", "distributed")+ sequence_parallel = _positive_int(+ distributed,+ "sequence_parallel_size",+ "distributed",+ )+ expected_sp = 8 if full_length else (4 if stage == "stage2" else 2)+ if full_length and world_size != 8:+ raise ConfigurationError("Source-length H3 inference requires one eight-GPU SP8 worker")+ if sequence_parallel != expected_sp or world_size % sequence_parallel:+ raise ConfigurationError(f"H3 inference requires a world divisible by SP{expected_sp}")+ for key, expected in (+ ("context_parallel_size", 1),+ ("sp_peers_share_sample", True),+ ("sp_peers_share_rng", True),+ ):+ _equal(distributed, key, expected, "distributed")+++def validate_h3_config(config: Mapping[str, Any]) -> H3RunContract:+ """Validate one config against the supported H3 profile."""++ if not isinstance(config, Mapping):+ raise ConfigurationError("config must be a mapping")+ action = str(config.get("action", "")).strip().lower()+ if action not in {"train", "infer", "preencode"}:+ raise ConfigurationError("action must be train, infer, or preencode")+ inference = config.get("inference", {})+ if not isinstance(inference, Mapping):+ raise ConfigurationError("inference must be a mapping")+ length_policy = inference.get("length_policy", "fixed")+ if length_policy not in {"fixed", "source"}:+ raise ConfigurationError("inference.length_policy must be fixed or source")+ full_length = length_policy == "source"+ if full_length:+ if action != "infer" or config.get("train", {}).get("stage") != "stage2":+ raise ConfigurationError("Source-length inference is only supported for H3 Stage2 SGF")+ for key in ("plan", "work_dir", "dataset_root"):+ _nonempty_path(inference, key, "inference")+ if inference.get("fps_policy") != "source":+ raise ConfigurationError("Source-length inference requires fps_policy=source")+ if inference.get("phase", "all") not in {"all", "prepare", "generate"}:+ raise ConfigurationError("inference.phase must be all, prepare or generate")+ source = config.get("checkpoint", {}).get("weight_source")+ if source not in {"live", "ema"}:+ raise ConfigurationError(+ "Source-length inference requires explicit live or ema weights"+ )+ _positive_int(inference, "expected_step", "inference")+ model = _mapping(config, "model")+ data = _mapping(config, "data")+ metadata = config.get("metadata", {})+ if not isinstance(metadata, Mapping):+ raise ConfigurationError("metadata must be a mapping")+ stage = "preencode" if action == "preencode" else str(_mapping(config, "train")["stage"])+ if stage not in {"preencode", "stage0p5", "stage1", "stage2"}:+ raise ConfigurationError(f"unsupported H3 stage {stage!r}")+ declared_input_mode = str(data.get("input_mode", "")).strip().lower()+ _validate_model(model, action=action, input_mode=declared_input_mode, stage=stage)+ input_mode = _validate_data(data, action=action, stage=stage)+ _validate_geometry(config, action=action, stage=stage)++ if action != "preencode":+ train = _mapping(config, "train")+ _validate_route(train)+ if stage == "stage0p5" and str(train.get("objective")) == "anyflow_forward_map":+ _validate_flow_map(config)+ _validate_observations(train, action=action, stage=stage)+ if train.get("causal_omni") is not None:+ # block-causal omni rendering: an omni Stage0.5 flow-matching run trained chunk by chunk+ if stage != "stage0p5" or str(train.get("objective")) != "flow_matching" or not data.get("omni_index"):+ raise ConfigurationError("train.causal_omni applies to omni Stage0.5 flow-matching training")+ from .omni_causal import validate_causal_omni++ validate_causal_omni(train["causal_omni"])+ if stage in {"stage1", "stage2"}:+ validation = _mapping(config, "validation")+ if validation.get("prepared_plan") is not None:+ _nonempty_path(validation, "prepared_plan", "validation")+ elif stage == "stage1":+ _nonempty_path(data, "raw_test_index", "data")+ for key, expected in (+ ("rollout_latents", 50),+ ("decode_latents", 47),+ ("camera_frames", 170 if stage == "stage1" else 158),+ ):+ _equal(validation, key, expected, "validation")+ if stage == "stage2":+ _equal(model, "student_rope_mode", "sliding_local", "model")+ _equal(model, "score_rope_mode", "native_absolute", "model")+ if action == "train":+ initialization = _mapping(_mapping(config, "checkpoint"), "initialization")+ for role in ("student", "teacher", "critic") if stage == "stage2" else ("student",):+ item = _mapping(initialization, role)+ _nonempty_path(item, "path", f"checkpoint.initialization.{role}")+ _equal(item, "weight_source", "ema", f"checkpoint.initialization.{role}")+ _equal(+ item,+ "stage",+ "stage1" if stage == "stage2" and role == "student" else "stage0p5",+ f"checkpoint.initialization.{role}",+ )++ _validate_validation(_mapping(config, "validation"), action=action, stage=stage)+ if action == "train":+ _validate_training(train, _mapping(config, "distributed"))+ _validate_checkpoint(_mapping(config, "checkpoint"), stage=stage)+ else:+ _validate_inference_distributed(+ _mapping(config, "distributed"), stage=stage, full_length=full_length+ )+ _nonempty_path(_mapping(config, "checkpoint"), "resume_from", "checkpoint")+ else:+ preencode = _mapping(config, "preencode")+ _equal(preencode, "codec_protocol", "solarwm.minimax_h3.codec.v1", "preencode")+ _nonempty_path(preencode, "output_root", "preencode")++ runtime = _mapping(config, "runtime")+ _nonempty_path(runtime, "output_dir", "runtime")+ contract = h3_fused_prope_contract(+ str(_mapping(config, "model").get("camera_intrinsics_mode", "wan_fixed"))+ )+ if contract["camera_prope_head_slice"] != [96, 128]:+ raise ConfigurationError("internal H3 camera suffix contract is inconsistent")+ return H3RunContract(+ action=action,+ stage=stage,+ pixel_frames=158,+ encoded_latents=47,+ sequence_parallel_size=(+ 0+ if action == "preencode"+ else int(_mapping(config, "distributed")["sequence_parallel_size"])+ ),+ adapter_rank=0 if action == "preencode" else 384,+ camera_translation_transform="logd4",+ data_input_mode=input_mode,+ )+++__all__ = ["H3RunContract", "validate_h3_config"]diff --git a/src/solarwm/backends/minimax_h3/controlnet.py b/src/solarwm/backends/minimax_h3/controlnet.pynew file mode 100644index 0000000..69c89da--- /dev/null+++ b/src/solarwm/backends/minimax_h3/controlnet.py@@ -0,0 +1,744 @@+"""Geometry ControlNet trunk for MiniMax-H3 video-to-video conditioning.++A narrow transformer reads VAE-encoded renders of the target clip and adds a+zero-initialised residual into the backbone every ``layer_stride`` blocks.+Structure, parameter names and initialisation follow FastVideo's spatial-AI+trunk; only the attention plumbing is SolarWM's.++The control stream lies over the same packed rows as the backbone stream: zero+everywhere except the rows that receive control. Both streams are therefore+sharded identically by sequence parallelism, and the residual add is local.+Every row attends, but only the control rows receive a residual.++Two attention patterns cover every stage. ``dense`` is full bidirectional+attention (Stage0.5, bidirectional sampling, SGF score models). ``segments``+is chunk-local: condition rows see condition rows, audio rows see the prefix,+and each (clean|noisy, chunk) group sees the prefix plus itself. A chunk's+control never reads another chunk, so autoregressive rollouts need no control+KV cache, and teacher-forced windows, rollouts and replays compute identical+control for a chunk.+"""++from __future__ import annotations++import hashlib+import math+from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from itertools import pairwise+from typing import Any++import torch+import torch.nn as nn+import torch.nn.functional as F++from .controlnet_contract import (+ H3_GEOMETRY_MODALITIES,+ H3_GEOMETRY_MODULE,+ H3_ROTARY_CHANNELS,+ H3GeometryProfile,+)+from .layout import AUDIO_ROLE, CONDITION_ROLE++Segment = tuple[int, int, tuple[tuple[int, int], ...]]+++def apply_rotary(hidden_states: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:+ """H3's rotate-half RoPE over the leading channels of ``[B,S,H,D]`` heads."""++ rotary_dim = cos.shape[-1]+ rotary, passthrough = hidden_states[..., :rotary_dim], hidden_states[..., rotary_dim:]+ cos = cos.to(hidden_states.dtype)[None, :, None, :]+ sin = sin.to(hidden_states.dtype)[None, :, None, :]+ first, second = rotary.chunk(2, dim=-1)+ rotated = torch.cat((-second, first), dim=-1)+ return torch.cat((rotary * cos + rotated * sin, passthrough), dim=-1).contiguous()+++@dataclass(frozen=True)+class H3GeometryControl:+ """Per-forward ControlNet inputs, laid out against one packed document.++ ``rows`` holds one ``[B,N,96]`` patchified latent per modality, in the order+ of ``row_indices`` (the packed rows that receive control). ``position_ids``+ is the ``[S,3]`` rotary grid the control trunk uses for every packed row.+ ``segments`` is ``None`` for dense attention.+ """++ rows: Mapping[str, Any]+ row_indices: Any+ position_ids: Any+ segments: tuple[Segment, ...] | None = None+ wide_segments: tuple[Segment, ...] | None = None+ wide_rows: Mapping[str, Any] | None = None+ wide_position_ids: Any = None+ # Scales the residual. Zero reproduces the backbone exactly while every trunk+ # parameter still receives a gradient, so replicated ranks that drop the control+ # keep the same gradient pattern as ranks that use it.+ gain: float = 1.0+++def _as_numpy(value: Any) -> Any:+ import numpy as np++ if isinstance(value, torch.Tensor):+ return value.detach().cpu().numpy()+ return np.asarray(value)+++def _runs(roles: Any, chunks: Any) -> list[tuple[int, int, tuple[int, int]]]:+ """Contiguous ``(start, stop, (role, chunk))`` runs over the packed rows."""++ import numpy as np++ if roles.shape != chunks.shape or roles.ndim != 1 or not roles.size:+ raise ValueError("H3 control segments need one role and chunk ID per packed row")+ changed = (roles[1:] != roles[:-1]) | (chunks[1:] != chunks[:-1])+ bounds = [0, *(np.flatnonzero(changed) + 1).tolist(), int(roles.size)]+ return [+ (start, stop, (int(roles[start]), int(chunks[start]))) for start, stop in pairwise(bounds)+ ]+++def control_segments(layout: Any) -> tuple[tuple[Segment, ...], tuple[Segment, ...]]:+ """Chunk-local self-attention and wide cross-attention segments for ``layout``.++ Derived from the layout's row roles and chunk IDs, so Stage1 windows,+ sampler windows and SGF rollout/replay documents share one rule.+ """++ roles = _as_numpy(layout.row_roles).astype("int64")+ chunks = _as_numpy(layout.target_video_chunk_ids).astype("int64")+ runs = _runs(roles, chunks)+ if len(runs) < 3 or runs[0][2][0] != CONDITION_ROLE or runs[1][2][0] != AUDIO_ROLE:+ raise ValueError("H3 control segments require [condition | audio | video groups]")+ condition = (runs[0][0], runs[0][1])+ audio = (runs[1][0], runs[1][1])+ prefix = (condition, audio)+ seen: set[tuple[int, int]] = set()+ segments: list[Segment] = [+ (condition[0], condition[1], (condition,)),+ (audio[0], audio[1], prefix),+ ]+ wide: list[Segment] = []+ for start, stop, label in runs[2:]:+ if label[0] in (CONDITION_ROLE, AUDIO_ROLE) or label in seen:+ raise ValueError("H3 control segments require one contiguous run per video group")+ seen.add(label)+ segments.append((start, stop, (*prefix, (start, stop))))+ wide.append((start, stop, ((start, stop),)))+ return tuple(segments), tuple(wide)+++def wide_position_ids(position_ids: Any, row_indices: Any, fov_scale: float) -> Any:+ """Place wide-render rows on the narrow render's ray grid.++ Wide and narrow renders share pose, resolution and principal point; only+ the focal length differs by ``k``. A ray ``d`` from the centre of the narrow+ image lands at ``k*d`` in the wide one, so the wide rows sit at+ ``pivot + (pos - pivot) / k``. The pivot is the midpoint of the actual+ (endpoint-excluded) grid. Time is unchanged.+ """++ if not fov_scale > 0.0:+ raise ValueError("wide fov_scale must be positive")+ result = position_ids.clone()+ if float(fov_scale) == 1.0:+ return result+ rows = row_indices.to(device=position_ids.device, dtype=torch.long)+ spatial = result.index_select(0, rows)[:, 1:]+ pivot = 0.5 * (spatial.amin(dim=0) + spatial.amax(dim=0))+ result[rows, 1:] = pivot + (spatial - pivot) / float(fov_scale)+ return result+++def pad_control_latents(latents: Any, frames: int) -> Any:+ """Extend ``[B,C,F,H,W]`` latents to ``frames`` by repeating the last frame.++ Matches the SGF camera tail policy (``repeat_last_latent``) for rollouts+ longer than the encoded clip.+ """++ available = int(latents.shape[2])+ if frames < available:+ return latents[:, :, :frames]+ if frames == available:+ return latents+ tail = latents[:, :, -1:].expand(-1, -1, frames - available, -1, -1)+ return torch.cat((latents, tail), dim=2)+++def geometry_control_for_layout(+ layout: Any,+ rows: Mapping[str, Any],+ *,+ profile: H3GeometryProfile,+ wide_rows: Mapping[str, Any] | None = None,+ position_ids: Any = None,+ row_indices: Any = None,+ attention: str = "dense",+) -> H3GeometryControl:+ """Bind patchified geometry rows to every non-anchor video row of ``layout``.++ ``row_indices``/``position_ids`` default to the layout's own arrays; callers+ holding device copies pass them to avoid a host round trip.+ """++ if attention not in {"dense", "segments"}:+ raise ValueError("H3 control attention must be dense or segments")+ if row_indices is None:+ video_indices = layout.video_indices+ if not isinstance(video_indices, torch.Tensor):+ video_indices = torch.as_tensor(video_indices)+ row_indices = video_indices[int(layout.num_condition_video_rows) :]+ row_indices = row_indices.long()+ positions = layout.position_ids if position_ids is None else position_ids+ if not isinstance(positions, torch.Tensor):+ positions = torch.as_tensor(positions)+ segments = wide_segments = None+ if attention == "segments":+ segments, wide_segments = control_segments(layout)+ wide_positions = None+ if profile.wide_enabled:+ if wide_rows is None:+ raise ValueError("H3 wide geometry profile requires wide rows")+ wide_positions = wide_position_ids(positions, row_indices, float(profile.wide_fov_scale))+ return H3GeometryControl(+ rows=dict(rows),+ row_indices=row_indices,+ position_ids=positions,+ segments=segments,+ wide_segments=wide_segments,+ wide_rows=None if wide_rows is None else dict(wide_rows),+ wide_position_ids=wide_positions,+ )+++def teacher_forced_geometry_control(+ layout: Any,+ latents: Mapping[str, Any] | None,+ *,+ profile: H3GeometryProfile,+ frames: slice,+) -> H3GeometryControl | None:+ """Chunk-local control for a ``[condition | audio | clean | noisy]`` document.++ Both copies of the window carry the geometry of their own frames: a history+ frame's control reaches the target through the backbone's causal residual+ stream, exactly as it does when a rollout commits that frame.+ """++ if latents is None:+ return None+ from .layout import patchify_video++ def rows(name: str) -> Any:+ window = patchify_video(latents[name][:, :, frames])+ return torch.cat((window, window), dim=1)++ return geometry_control_for_layout(+ layout,+ {name: rows(name) for name in profile.modalities},+ profile=profile,+ wide_rows=(+ {name: rows(f"{name}_wide") for name in profile.wide_modalities}+ if profile.wide_enabled+ else None+ ),+ attention="segments",+ )+++class _SequenceExchange(torch.autograd.Function):+ """Ulysses all-to-all that restores its own document layout in both passes.++ The shared SP state holds the most recent document's per-rank lengths. A+ Stage2 student backward can run after score forwards over other documents,+ so the control exchange carries its lengths rather than reading them late.+ """++ @staticmethod+ def forward(ctx: Any, value: Any, scatter_dim: int, gather_dim: int, lengths: Any) -> Any:+ ctx.scatter_dim, ctx.gather_dim, ctx.lengths = scatter_dim, gather_dim, lengths+ return _exchange(value, scatter_dim, gather_dim, lengths)++ @staticmethod+ def backward(ctx: Any, gradient: Any) -> tuple[Any, None, None, None]:+ return _exchange(gradient, ctx.gather_dim, ctx.scatter_dim, ctx.lengths), None, None, None+++def _exchange(value: Any, scatter_dim: int, gather_dim: int, lengths: Any) -> Any:+ from .distributed import _all_to_all_raw, state++ current = state()+ previous = current.sequence_lengths+ current.sequence_lengths = tuple(lengths)+ try:+ return _all_to_all_raw(value.contiguous(), scatter_dim, gather_dim)+ finally:+ current.sequence_lengths = previous+++def _attend(query: Any, key: Any, value: Any, backend: Any) -> Any:+ """Unmasked attention over ``[B,S,H,D]`` heads."""++ if backend is None:+ output = F.scaled_dot_product_attention(+ query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)+ )+ return output.transpose(1, 2)+ from diffusers.models.attention_dispatch import dispatch_attention_fn++ return dispatch_attention_fn(+ query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, backend=backend+ )+++def _segment_attention(+ query: Any, key: Any, value: Any, segments: Sequence[Segment], backend: Any+) -> Any:+ outputs = []+ for start, stop, key_ranges in segments:+ if not key_ranges:+ outputs.append(query.new_zeros(query[:, start:stop].shape))+ continue+ keys = torch.cat([key[:, first:last] for first, last in key_ranges], dim=1)+ values = torch.cat([value[:, first:last] for first, last in key_ranges], dim=1)+ outputs.append(_attend(query[:, start:stop], keys, values, backend))+ return torch.cat(outputs, dim=1)+++@dataclass(frozen=True)+class _AttentionContext:+ segments: tuple[Segment, ...] | None+ wide_segments: tuple[Segment, ...] | None+ sequence_lengths: tuple[int, ...]+ backend: Any+++class H3ControlAttention(nn.Module):+ """Bias-free Q/K/V/O self-attention with per-head RMS query/key norms."""++ def __init__(self, dim: int, num_heads: int, head_dim: int, eps: float) -> None:+ super().__init__()+ self.num_heads = num_heads+ inner = num_heads * head_dim+ self.to_q = nn.Linear(dim, inner, bias=False)+ self.to_k = nn.Linear(dim, inner, bias=False)+ self.to_v = nn.Linear(dim, inner, bias=False)+ self.norm_q = nn.RMSNorm(head_dim, eps=eps)+ self.norm_k = nn.RMSNorm(head_dim, eps=eps)+ self.to_out = nn.Linear(inner, dim, bias=False)++ def _heads(self, projection: nn.Module, states: Any) -> Any:+ return projection(states).unflatten(-1, (self.num_heads, -1))++ def _exchange(self, value: Any, context: _AttentionContext, *, to_full: bool) -> Any:+ if len(context.sequence_lengths) <= 1:+ return value+ if to_full:+ return _SequenceExchange.apply(value, 2, 1, context.sequence_lengths)+ return _SequenceExchange.apply(value, 1, 2, context.sequence_lengths)++ def _finish(self, attended: Any, context: _AttentionContext, dtype: Any) -> Any:+ attended = self._exchange(attended, context, to_full=False)+ return self.to_out(attended.flatten(2, 3).to(dtype))++ def forward(self, states: Any, rotary: tuple[Any, Any], context: _AttentionContext) -> Any:+ query = apply_rotary(self.norm_q(self._heads(self.to_q, states)), *rotary)+ key = apply_rotary(self.norm_k(self._heads(self.to_k, states)), *rotary)+ value = self._heads(self.to_v, states)+ query = self._exchange(query, context, to_full=True)+ key = self._exchange(key, context, to_full=True)+ value = self._exchange(value, context, to_full=True)+ if context.segments is None:+ attended = _attend(query, key, value, context.backend)+ else:+ attended = _segment_attention(query, key, value, context.segments, context.backend)+ return self._finish(attended, context, states.dtype)+++class H3ControlWideCrossAttention(H3ControlAttention):+ """Cross-attention from the control stream onto the wide-FOV render.++ ``to_k``/``to_v`` carry no bias: packed rows without wide content stay+ exactly zero and act as a fixed attention sink under dense attention.+ """++ def forward( # type: ignore[override]+ self,+ states: Any,+ wide_states: Any,+ rotary: tuple[Any, Any],+ wide_rotary: tuple[Any, Any],+ context: _AttentionContext,+ ) -> Any:+ query = apply_rotary(self.norm_q(self._heads(self.to_q, states)), *rotary)+ key = apply_rotary(self.norm_k(self._heads(self.to_k, wide_states)), *wide_rotary)+ value = self._heads(self.to_v, wide_states)+ query = self._exchange(query, context, to_full=True)+ key = self._exchange(key, context, to_full=True)+ value = self._exchange(value, context, to_full=True)+ if context.wide_segments is None:+ attended = _attend(query, key, value, context.backend)+ else:+ segments = list(context.wide_segments)+ covered = sorted((start, stop) for start, stop, _ in segments)+ # Rows outside the video groups carry no wide content; their output is zero.+ cursor = 0+ for start, stop in covered:+ if cursor < start:+ segments.append((cursor, start, ()))+ cursor = stop+ if cursor < query.shape[1]:+ segments.append((cursor, int(query.shape[1]), ()))+ segments.sort(key=lambda item: item[0])+ attended = _segment_attention(query, key, value, segments, context.backend)+ return self._finish(attended, context, states.dtype)+++class H3ControlSwiGLU(nn.Module):+ """``fc_in`` emits ``[value; gate]``."""++ def __init__(self, dim: int, ffn_dim: int) -> None:+ super().__init__()+ self.fc_in = nn.Linear(dim, 2 * ffn_dim, bias=False)+ self.fc_out = nn.Linear(ffn_dim, dim, bias=False)++ def forward(self, states: Any) -> Any:+ value, gate = self.fc_in(states).chunk(2, dim=-1)+ return self.fc_out(value * F.silu(gate))+++class H3GeometryControlBlock(nn.Module):+ """One control block plus its zero-initialised projection into the backbone."""++ def __init__(+ self,+ *,+ dim: int,+ ffn_dim: int,+ num_heads: int,+ head_dim: int,+ temb_dim: int,+ out_dim: int,+ norm_eps: float,+ qk_norm_eps: float,+ wide: bool,+ ) -> None:+ super().__init__()+ self.norm1 = nn.RMSNorm(dim, eps=norm_eps, elementwise_affine=False)+ self.attn = H3ControlAttention(dim, num_heads, head_dim, qk_norm_eps)+ self.wca = (+ H3ControlWideCrossAttention(dim, num_heads, head_dim, qk_norm_eps) if wide else None+ )+ self.norm2 = nn.RMSNorm(dim, eps=norm_eps, elementwise_affine=False)+ self.ff = H3ControlSwiGLU(dim, ffn_dim)+ self.adaln_proj = nn.Linear(temb_dim, 6 * dim, bias=True)+ self.proj_out = nn.Linear(dim, out_dim, bias=True)++ def forward(+ self,+ states: Any,+ temb: Any,+ timestep_indices: Any,+ rotary: tuple[Any, Any],+ context: _AttentionContext,+ wide_states: Any = None,+ wide_rotary: tuple[Any, Any] | None = None,+ ) -> tuple[Any, Any]:+ modulation = self.adaln_proj(F.silu(temb).to(self.adaln_proj.weight.dtype))+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = modulation.chunk(6, -1)++ def pick(table: Any) -> Any:+ return table.index_select(0, timestep_indices)++ normalized = self.norm1(states) * (1.0 + pick(scale_msa)) + pick(shift_msa)+ states = states + pick(gate_msa) * self.attn(normalized, rotary, context)+ if self.wca is not None and wide_states is not None:+ if wide_rotary is None:+ raise ValueError("H3 wide cross-attention requires the wide rotary table")+ # Same query and modulation as the self-attention, ungated: the trunk's+ # single gate is the zero-initialised proj_out.+ states = states + self.wca(normalized, wide_states, rotary, wide_rotary, context)+ normalized = self.norm2(states) * (1.0 + pick(scale_mlp)) + pick(shift_mlp)+ states = states + pick(gate_mlp) * self.ff(normalized)+ return states, self.proj_out(states)+++@dataclass+class H3ControlStream:+ """The trunk's per-forward state on this rank's packed rows."""++ states: Any+ mask: Any+ rotary: tuple[Any, Any]+ temb: Any+ timestep_indices: Any+ context: _AttentionContext+ wide_states: Any = None+ wide_rotary: tuple[Any, Any] | None = None+++class H3GeometryControlNet(nn.Module):+ """Geometry embedding, modality fusion and one block per injection point."""++ def __init__(+ self,+ profile: H3GeometryProfile,+ *,+ video_patch_dim: int = 96,+ hidden_size: int = 5376,+ time_embed_dim: int = 2688,+ num_layers: int = 50,+ norm_eps: float = 1e-5,+ qk_norm_eps: float = 1e-5,+ rotary_channels: int = H3_ROTARY_CHANNELS,+ ) -> None:+ super().__init__()+ if profile.head_dim < rotary_channels:+ raise ValueError(+ f"H3 control head_dim={profile.head_dim} is below the {rotary_channels} "+ "rotary channels of the backbone table it reuses"+ )+ self.profile = profile+ self.modalities = tuple(+ name for name in H3_GEOMETRY_MODALITIES if name in profile.modalities+ )+ self.wide_modalities = tuple(+ name for name in H3_GEOMETRY_MODALITIES if name in profile.wide_modalities+ )+ dim = profile.dim+ self.embeddings = nn.ModuleDict(+ {name: nn.Linear(video_patch_dim, dim, bias=True) for name in self.modalities}+ )+ self.fusion_proj = (+ nn.Linear(dim * len(self.modalities), dim, bias=True)+ if len(self.modalities) > 1+ else None+ )+ self.wide_embeddings = None+ self.wide_fusion_proj = None+ self.norm_wide = None+ if self.wide_modalities:+ self.wide_embeddings = nn.ModuleDict(+ {name: nn.Linear(video_patch_dim, dim, bias=True) for name in self.wide_modalities}+ )+ if len(self.wide_modalities) > 1:+ self.wide_fusion_proj = nn.Linear(dim * len(self.wide_modalities), dim, bias=True)+ self.norm_wide = nn.RMSNorm(dim, eps=norm_eps, elementwise_affine=False)+ self.temb_proj = nn.Linear(time_embed_dim, profile.temb_dim, bias=True)+ self.control_blocks = nn.ModuleList()+ self.block_to_control: dict[int, int] = {}+ for block_index in range(num_layers):+ if block_index % profile.layer_stride:+ continue+ self.block_to_control[block_index] = len(self.control_blocks)+ self.control_blocks.append(+ H3GeometryControlBlock(+ dim=dim,+ ffn_dim=profile.ffn_dim,+ num_heads=profile.num_heads,+ head_dim=profile.head_dim,+ temb_dim=profile.temb_dim,+ out_dim=hidden_size,+ norm_eps=norm_eps,+ qk_norm_eps=qk_norm_eps,+ wide=bool(self.wide_modalities),+ )+ )+ self.attention_backend: Any = None+ self.activation_checkpointing = False++ def _fuse(+ self,+ rows: Mapping[str, Any],+ names: tuple[str, ...],+ embeddings: nn.ModuleDict,+ fusion: nn.Module | None,+ ) -> Any:+ missing = [name for name in names if name not in rows]+ if missing:+ raise ValueError(f"H3 geometry control lacks modalities {missing}")+ embedded = []+ for name in names:+ projection = embeddings[name]+ embedded.append(projection(rows[name].to(projection.weight.dtype)))+ fused = embedded[0] if len(embedded) == 1 else torch.cat(embedded, dim=-1)+ return fused if fusion is None else fusion(fused)++ @staticmethod+ def _scatter(fused: Any, row_indices: Any, sequence_length: int) -> Any:+ if int(fused.shape[1]) != int(row_indices.numel()):+ raise ValueError(+ f"H3 geometry has {int(fused.shape[1])} rows for "+ f"{int(row_indices.numel())} control rows"+ )+ stream = fused.new_zeros((fused.shape[0], sequence_length, fused.shape[-1]))+ return stream.index_copy(1, row_indices, fused)++ def prepare(+ self,+ control: H3GeometryControl,+ *,+ sequence_length: int,+ temb: Any,+ timestep_indices: Any,+ rope: Any,+ start: int = 0,+ stop: int | None = None,+ sequence_lengths: tuple[int, ...] = (),+ ) -> H3ControlStream:+ """Embed, scatter and shard the control stream for ``[start, stop)``."""++ stop = sequence_length if stop is None else int(stop)+ device = temb.device+ row_indices = control.row_indices.to(device=device, dtype=torch.long)+ positions = control.position_ids.to(device)+ if tuple(positions.shape) != (sequence_length, 3):+ raise ValueError("H3 control position_ids must be [sequence, 3]")+ fused = self._fuse(control.rows, self.modalities, self.embeddings, self.fusion_proj)+ states = self._scatter(fused, row_indices, sequence_length)+ mask = torch.zeros((1, sequence_length, 1), device=device, dtype=states.dtype)+ mask = mask.index_fill(1, row_indices, float(control.gain))+ wide_states = wide_rotary = None+ if self.wide_modalities:+ if control.wide_rows is None or control.wide_position_ids is None:+ raise ValueError("H3 wide geometry profile requires wide rows and positions")+ wide = self._fuse(+ control.wide_rows,+ self.wide_modalities,+ self.wide_embeddings,+ self.wide_fusion_proj,+ )+ wide_states = self._scatter(self.norm_wide(wide), row_indices, sequence_length)+ wide_states = wide_states[:, start:stop]+ wide_positions = control.wide_position_ids.to(device)[start:stop]+ wide_rotary = rope(wide_positions)+ context = _AttentionContext(+ segments=control.segments,+ wide_segments=control.wide_segments,+ sequence_lengths=tuple(int(value) for value in sequence_lengths),+ backend=self.attention_backend,+ )+ projected = self.temb_proj(F.silu(temb).to(self.temb_proj.weight.dtype))+ return H3ControlStream(+ states=states[:, start:stop],+ mask=mask[:, start:stop],+ rotary=rope(positions[start:stop]),+ temb=projected,+ timestep_indices=timestep_indices,+ context=context,+ wide_states=wide_states,+ wide_rotary=wide_rotary,+ )++ def inject(self, block_index: int, hidden_states: Any, stream: H3ControlStream) -> Any:+ """Advance the trunk past backbone block ``block_index`` and add its residual."""++ control_index = self.block_to_control.get(block_index)+ if control_index is None:+ return hidden_states+ block = self.control_blocks[control_index]+ arguments = (+ stream.states,+ stream.temb,+ stream.timestep_indices,+ stream.rotary,+ stream.context,+ stream.wide_states,+ stream.wide_rotary,+ )+ if self.activation_checkpointing and torch.is_grad_enabled():+ from torch.utils.checkpoint import checkpoint++ stream.states, residual = checkpoint(block, *arguments, use_reentrant=False)+ else:+ stream.states, residual = block(*arguments)+ return hidden_states + residual.to(hidden_states.dtype) * stream.mask.to(+ hidden_states.dtype+ )+++def _parameter_seed(name: str) -> int:+ digest = hashlib.sha256(name.encode("utf-8")).digest()+ return int.from_bytes(digest[:8], "big") % (2**63 - 1)++[email protected]_grad()+def initialize_geometry_controlnet(module: H3GeometryControlNet) -> None:+ """FastVideo's rule for parameters no checkpoint provides.++ Biases and every ``proj_out`` are zero (an untrained trunk is an exact+ no-op), norm weights are one, and each remaining matrix is kaiming-uniform+ from a generator seeded by its stable FastVideo name. Global RNG streams+ are untouched, so LoRA initialisation does not depend on the ControlNet.+ """++ for local_name, parameter in module.named_parameters():+ name = f"{H3_GEOMETRY_MODULE}.{local_name}"+ if name.endswith("bias") or "proj_out" in name:+ parameter.zero_()+ elif "norm_" in name and name.endswith("weight"):+ parameter.fill_(1.0)+ else:+ generator = torch.Generator(device=parameter.device).manual_seed(_parameter_seed(name))+ value = torch.empty(parameter.shape, device=parameter.device, dtype=torch.float32)+ if value.ndim >= 2:+ fan_in = value.shape[1] * math.prod(value.shape[2:])+ gain = math.sqrt(2.0 / (1.0 + 5.0))+ bound = math.sqrt(3.0) * gain / math.sqrt(fan_in)+ value.uniform_(-bound, bound, generator=generator)+ else:+ value.normal_(mean=0.0, std=0.02, generator=generator)+ parameter.copy_(value.to(parameter.dtype))+++def build_geometry_controlnet(+ profile: H3GeometryProfile,+ *,+ device: Any,+ dtype: Any = torch.bfloat16,+ hidden_size: int = 5376,+ time_embed_dim: int = 2688,+ num_layers: int = 50,+ video_patch_dim: int = 96,+ norm_eps: float = 1e-5,+ qk_norm_eps: float = 1e-5,+ rotary_channels: int = H3_ROTARY_CHANNELS,+) -> H3GeometryControlNet:+ """Construct on ``meta`` (no RNG use), materialize on ``device``, then initialise."""++ with torch.device("meta"):+ module = H3GeometryControlNet(+ profile,+ video_patch_dim=video_patch_dim,+ hidden_size=hidden_size,+ time_embed_dim=time_embed_dim,+ num_layers=num_layers,+ norm_eps=norm_eps,+ qk_norm_eps=qk_norm_eps,+ rotary_channels=rotary_channels,+ )+ module = module.to_empty(device=device).to(dtype)+ initialize_geometry_controlnet(module)+ return module+++__all__ = [+ "H3ControlAttention",+ "H3ControlStream",+ "H3ControlSwiGLU",+ "H3ControlWideCrossAttention",+ "H3GeometryControl",+ "H3GeometryControlBlock",+ "H3GeometryControlNet",+ "apply_rotary",+ "build_geometry_controlnet",+ "control_segments",+ "geometry_control_for_layout",+ "initialize_geometry_controlnet",+ "pad_control_latents",+ "teacher_forced_geometry_control",+ "wide_position_ids",+]diff --git a/src/solarwm/backends/minimax_h3/controlnet_contract.py b/src/solarwm/backends/minimax_h3/controlnet_contract.pynew file mode 100644index 0000000..9bd7eeb--- /dev/null+++ b/src/solarwm/backends/minimax_h3/controlnet_contract.py@@ -0,0 +1,310 @@+"""Torch-free identity and configuration contract for the H3 geometry ControlNet.++The ControlNet reads VAE-encoded renders of the target clip (canonical coordinate+maps, optionally depth/normal and a wider-FOV render) and adds zero-initialised+residuals to the target video rows. Its architecture and parameter names follow+the FastVideo spatial-AI trunk so trained weights transfer between the two.+"""++from __future__ import annotations++import math+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++from solarwm.errors import ConfigurationError++# Fixed concatenation order into ``fusion_proj``; part of the state-dict contract.+# ``observation`` is not a kind of signal. It is the single input of a trunk that reads whatever+# the sample lays over the target frame for frame - a render, a pose or edge video, the source of+# an edit - without being told which, so it never stands next to a named render.+H3_GEOMETRY_MODALITIES = ("depth", "normal", "ccm", "observation")+H3_OBSERVATION_INPUT = "observation"+H3_GEOMETRY_MODULE = "geometry_controlnet"+H3_GEOMETRY_STATE_PREFIX = f"base_model.model.{H3_GEOMETRY_MODULE}."+H3_GEOMETRY_SCHEMA = "solarwm.minimax-h3-geometry-controlnet.v1"+# MiniMax-H3 rotates the leading 2 * 3 * rope_freq_dim channels of every head.+H3_ROTARY_CHANNELS = 96+H3_BACKBONE_LAYERS = 50++_ARCHITECTURE_KEYS = ("layer_stride", "dim", "ffn_dim", "num_heads", "temb_dim")+_ALLOWED_KEYS = frozenset({"enabled", "modalities", "wide", "lora_trainable", *_ARCHITECTURE_KEYS})+_ALLOWED_WIDE_KEYS = frozenset({"enabled", "modalities", "fov_scale"})+# Raw preencode defines the pixel encoding of canonical coordinate maps only.+H3_RAW_GEOMETRY_MODALITIES = ("ccm",)+++@dataclass(frozen=True)+class H3GeometryRenders:+ """Which rendered videos a latent release carries, independent of the trunk."""++ modalities: tuple[str, ...]+ wide_modalities: tuple[str, ...] = ()+ wide_fov_scale: float | None = None++ @property+ def wide_enabled(self) -> bool:+ return bool(self.wide_modalities)++ def tensor_keys(self) -> tuple[str, ...]:+ """Artifact tensors, narrow before wide."""++ return tuple(geometry_tensor_key(name) for name in self.modalities) + tuple(+ geometry_tensor_key(name, wide=True) for name in self.wide_modalities+ )+++@dataclass(frozen=True)+class H3GeometryProfile:+ """Resolved ControlNet architecture plus the one training-surface flag."""++ modalities: tuple[str, ...]+ layer_stride: int+ dim: int+ ffn_dim: int+ num_heads: int+ temb_dim: int+ wide_modalities: tuple[str, ...] = ()+ wide_fov_scale: float | None = None+ lora_trainable: bool = True++ @property+ def head_dim(self) -> int:+ return self.dim // self.num_heads++ @property+ def wide_enabled(self) -> bool:+ return bool(self.wide_modalities)++ @property+ def injected_blocks(self) -> tuple[int, ...]:+ return tuple(range(0, H3_BACKBONE_LAYERS, self.layer_stride))++ def architecture(self) -> dict[str, Any]:+ """Portable identity of everything that changes the ControlNet's computation."""++ return {+ "schema": H3_GEOMETRY_SCHEMA,+ "parameter_names": "fastvideo-minimax-h3-geometry-controlnet",+ "modalities": list(self.modalities),+ "layer_stride": self.layer_stride,+ "dim": self.dim,+ "ffn_dim": self.ffn_dim,+ "num_heads": self.num_heads,+ "temb_dim": self.temb_dim,+ "wide": (+ {"modalities": list(self.wide_modalities), "fov_scale": self.wide_fov_scale}+ if self.wide_enabled+ else None+ ),+ }++ @property+ def renders(self) -> H3GeometryRenders:+ return H3GeometryRenders(self.modalities, self.wide_modalities, self.wide_fov_scale)++ @property+ def reads_observation(self) -> bool:+ """Whether the trunk reads the sample's aligned observation instead of named renders."""++ return self.modalities == (H3_OBSERVATION_INPUT,)++ def tensor_keys(self) -> tuple[str, ...]:+ """Artifact tensors this profile reads, narrow before wide; none for the observation input."""++ return () if self.reads_observation else self.renders.tensor_keys()+++def geometry_tensor_key(modality: str, *, wide: bool = False) -> str:+ if modality == H3_OBSERVATION_INPUT:+ raise ValueError(+ "the observation input is read from the sample's observations, not an artifact tensor"+ )+ if modality not in H3_GEOMETRY_MODALITIES:+ raise ValueError(f"unsupported H3 geometry modality {modality!r}")+ return f"geometry_{modality}_{'wide_' if wide else ''}latents"+++def _canonical_modalities(value: Any, path: str) -> tuple[str, ...]:+ if not isinstance(value, (list, tuple)) or not value:+ raise ConfigurationError(f"{path} must be a non-empty list of {H3_GEOMETRY_MODALITIES}")+ names = [str(item) for item in value]+ if len(set(names)) != len(names):+ raise ConfigurationError(f"{path} contains duplicate modalities")+ unknown = sorted(set(names) - set(H3_GEOMETRY_MODALITIES))+ if unknown:+ raise ConfigurationError(f"{path} has unsupported modalities {unknown}")+ if H3_OBSERVATION_INPUT in names and len(names) > 1:+ raise ConfigurationError(+ f"{path}: {H3_OBSERVATION_INPUT!r} is the trunk's only input - it reads the sample's "+ "aligned observation, whatever that is - and cannot be combined with named renders"+ )+ return tuple(name for name in H3_GEOMETRY_MODALITIES if name in names)+++def _positive(block: Mapping[str, Any], key: str, path: str) -> int:+ if key not in block:+ raise ConfigurationError(f"{path}.{key} is required")+ value = block[key]+ if isinstance(value, bool) or not isinstance(value, int) or value <= 0:+ raise ConfigurationError(f"{path}.{key} must be a positive integer")+ return value+++def _wide(+ wide: Any, modalities: tuple[str, ...], path: str+) -> tuple[tuple[str, ...], float | None]:+ if wide is None:+ return (), None+ if not isinstance(wide, Mapping):+ raise ConfigurationError(f"{path}.wide must be a mapping")+ unknown = sorted(set(wide) - _ALLOWED_WIDE_KEYS)+ if unknown:+ raise ConfigurationError(f"{path}.wide has unsupported keys {unknown}")+ if not isinstance(wide.get("enabled"), bool):+ raise ConfigurationError(f"{path}.wide.enabled must be true or false")+ if not wide["enabled"]:+ return (), None+ wide_modalities = _canonical_modalities(wide.get("modalities"), f"{path}.wide.modalities")+ if H3_OBSERVATION_INPUT in wide_modalities:+ raise ConfigurationError(f"{path}.wide: the observation input has no wide render")+ missing = sorted(set(wide_modalities) - set(modalities))+ if missing:+ raise ConfigurationError(+ f"{path}.wide.modalities {missing} must be a subset of {path}.modalities: "+ "the wide render is peripheral context for a modality the trunk reads"+ )+ scale = wide.get("fov_scale")+ if (+ isinstance(scale, bool)+ or not isinstance(scale, (int, float))+ or not math.isfinite(float(scale))+ or float(scale) <= 0.0+ ):+ raise ConfigurationError(f"{path}.wide.fov_scale must be a positive number")+ return wide_modalities, float(scale)+++def geometry_renders_from_preencode_config(+ preencode: Mapping[str, Any],+) -> H3GeometryRenders | None:+ """Renders a raw preencode should encode next to the RGB target, if any."""++ path = "preencode.geometry"+ block = preencode.get("geometry")+ if block is None:+ return None+ if not isinstance(block, Mapping):+ raise ConfigurationError(f"{path} must be a mapping")+ unknown = sorted(set(block) - {"modalities", "wide"})+ if unknown:+ raise ConfigurationError(f"{path} has unsupported keys {unknown}")+ modalities = _canonical_modalities(block.get("modalities"), f"{path}.modalities")+ unsupported = sorted(set(modalities) - set(H3_RAW_GEOMETRY_MODALITIES))+ if unsupported:+ raise ConfigurationError(+ f"{path}.modalities {unsupported}: raw preencode defines the pixel encoding of "+ f"{list(H3_RAW_GEOMETRY_MODALITIES)} renders only"+ )+ wide_modalities, wide_fov_scale = _wide(block.get("wide"), modalities, path)+ return H3GeometryRenders(modalities, wide_modalities, wide_fov_scale)+++def geometry_member_field(tensor_key: str) -> str:+ """Raw index field naming the render member for one artifact tensor."""++ if not tensor_key.startswith("geometry_") or not tensor_key.endswith("_latents"):+ raise ValueError(f"not an H3 geometry tensor key: {tensor_key!r}")+ return f"{tensor_key.removesuffix('_latents')}_member"+++def geometry_profile_from_model_config(model: Mapping[str, Any]) -> H3GeometryProfile | None:+ """Return the enabled geometry profile, ``None`` when the block is absent or off."""++ path = "model.geometry_controlnet"+ block = model.get("geometry_controlnet")+ if block is None:+ return None+ if not isinstance(block, Mapping):+ raise ConfigurationError(f"{path} must be a mapping")+ unknown = sorted(set(block) - _ALLOWED_KEYS)+ if unknown:+ raise ConfigurationError(f"{path} has unsupported keys {unknown}")+ enabled = block.get("enabled")+ if not isinstance(enabled, bool):+ raise ConfigurationError(f"{path}.enabled must be true or false")+ if not enabled:+ return None+ modalities = _canonical_modalities(block.get("modalities"), f"{path}.modalities")+ layer_stride = _positive(block, "layer_stride", path)+ if layer_stride > H3_BACKBONE_LAYERS:+ raise ConfigurationError(f"{path}.layer_stride must be at most {H3_BACKBONE_LAYERS}")+ dim = _positive(block, "dim", path)+ ffn_dim = _positive(block, "ffn_dim", path)+ num_heads = _positive(block, "num_heads", path)+ temb_dim = _positive(block, "temb_dim", path)+ if dim % num_heads:+ raise ConfigurationError(f"{path}.dim={dim} must be divisible by num_heads={num_heads}")+ if dim // num_heads < H3_ROTARY_CHANNELS:+ raise ConfigurationError(+ f"{path}: dim / num_heads = {dim // num_heads} is below the "+ f"{H3_ROTARY_CHANNELS} channels H3's rotary embedding rotates per head"+ )+ lora_trainable = block.get("lora_trainable")+ if not isinstance(lora_trainable, bool):+ raise ConfigurationError(f"{path}.lora_trainable must be true or false")+ wide_modalities, wide_fov_scale = _wide(block.get("wide"), modalities, path)+ return H3GeometryProfile(+ modalities=modalities,+ layer_stride=layer_stride,+ dim=dim,+ ffn_dim=ffn_dim,+ num_heads=num_heads,+ temb_dim=temb_dim,+ wide_modalities=wide_modalities,+ wide_fov_scale=wide_fov_scale,+ lora_trainable=lora_trainable,+ )+++def validate_geometry_parallelism(profile: H3GeometryProfile, sequence_parallel_size: int) -> None:+ """The Ulysses exchange scatters ControlNet heads just like backbone heads."""++ if profile.num_heads % int(sequence_parallel_size):+ raise ConfigurationError(+ f"model.geometry_controlnet.num_heads={profile.num_heads} must be divisible by "+ f"distributed.sequence_parallel_size={sequence_parallel_size}"+ )+++def geometry_checkpoint_extras(profile: H3GeometryProfile | None) -> dict[str, Any]:+ """Checkpoint-contract extras; empty for runs without the ControlNet."""++ if profile is None:+ return {}+ return {+ "geometry_controlnet": {+ "architecture": profile.architecture(),+ "lora_trainable": profile.lora_trainable,+ }+ }+++__all__ = [+ "H3_GEOMETRY_MODALITIES",+ "H3_GEOMETRY_MODULE",+ "H3_GEOMETRY_SCHEMA",+ "H3_GEOMETRY_STATE_PREFIX",+ "H3_OBSERVATION_INPUT",+ "H3_RAW_GEOMETRY_MODALITIES",+ "H3_ROTARY_CHANNELS",+ "H3GeometryProfile",+ "H3GeometryRenders",+ "geometry_checkpoint_extras",+ "geometry_member_field",+ "geometry_profile_from_model_config",+ "geometry_renders_from_preencode_config",+ "geometry_tensor_key",+ "validate_geometry_parallelism",+]diff --git a/src/solarwm/backends/minimax_h3/demo_conditions.py b/src/solarwm/backends/minimax_h3/demo_conditions.pynew file mode 100644index 0000000..60c1da3--- /dev/null+++ b/src/solarwm/backends/minimax_h3/demo_conditions.py@@ -0,0 +1,140 @@+"""First-image and supplied full camera tracks for the public demo collection."""++from __future__ import annotations++import hashlib+import json+import shutil+from pathlib import Path++from .source_length_cache import atomic_json, file_sha256, preparation_identity+++def source_digest(case):+ return hashlib.sha256(+ json.dumps(+ {+ name: file_sha256(case / name)+ for name in ("source.image.png", "source.camera.npy", "source.prompt.txt")+ },+ sort_keys=True,+ ).encode()+ ).hexdigest()+++def stage_source(row, args):+ root = Path(args.dataset_root).resolve()+ case = Path(args.work_dir) / "cases" / row["key"]+ case.mkdir(parents=True, exist_ok=True)+ for field, name in [+ ("image", "source.image.png"),+ ("camera", "source.camera.npy"),+ ("prompt", "source.prompt.txt"),+ ]:+ source = (root / row[field]).resolve()+ if not source.is_relative_to(root) or file_sha256(source) != row[field + "_sha256"]:+ raise ValueError("Demo source identity changed")+ target = case / name+ if target.exists():+ if file_sha256(target) != row[field + "_sha256"]:+ raise ValueError("Demo cache contains another input")+ else:+ shutil.copyfile(source, target)+ return case+++def prepare_demo(row, args, components, device):+ import time+ from dataclasses import asdict++ import numpy as np+ import torch+ from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition+ from PIL import Image+ from safetensors.torch import save_file++ from .full_length import full_length_geometry+ from .official_codec import encode_joint_prompt_condition+ from .torch_prope import _invert_se3++ started = time.perf_counter()+ if args.config["model"]["camera_intrinsics_mode"] != "wan_fixed":+ raise ValueError("Image/camera demo without intrinsic sidecars requires wan_fixed")+ case = stage_source(row, args)+ c2w = np.load(case / "source.camera.npy", allow_pickle=False)+ frames = int(row["num_frames"])+ if c2w.dtype != np.float64 or c2w.shape != (frames, 4, 4) or not np.isfinite(c2w).all():+ raise ValueError("Demo needs its complete authoritative float64 C2W trajectory")+ np.testing.assert_allclose(c2w[:, 3], np.broadcast_to([0, 0, 0, 1], (frames, 4)), atol=1e-5)+ image = (+ Image.open(case / "source.image.png")+ .convert("RGB")+ .resize((1344, 768), Image.Resampling.LANCZOS)+ )+ image.save(case / "first.png")+ poses = torch.from_numpy(c2w).float()+ relative = _invert_se3(poses[:1]) @ poses+ relative[0] = torch.eye(4)+ views = _invert_se3(relative).contiguous()+ geometry = full_length_geometry(frames)+ idx = torch.tensor(geometry.camera_indices)+ # PRoPE replaces this placeholder with the trained fixed Wan intrinsics.+ K = (+ torch.tensor([[1.0, 0.0, 0.5], [0.0, 1.0, 0.5], [0.0, 0.0, 1.0]])+ .expand(len(idx), -1, -1)+ .contiguous()+ )+ caption = (case / "source.prompt.txt").read_text().strip()+ if caption != row["caption"].strip() or not caption:+ raise ValueError("Demo caption changed")+ with torch.inference_mode():+ prompt, tags = encode_joint_prompt_condition(+ image,+ caption,+ processor=components.processor,+ tokenizer=components.tokenizer,+ text_encoder=components.text_encoder,+ device=device,+ )+ pixels = (+ torch.from_numpy(np.asarray(image).copy()).permute(2, 0, 1)[None, :, None].to(device)+ )+ anchor = encode_vae_condition(+ components.video_vae,+ pixels,+ (0.485, 0.456, 0.406),+ (0.229, 0.224, 0.225),+ encode_seed=42,+ )+ values = dict(+ prompt_embeds=prompt.cpu().contiguous(),+ text_token_tags=tags.cpu().contiguous(),+ anchor_latents=anchor.cpu().contiguous(),+ viewmats=views[idx].contiguous(),+ K=K,+ )+ if not all(bool(torch.isfinite(value).all()) for value in values.values()):+ raise ValueError("Non-finite demo condition")+ target = case / "condition.safetensors"+ save_file(values, str(target.with_suffix(".tmp")))+ target.with_suffix(".tmp").replace(target)+ torch.cuda.synchronize()+ atomic_json(+ case / "READY.json",+ dict(+ schema="solarwm.h3-full-condition.v1",+ row=row,+ preparation_identity=preparation_identity(row, args),+ geometry=asdict(geometry),+ condition_sha256=file_sha256(target),+ source_sha256=source_digest(case),+ preparation_seconds=time.perf_counter() - started,+ source_fps=float(row["fps"]),+ image_resize="PIL_LANCZOS_stretch_1344x768",+ model_fps=24,+ camera_convention="relative_w2c+wan_fixed_K",+ source_frame_policy="all_supplied_camera_rows_no_resampling",+ ground_truth_video_required=False,+ ),+ )+ print(f"DEMO_PREPARED key={row['key']} frames={frames}", flush=True)diff --git a/src/solarwm/backends/minimax_h3/diffusers_compat.py b/src/solarwm/backends/minimax_h3/diffusers_compat.pynew file mode 100644index 0000000..8cafd2b--- /dev/null+++ b/src/solarwm/backends/minimax_h3/diffusers_compat.py@@ -0,0 +1,59 @@+"""Narrow compatibility patch for the H3 Diffusers build."""++from __future__ import annotations++from collections.abc import Callable+from typing import Any++_KNOWN_ERRORS = (+ "local variable 'tuple' referenced before assignment",+ "cannot access local variable 'tuple' where it is not associated with a value",+)+++def _tensor_attribute_dtype(module: Any) -> Any:+ import torch++ def tensor_attributes(owner: Any) -> list[tuple[str, Any]]:+ return [(name, value) for name, value in owner.__dict__.items() if torch.is_tensor(value)]++ last = None+ for _name, tensor in module._named_members(get_members_fn=tensor_attributes):+ last = tensor.dtype+ if tensor.is_floating_point():+ return tensor.dtype+ return last+++def patch_minimax_h3_parameter_dtype() -> Callable[[Any], Any]:+ """Patch the Python-3.10 local-``tuple`` FSDP failure.++ FSDP temporarily exposes leaf parameters as plain tensor attributes. The+ H3 Diffusers implementation reaches a fallback whose annotation shadows+ ``tuple`` on Python 3.10. No other exception or normal code path is hidden.+ """++ import diffusers.models.transformers.transformer_minimax_h3 as module++ current = module.get_parameter_dtype+ if getattr(current, "_solarwm_h3_tuple_compat", False):+ return current++ def compatible(owner: Any) -> Any:+ try:+ return current(owner)+ except UnboundLocalError as exc:+ if not any(message in str(exc) for message in _KNOWN_ERRORS):+ raise+ if any(owner.named_parameters()) or any(owner.buffers()):+ raise+ return _tensor_attribute_dtype(owner)++ compatible.__name__ = current.__name__+ compatible.__doc__ = current.__doc__+ compatible._solarwm_h3_tuple_compat = True # type: ignore[attr-defined]+ module.get_parameter_dtype = compatible+ return compatible+++__all__ = ["patch_minimax_h3_parameter_dtype"]diff --git a/src/solarwm/backends/minimax_h3/distributed.py b/src/solarwm/backends/minimax_h3/distributed.pynew file mode 100644index 0000000..fbb8829--- /dev/null+++ b/src/solarwm/backends/minimax_h3/distributed.py@@ -0,0 +1,612 @@+"""MiniMax-H3 packed sequence parallelism.++The public topology helpers stay importable without PyTorch. CUDA and+``torch.distributed`` are resolved only when a process group or collective is+actually requested. H3 uses contiguous SP groups and strided logical-DP+groups::++ global_rank = dp_rank * sp_size + sp_rank++Stage0.5 shards the *already packed* heterogeneous sequence. Uneven caption+lengths are supported without attention-visible padding.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any+++@dataclass+class H3DistributedState:+ initialized: bool = False+ sp_size: int = 1+ sp_rank: int = 0+ dp_rank: int = 0+ dp_world_size: int = 1+ sp_group: Any = None+ dp_group: Any = None+ sp_group_ranks: tuple[int, ...] = (0,)+ dp_group_ranks: tuple[int, ...] = (0,)+ sequence_lengths: tuple[int, ...] = ()+++_STATE = H3DistributedState()+++def contiguous_packed_bounds(total_tokens: int, *, sp_size: int, sp_rank: int) -> tuple[int, int]:+ """Return the balanced contiguous ``[start, stop)`` shard for one rank."""++ for name, value in (("total_tokens", total_tokens), ("sp_size", sp_size)):+ if isinstance(value, bool) or not isinstance(value, int):+ raise TypeError(f"{name} must be an integer")+ if value <= 0:+ raise ValueError(f"{name} must be positive")+ if isinstance(sp_rank, bool) or not isinstance(sp_rank, int):+ raise TypeError("sp_rank must be an integer")+ if not 0 <= sp_rank < sp_size:+ raise ValueError(f"sp_rank must be in [0,{sp_size})")+ if total_tokens < sp_size:+ raise ValueError("every H3 SP rank must own at least one packed row")+ base, remainder = divmod(total_tokens, sp_size)+ count = base + int(sp_rank < remainder)+ start = sp_rank * base + min(sp_rank, remainder)+ return start, start + count+++def logical_node_data_world_size(*, world_size: int, local_world_size: int, sp_size: int) -> int:+ """Return logical readers per node and prove SP groups cannot cross nodes."""++ for name, value in {+ "world_size": world_size,+ "local_world_size": local_world_size,+ "sp_size": sp_size,+ }.items():+ if isinstance(value, bool) or not isinstance(value, int):+ raise TypeError(f"{name} must be an integer")+ if value <= 0:+ raise ValueError(f"{name} must be positive")+ if world_size % local_world_size:+ raise ValueError("world_size must be divisible by LOCAL_WORLD_SIZE")+ if world_size % sp_size or local_world_size % sp_size:+ raise ValueError("global and node-local worlds must be divisible by SP size")+ return local_world_size // sp_size+++def init_sequence_parallel(sp_size: int = 1) -> H3DistributedState:+ """Create H3 SP and logical-DP groups after process-group initialization."""++ global _STATE+ import torch.distributed as dist++ sp_size = int(sp_size)+ if sp_size < 1:+ raise ValueError("sp_size must be positive")+ if not dist.is_available() or not dist.is_initialized():+ if sp_size != 1:+ raise RuntimeError("SP2 requires initialized torch.distributed")+ _STATE = H3DistributedState(initialized=True)+ return _STATE++ rank = dist.get_rank()+ world = dist.get_world_size()+ if world % sp_size:+ raise ValueError(f"world_size={world} is not divisible by sp_size={sp_size}")+ dp_world = world // sp_size+ sp_group = None+ dp_group = None+ sp_ranks: tuple[int, ...] = ()+ dp_ranks: tuple[int, ...] = ()+ # Every process creates groups in exactly the same order.+ for dp_index in range(dp_world):+ ranks = tuple(range(dp_index * sp_size, (dp_index + 1) * sp_size))+ group = dist.new_group(ranks=list(ranks))+ if rank in ranks:+ sp_group, sp_ranks = group, ranks+ for sp_index in range(sp_size):+ ranks = tuple(range(sp_index, world, sp_size))+ group = dist.new_group(ranks=list(ranks))+ if rank in ranks:+ dp_group, dp_ranks = group, ranks+ if sp_size > 1 and (sp_group is None or dp_group is None):+ raise RuntimeError("failed to construct H3 distributed groups")+ _STATE = H3DistributedState(+ initialized=True,+ sp_size=sp_size,+ sp_rank=rank % sp_size,+ dp_rank=rank // sp_size,+ dp_world_size=dp_world,+ sp_group=sp_group,+ dp_group=dp_group,+ sp_group_ranks=sp_ranks or (rank,),+ dp_group_ranks=dp_ranks or tuple(range(world)),+ )+ return _STATE+++def state() -> H3DistributedState:+ return _STATE if _STATE.initialized else H3DistributedState(initialized=True)+++def get_sp_size() -> int:+ return state().sp_size+++def get_sp_rank() -> int:+ return state().sp_rank+++def get_dp_rank() -> int:+ return state().dp_rank+++def get_dp_world_size() -> int:+ return state().dp_world_size+++def get_sp_group() -> Any:+ return state().sp_group+++def get_dp_group() -> Any:+ return state().dp_group+++def is_sequence_parallel_enabled() -> bool:+ return get_sp_size() > 1+++def register_sequence_length(local_length: int) -> tuple[int, ...]:+ import torch+ import torch.distributed as dist++ current = state()+ local_length = int(local_length)+ if local_length < 0:+ raise ValueError("local sequence length must be non-negative")+ if current.sp_size == 1:+ current.sequence_lengths = (local_length,)+ return current.sequence_lengths+ value = torch.tensor([local_length], dtype=torch.int64, device="cuda")+ values = [torch.empty_like(value) for _ in range(current.sp_size)]+ dist.all_gather(values, value, group=current.sp_group)+ current.sequence_lengths = tuple(int(item.item()) for item in values)+ return current.sequence_lengths+++def broadcast_sp_tensor(tensor: Any, src_sp_rank: int = 0) -> Any:+ import torch.distributed as dist++ current = state()+ if current.sp_size == 1:+ return tensor+ if not 0 <= src_sp_rank < current.sp_size:+ raise ValueError("src_sp_rank is outside the SP group")+ source = current.sp_group_ranks[src_sp_rank]+ if tensor.is_contiguous():+ dist.broadcast(tensor, src=source, group=current.sp_group)+ else:+ staging = tensor.contiguous()+ dist.broadcast(staging, src=source, group=current.sp_group)+ tensor.copy_(staging)+ return tensor+++def _all_gather_apply(input_: Any, dim: int) -> Any:+ import torch+ import torch.distributed as dist++ current = state()++ class _AllGather(torch.autograd.Function):+ @staticmethod+ def forward(ctx: Any, value: Any) -> Any:+ normalized_dim = dim if dim >= 0 else value.dim() + dim+ ctx.dim = normalized_dim+ ctx.input_shape = value.shape+ ctx.lengths = current.sequence_lengths+ ctx.sp_rank = current.sp_rank+ variable = (+ normalized_dim == 1+ and len(ctx.lengths) == current.sp_size+ and value.size(normalized_dim) == ctx.lengths[current.sp_rank]+ and len(set(ctx.lengths)) > 1+ )+ ctx.variable = variable+ if variable:+ maximum = max(ctx.lengths)+ shape = list(value.shape)+ shape[normalized_dim] = maximum+ padded = value.new_zeros(shape)+ padded.narrow(normalized_dim, 0, value.size(normalized_dim)).copy_(value)+ gathered = [torch.empty_like(padded) for _ in range(current.sp_size)]+ dist.all_gather(gathered, padded.contiguous(), group=current.sp_group)+ return torch.cat(+ [+ item.narrow(normalized_dim, 0, length)+ for item, length in zip(gathered, ctx.lengths, strict=True)+ ],+ dim=normalized_dim,+ )+ input_shape = value.size()+ output_shape = (input_shape[0] * current.sp_size, *input_shape[1:])+ output = torch.empty(output_shape, dtype=value.dtype, device=value.device)+ dist.all_gather_into_tensor(output, value.contiguous(), group=current.sp_group)+ output = output.reshape((current.sp_size, *input_shape))+ output = output.movedim(0, normalized_dim)+ return output.reshape(+ (+ *input_shape[:normalized_dim],+ current.sp_size * input_shape[normalized_dim],+ *input_shape[normalized_dim + 1 :],+ )+ )++ @staticmethod+ def backward(ctx: Any, gradient: Any) -> tuple[Any]:+ if ctx.variable:+ # Uneven documents cannot use fixed-size reduce-scatter.+ reduced = gradient.contiguous()+ dist.all_reduce(reduced, group=current.sp_group)+ start = sum(ctx.lengths[: ctx.sp_rank])+ length = ctx.lengths[ctx.sp_rank]+ return (reduced.narrow(ctx.dim, start, length).contiguous(),)+ length = gradient.size(ctx.dim) // current.sp_size+ chunks = gradient.reshape(+ (+ *gradient.shape[: ctx.dim],+ current.sp_size,+ length,+ *gradient.shape[ctx.dim + 1 :],+ )+ )+ chunks = chunks.movedim(ctx.dim, 0).contiguous()+ output = torch.empty(+ ctx.input_shape,+ dtype=gradient.dtype,+ device=gradient.device,+ )+ dist.reduce_scatter_tensor(output, chunks, group=current.sp_group)+ return (output,)++ return _AllGather.apply(input_)+++def sequence_all_gather(input_: Any, dim: int = 1) -> Any:+ if get_sp_size() == 1:+ return input_+ return _all_gather_apply(input_, dim)+++def _all_to_all_raw(input_: Any, scatter_dim: int, gather_dim: int) -> Any:+ import torch+ import torch.distributed as dist++ current = state()+ world = current.sp_size+ if world == 1:+ return input_+ if input_.dim() != 4:+ raise RuntimeError("H3 Ulysses all-to-all requires [B,S,H,D]")+ lengths = current.sequence_lengths+ if len(lengths) != world:+ raise RuntimeError("SP sequence lengths were not registered")++ if scatter_dim == 2 and gather_dim == 1:+ batch, local_sequence, heads, head_dim = input_.shape+ if heads % world:+ raise RuntimeError("attention heads must be divisible by SP size")+ if local_sequence != lengths[current.sp_rank]:+ raise RuntimeError("local sequence length differs from registered SP layout")+ local_heads = heads // world+ transposed = input_.transpose(0, 2).contiguous()+ if len(set(lengths)) == 1:+ output = torch.empty_like(transposed)+ dist.all_to_all_single(output, transposed, group=current.sp_group)+ output = torch.cat(output.split(local_heads), dim=1)+ return output.transpose(0, 2).contiguous()+ factor = local_heads * batch * head_dim+ input_splits = [factor * local_sequence] * world+ output_splits = [factor * length for length in lengths]+ output = torch.empty(sum(output_splits), dtype=input_.dtype, device=input_.device)+ dist.all_to_all_single(+ output,+ transposed.reshape(-1),+ output_split_sizes=output_splits,+ input_split_sizes=input_splits,+ group=current.sp_group,+ )+ chunks = []+ offset = 0+ for length, size in zip(lengths, output_splits, strict=True):+ chunks.append(+ output.narrow(0, offset, size).reshape(local_heads, length, batch, head_dim)+ )+ offset += size+ return torch.cat(chunks, dim=1).transpose(0, 2).contiguous()++ if scatter_dim == 1 and gather_dim == 2:+ batch, sequence, local_heads, head_dim = input_.shape+ if sequence != sum(lengths):+ raise RuntimeError("full sequence length differs from registered SP layout")+ local_sequence = lengths[current.sp_rank]+ transposed = input_.transpose(0, 2).contiguous()+ if len(set(lengths)) == 1:+ transposed = (+ transposed.reshape(local_heads, world, local_sequence, batch, head_dim)+ .transpose(0, 1)+ .reshape(local_heads * world, local_sequence, batch, head_dim)+ .contiguous()+ )+ output = torch.empty_like(transposed)+ dist.all_to_all_single(output, transposed, group=current.sp_group)+ return output.transpose(0, 2).contiguous()+ starts = [sum(lengths[:rank]) for rank in range(world)]+ pieces = [+ transposed.narrow(1, start, length).contiguous().reshape(-1)+ for start, length in zip(starts, lengths, strict=True)+ ]+ factor = local_heads * batch * head_dim+ input_splits = [factor * length for length in lengths]+ output_splits = [factor * local_sequence] * world+ output = torch.empty(sum(output_splits), dtype=input_.dtype, device=input_.device)+ dist.all_to_all_single(+ output,+ torch.cat(pieces),+ output_split_sizes=output_splits,+ input_split_sizes=input_splits,+ group=current.sp_group,+ )+ chunks = []+ offset = 0+ for size in output_splits:+ chunks.append(+ output.narrow(0, offset, size).reshape(local_heads, local_sequence, batch, head_dim)+ )+ offset += size+ return torch.cat(chunks, dim=0).transpose(0, 2).contiguous()+ raise RuntimeError(f"unsupported H3 all-to-all dimensions {scatter_dim}->{gather_dim}")+++def sequence_all_to_all(input_: Any, scatter_dim: int = 2, gather_dim: int = 1) -> Any:+ if get_sp_size() == 1:+ return input_+ import torch++ class _AllToAll(torch.autograd.Function):+ @staticmethod+ def forward(ctx: Any, value: Any) -> Any:+ ctx.scatter_dim = scatter_dim+ ctx.gather_dim = gather_dim+ return _all_to_all_raw(value, scatter_dim, gather_dim)++ @staticmethod+ def backward(ctx: Any, gradient: Any) -> tuple[Any]:+ return (_all_to_all_raw(gradient, ctx.gather_dim, ctx.scatter_dim),)++ return _AllToAll.apply(input_)+++def _sync_gradient_bucket(parameters: tuple[Any, ...], *, group: Any, average: bool) -> None:+ import os++ import torch+ import torch.distributed as dist++ if not parameters:+ return+ gradients = [parameter.grad for parameter in parameters if parameter.grad is not None]+ flag_device = gradients[0].device if gradients else parameters[0].device+ flags = torch.tensor(+ [int(parameter.grad is not None) for parameter in parameters],+ device=flag_device,+ dtype=torch.int32,+ )+ group_size = dist.get_world_size(group=group)+ dist.all_reduce(flags, group=group)+ if bool(((flags != 0) & (flags != group_size)).any().item()):+ raise RuntimeError("H3 distributed ranks have different gradient patterns")+ if not gradients:+ return+ bucket_limit = max(+ 1,+ int(os.environ.get("SOLARWM_H3_GRAD_BUCKET_MB", "16")),+ ) * (1 << 20)+ buckets: list[list[Any]] = []+ current_bucket: list[Any] = []+ current_bytes = 0+ current_key: tuple[Any, Any] | None = None+ for gradient in gradients:+ key = (gradient.device, gradient.dtype)+ size = int(gradient.numel()) * int(gradient.element_size())+ if current_bucket and (key != current_key or current_bytes + size > bucket_limit):+ buckets.append(current_bucket)+ current_bucket = []+ current_bytes = 0+ current_bucket.append(gradient)+ current_bytes += size+ current_key = key+ if current_bucket:+ buckets.append(current_bucket)+ for bucket in buckets:+ # Match FSDP's configured FP32 reduction. LoRA parameters are BF16+ # and deliberately ignored by FSDP, so a native-dtype all-reduce here+ # would silently lower both SP-sum and logical-DP-average precision.+ flat = torch.cat([gradient.detach().reshape(-1).float() for gradient in bucket])+ dist.all_reduce(flat, group=group)+ if average:+ flat.div_(group_size)+ offset = 0+ for gradient in bucket:+ count = gradient.numel()+ gradient.copy_(flat[offset : offset + count].view_as(gradient))+ offset += count+++def sync_lora_gradients(parameters: tuple[Any, ...]) -> None:+ """Sum complementary SP work, then average replicated LoRA over logical DP."""++ current = state()+ if current.sp_size > 1:+ _sync_gradient_bucket(parameters, group=current.sp_group, average=False)+ if current.dp_world_size > 1:+ _sync_gradient_bucket(parameters, group=current.dp_group, average=True)+++@dataclass(frozen=True)+class H3PackedSequenceShard:+ hidden_states: Any+ position_ids: Any+ token_tags: Any+ timestep_indices: Any+ prope_token_indices: Any+ prope_frame_ids: Any+ camera_viewmats: Any+ camera_K: Any+ start: int+ stop: int+ total_tokens: int+ sequence_lengths: tuple[int, ...]+++def shard_stage0p5_packed_sequence(+ *,+ hidden_states: Any,+ position_ids: Any,+ token_tags: Any,+ timestep_indices: Any,+ prope_token_indices: Any,+ prope_frame_ids: Any,+ camera_viewmats: Any,+ camera_K: Any,+) -> H3PackedSequenceShard:+ """Shard the full packed document and localize camera row controls."""++ import torch++ if not isinstance(hidden_states, torch.Tensor) or hidden_states.ndim != 3:+ raise ValueError("hidden_states must be [B,S,D]")+ total = int(hidden_states.shape[1])+ for name, value in (+ ("position_ids", position_ids),+ ("token_tags", token_tags),+ ("timestep_indices", timestep_indices),+ ):+ if not isinstance(value, torch.Tensor) or int(value.shape[0]) != total:+ raise ValueError(f"{name} must have one row per packed token")+ start, stop = contiguous_packed_bounds(total, sp_size=get_sp_size(), sp_rank=get_sp_rank())+ lengths = tuple(+ contiguous_packed_bounds(total, sp_size=get_sp_size(), sp_rank=rank)[1]+ - contiguous_packed_bounds(total, sp_size=get_sp_size(), sp_rank=rank)[0]+ for rank in range(get_sp_size())+ )+ if register_sequence_length(stop - start) != lengths:+ raise RuntimeError("SP ranks disagree on packed sequence lengths")++ controls = (prope_token_indices, prope_frame_ids, camera_viewmats, camera_K)+ if all(value is None for value in controls):+ local_indices = local_frames = local_views = local_K = None+ else:+ if prope_token_indices is None or camera_viewmats is None or camera_K is None:+ raise ValueError("camera PRoPE requires indices, viewmats, and K")+ indices = prope_token_indices.to(dtype=torch.long)+ if indices.ndim != 1 or (+ indices.numel() and (int(indices.min()) < 0 or int(indices.max()) >= total)+ ):+ raise ValueError("camera PRoPE indices are outside the packed sequence")+ selected = (indices >= start) & (indices < stop)+ local_indices = (indices[selected] - start).contiguous()+ if prope_frame_ids is not None:+ if tuple(prope_frame_ids.shape) != tuple(indices.shape):+ raise ValueError("camera frame IDs must align with PRoPE row indices")+ local_frames = prope_frame_ids[selected].contiguous()+ local_views, local_K = camera_viewmats, camera_K+ else:+ if camera_viewmats.shape[1] != indices.numel() or camera_K.shape[1] != indices.numel():+ raise ValueError("token-aligned camera tensors must align with PRoPE indices")+ local_frames = None+ local_views = camera_viewmats[:, selected].contiguous()+ local_K = camera_K[:, selected].contiguous()++ return H3PackedSequenceShard(+ hidden_states=hidden_states[:, start:stop].contiguous(),+ position_ids=position_ids[start:stop].contiguous(),+ token_tags=token_tags[start:stop].contiguous(),+ timestep_indices=timestep_indices[start:stop].contiguous(),+ prope_token_indices=local_indices,+ prope_frame_ids=local_frames,+ camera_viewmats=local_views,+ camera_K=local_K,+ start=start,+ stop=stop,+ total_tokens=total,+ sequence_lengths=lengths,+ )+++__all__ = [+ "H3DistributedState",+ "H3PackedSequenceShard",+ "broadcast_sp_tensor",+ "contiguous_packed_bounds",+ "get_dp_group",+ "get_dp_rank",+ "get_dp_world_size",+ "get_sp_group",+ "get_sp_rank",+ "get_sp_size",+ "init_sequence_parallel",+ "is_sequence_parallel_enabled",+ "logical_node_data_world_size",+ "register_sequence_length",+ "sequence_all_gather",+ "sequence_all_to_all",+ "shard_stage0p5_packed_sequence",+ "state",+ "sync_lora_gradients",+]+++def prepare_flex_attention_input(tensor: Any) -> Any:+ """Convert BSHD to contiguous BHSD without changing values or autograd.++ Ulysses returns sequence-major storage. PyTorch 2.6 Flex cannot order its+ symbolic transposed strides; the head-major layout matches native H3 SP1.+ An already head-major input view needs no copy.+ """++ return tensor.transpose(1, 2).contiguous()+++def validate_packed_attention_mask(attention_mask: Any, *, total_tokens: int) -> None:+ """Require a global, head-broadcast Flex mask before Ulysses sharding.++ Ulysses restores the full packed sequence while distributing heads. H3's+ W6 visibility is head-independent, so the original global mask is reused+ on every SP rank; neither its rows nor its head dimension may be sharded.+ """++ if attention_mask is None:+ return+ if type(attention_mask).__name__ == "H3SegmentPlan":+ # A segment plan addresses the global document after the Ulysses exchange, like a global BlockMask.+ if int(attention_mask.sequence_length) != int(total_tokens):+ raise ValueError("H3 packed SP segment plan must cover the complete global sequence")+ return+ if type(attention_mask).__name__ != "BlockMask":+ raise ValueError("H3 packed SP accepts only a global Flex BlockMask or no mask")+ if tuple(attention_mask.seq_lengths) != (total_tokens, total_tokens):+ raise ValueError("H3 packed SP BlockMask must cover the complete global sequence")+ for name in (+ "kv_num_blocks",+ "kv_indices",+ "full_kv_num_blocks",+ "full_kv_indices",+ "q_num_blocks",+ "q_indices",+ "full_q_num_blocks",+ "full_q_indices",+ ):+ metadata = getattr(attention_mask, name, None)+ if metadata is not None and (metadata.ndim < 2 or metadata.shape[1] != 1):+ raise ValueError("H3 packed SP requires head-broadcast BlockMask metadata")diff --git a/src/solarwm/backends/minimax_h3/ema.py b/src/solarwm/backends/minimax_h3/ema.pynew file mode 100644index 0000000..eb08b2c--- /dev/null+++ b/src/solarwm/backends/minimax_h3/ema.py@@ -0,0 +1,117 @@+"""Rank-local FP32 EMA aligned with H3 FSDP/LoRA parameter ownership."""++from __future__ import annotations++from collections.abc import Iterator+from contextlib import contextmanager+from typing import Any++import torch+++def _named_parameters(module: Any, *, trainable_only: bool) -> list[tuple[str, Any]]:+ root = module.module if hasattr(module, "module") else module+ return [+ (name, parameter)+ for name, parameter in root.named_parameters()+ if not trainable_only or parameter.requires_grad+ ]+++class H3ShardedEMA:+ """FP32 EMA over each rank's ``use_orig_params`` parameter view.++ Updates require no full-state gather. Ignored adapter parameters are+ replicated and identically updated; FSDP-owned+ parameters would retain their local shard shape.+ """++ def __init__(+ self,+ module: Any,+ *,+ decay: float = 0.9999,+ device: str | torch.device = "cuda",+ trainable_only: bool = True,+ ) -> None:+ self.decay = float(decay)+ if not 0 <= self.decay <= 1:+ raise ValueError("EMA decay must be in [0,1]")+ self.device = torch.device(device)+ self.trainable_only = bool(trainable_only)+ self.num_updates = 0+ self.shadow: dict[str, torch.Tensor] = {}+ self.reset_from(module)++ @property+ def initialized(self) -> bool:+ return bool(self.shadow)++ def _validate(self, module: Any) -> list[tuple[str, Any]]:+ named = _named_parameters(module, trainable_only=self.trainable_only)+ if {name for name, _ in named} != set(self.shadow):+ raise RuntimeError("EMA parameter names differ from the live model")+ for name, parameter in named:+ shadow = self.shadow[name]+ if tuple(shadow.shape) != tuple(parameter.shape) or shadow.dtype != torch.float32:+ raise RuntimeError(f"EMA shard layout differs for {name!r}")+ return named++ @torch.no_grad()+ def reset_from(self, module: Any) -> None:+ self.shadow = {+ name: parameter.detach().to(device=self.device, dtype=torch.float32).clone()+ for name, parameter in _named_parameters(module, trainable_only=self.trainable_only)+ }+ self.num_updates = 0++ @torch.no_grad()+ def update(self, module: Any) -> None:+ for name, parameter in self._validate(module):+ value = parameter.detach().to(device=self.shadow[name].device, dtype=torch.float32)+ self.shadow[name].mul_(self.decay).add_(value, alpha=1.0 - self.decay)+ self.num_updates += 1++ def state_dict(self) -> dict[str, Any]:+ return {+ "schema": "solarwm.minimax-h3-ema.v1",+ "decay": self.decay,+ "num_updates": self.num_updates,+ "trainable_only": self.trainable_only,+ "shadow": {name: value.detach().cpu() for name, value in self.shadow.items()},+ }++ def load_state_dict(self, values: dict[str, Any], module: Any) -> None:+ if values.get("schema") != "solarwm.minimax-h3-ema.v1":+ raise ValueError("unsupported H3 EMA checkpoint schema")+ if bool(values.get("trainable_only")) != self.trainable_only:+ raise ValueError("EMA trainable-only policy differs")+ shadow = values.get("shadow")+ if not isinstance(shadow, dict):+ raise ValueError("EMA checkpoint lacks shadow tensors")+ self.shadow = {+ name: value.detach().to(device=self.device, dtype=torch.float32).clone()+ for name, value in shadow.items()+ }+ self.decay = float(values["decay"])+ self.num_updates = int(values["num_updates"])+ self._validate(module)++ @torch.no_grad()+ def copy_to(self, module: Any) -> None:+ for name, parameter in self._validate(module):+ parameter.copy_(self.shadow[name].to(parameter.device, parameter.dtype))++ @contextmanager+ def swapped_into(self, module: Any) -> Iterator[None]:+ backup = {name: parameter.detach().clone() for name, parameter in self._validate(module)}+ try:+ self.copy_to(module)+ yield+ finally:+ with torch.no_grad():+ for name, parameter in self._validate(module):+ parameter.copy_(backup[name])+++__all__ = ["H3ShardedEMA"]diff --git a/src/solarwm/backends/minimax_h3/fastvideo_import.py b/src/solarwm/backends/minimax_h3/fastvideo_import.pynew file mode 100644index 0000000..7bed464--- /dev/null+++ b/src/solarwm/backends/minimax_h3/fastvideo_import.py@@ -0,0 +1,248 @@+"""Import FastVideo spatial-AI renders and geometry ControlNet weights into SolarWM.++FastVideo's renderer writes canonical coordinate maps as 16-bit RGB PNG frames+(``coord/NNNNNN.png`` or ``coord.zip``). SolarWM's raw render member is the+same bytes as one uint16 ``[F,H,W,3]`` array (see ``raw_data``). FastVideo+training saves the ControlNet under ``geometry_controlnet.*`` inside a DCP+checkpoint; the names are SolarWM's too.+"""++from __future__ import annotations++import io+import struct+import zipfile+import zlib+from collections.abc import Mapping+from pathlib import Path+from typing import Any++import numpy as np++from solarwm.errors import DataContractError++from .controlnet_contract import H3_GEOMETRY_MODULE+from .raw_data import H3_GEOMETRY_RENDER_KEY++_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n"+_CHANNELS = {2: 3, 6: 4}+FASTVIDEO_STUDENT_STATE_KEY = "roles.student.transformer"+H3_GEOMETRY_WEIGHTS_SCHEMA = "solarwm.minimax-h3-geometry-controlnet-weights.v1"+_WRAPPER_PARTS = frozenset({"_checkpoint_wrapped_module", "_fsdp_wrapped_module", "_orig_mod"})+++def _unfilter(raw: bytes, height: int, stride: int, pixel_bytes: int) -> np.ndarray:+ """Undo PNG scanline filters 0-4 (None, Sub, Up, Average, Paeth).++ None/Sub/Up are vectorized; Average and Paeth depend on the reconstructed+ left pixel and advance one pixel (all its bytes at once) at a time.+ """++ rows = np.zeros((height, stride), dtype=np.uint8)+ previous = np.zeros(stride, dtype=np.int32)+ offset = 0+ for row in range(height):+ kind = raw[offset]+ line = np.frombuffer(raw, dtype=np.uint8, count=stride, offset=offset + 1).astype(np.int32)+ offset += stride + 1+ if kind == 0:+ current = line+ elif kind == 1:+ current = np.cumsum(line.reshape(-1, pixel_bytes), axis=0).reshape(-1) & 0xFF+ elif kind == 2:+ current = (line + previous) & 0xFF+ elif kind in (3, 4):+ current = np.zeros(stride, dtype=np.int32)+ left = np.zeros(pixel_bytes, dtype=np.int32)+ upper_left = np.zeros(pixel_bytes, dtype=np.int32)+ for start in range(0, stride, pixel_bytes):+ pixel = slice(start, start + pixel_bytes)+ up = previous[pixel]+ if kind == 3:+ predictor = (left + up) // 2+ else:+ estimate = left + up - upper_left+ distance_left = np.abs(estimate - left)+ distance_up = np.abs(estimate - up)+ distance_corner = np.abs(estimate - upper_left)+ predictor = np.where(+ (distance_left <= distance_up) & (distance_left <= distance_corner),+ left,+ np.where(distance_up <= distance_corner, up, upper_left),+ )+ current[pixel] = (line[pixel] + predictor) & 0xFF+ left = current[pixel]+ upper_left = up+ else:+ raise DataContractError(f"PNG scanline uses unknown filter {kind}")+ rows[row] = current+ previous = current+ return rows+++def _decode_with_opencv(data: bytes) -> np.ndarray | None:+ try:+ import cv2+ except ImportError:+ return None+ image = cv2.imdecode(np.frombuffer(data, dtype=np.uint8), cv2.IMREAD_UNCHANGED)+ if image is None or image.ndim != 3 or image.shape[2] < 3:+ raise DataContractError("coordinate PNG must be RGB(A)")+ rgb = image[..., 2::-1]+ if rgb.dtype == np.uint8:+ return rgb.astype(np.uint16) * np.uint16(257)+ if rgb.dtype != np.uint16:+ raise DataContractError(f"coordinate PNG has unsupported sample type {rgb.dtype}")+ return np.ascontiguousarray(rgb)+++def decode_png_rgb16(data: bytes, *, prefer_opencv: bool = True) -> np.ndarray:+ """Decode a non-interlaced RGB/RGBA PNG to uint16 ``[H,W,3]`` without resampling.++ 16-bit samples are returned as stored; 8-bit samples are widened by ``*257``+ so that full scale stays full scale. Alpha is dropped. OpenCV (the decoder+ FastVideo reads these files with) is used when installed; the built-in+ decoder is exact but slow on Average/Paeth-filtered scanlines.+ """++ if prefer_opencv:+ decoded = _decode_with_opencv(data)+ if decoded is not None:+ return decoded+ if not data.startswith(_PNG_SIGNATURE):+ raise DataContractError("not a PNG file")+ offset = len(_PNG_SIGNATURE)+ header: tuple[int, ...] | None = None+ compressed = bytearray()+ while offset < len(data):+ length, kind = struct.unpack(">I4s", data[offset : offset + 8])+ chunk = data[offset + 8 : offset + 8 + length]+ offset += 12 + length+ if kind == b"IHDR":+ header = struct.unpack(">IIBBBBB", chunk)+ elif kind == b"IDAT":+ compressed.extend(chunk)+ elif kind == b"IEND":+ break+ if header is None:+ raise DataContractError("PNG lacks IHDR")+ width, height, depth, color, _, _, interlace = header+ if color not in _CHANNELS or depth not in (8, 16) or interlace:+ raise DataContractError(+ f"coordinate PNG must be non-interlaced 8/16-bit RGB(A); got color={color} "+ f"depth={depth} interlace={interlace}"+ )+ channels = _CHANNELS[color]+ pixel_bytes = channels * depth // 8+ rows = _unfilter(zlib.decompress(bytes(compressed)), height, width * pixel_bytes, pixel_bytes)+ if depth == 16:+ samples = rows.reshape(height, width * channels, 2)+ values = (samples[..., 0].astype(np.uint16) << 8) | samples[..., 1]+ else:+ values = rows.astype(np.uint16) * np.uint16(257)+ return values.reshape(height, width, channels)[..., :3].copy()+++def read_fastvideo_coordinates(source: str | Path) -> np.ndarray:+ """Read a FastVideo ``coord/`` directory or ``coord.zip`` as uint16 ``[F,H,W,3]``.++ Frames are ordered by file name, as the FastVideo renderer numbers them.+ """++ path = Path(source)+ frames: list[np.ndarray] = []+ if path.is_dir():+ names = sorted(item for item in path.iterdir() if item.suffix.lower() == ".png")+ frames = [decode_png_rgb16(item.read_bytes()) for item in names]+ elif zipfile.is_zipfile(path):+ with zipfile.ZipFile(path) as archive:+ names = sorted(name for name in archive.namelist() if name.lower().endswith(".png"))+ frames = [decode_png_rgb16(archive.read(name)) for name in names]+ else:+ raise DataContractError(f"{path} is neither a coordinate directory nor a zip")+ if not frames:+ raise DataContractError(f"{path} contains no coordinate PNG frames")+ if len({frame.shape for frame in frames}) != 1:+ raise DataContractError(f"{path} mixes coordinate frame sizes")+ return np.stack(frames)+++def render_member_bytes(coords: np.ndarray) -> bytes:+ """Serialize a render as SolarWM's raw member: a stored (uncompressed) npz.++ Stored members let preencode slice one window by byte offset instead of+ inflating a whole source clip.+ """++ array = np.ascontiguousarray(coords)+ if array.dtype != np.uint16 or array.ndim != 4 or array.shape[-1] != 3:+ raise DataContractError(f"render must be uint16 [F,H,W,3], got {array.dtype} {array.shape}")+ buffer = io.BytesIO()+ np.savez(buffer, **{H3_GEOMETRY_RENDER_KEY: array.astype("<u2", copy=False)})+ return buffer.getvalue()+++def _canonical_controlnet_name(key: str, state_key: str) -> str | None:+ if not key.startswith(f"{state_key}."):+ return None+ parts = [part for part in key[len(state_key) + 1 :].split(".") if part not in _WRAPPER_PARTS]+ if H3_GEOMETRY_MODULE not in parts:+ return None+ return ".".join(parts[parts.index(H3_GEOMETRY_MODULE) :])+++def read_fastvideo_controlnet(+ checkpoint: str | Path, *, state_key: str = FASTVIDEO_STUDENT_STATE_KEY+) -> dict[str, Any]:+ """Load only the ``geometry_controlnet.*`` tensors of a FastVideo DCP checkpoint."""++ import torch+ import torch.distributed.checkpoint as dcp+ from torch.distributed.checkpoint.metadata import TensorStorageMetadata++ reader = dcp.FileSystemReader(str(checkpoint))+ metadata = reader.read_metadata()+ selected: dict[str, str] = {}+ for key, value in metadata.state_dict_metadata.items():+ name = _canonical_controlnet_name(key, state_key)+ if name is None or not isinstance(value, TensorStorageMetadata):+ continue+ if name in selected.values():+ raise DataContractError(f"FastVideo checkpoint has duplicate tensors for {name!r}")+ selected[key] = name+ if not selected:+ raise DataContractError(+ f"{checkpoint} holds no {H3_GEOMETRY_MODULE}.* tensors under {state_key!r}"+ )+ buffers = {+ key: torch.empty(+ tuple(metadata.state_dict_metadata[key].size),+ dtype=metadata.state_dict_metadata[key].properties.dtype,+ )+ for key in selected+ }+ dcp.load(buffers, storage_reader=reader)+ return {selected[key]: value for key, value in buffers.items()}+++def save_controlnet_weights(tensors: Mapping[str, Any], path: str | Path) -> None:+ """Write ``geometry_controlnet.*`` tensors as a SolarWM initialization file."""++ import torch+ from safetensors.torch import save_file++ if not tensors or any(not key.startswith(f"{H3_GEOMETRY_MODULE}.") for key in tensors):+ raise DataContractError(f"ControlNet weights must all be {H3_GEOMETRY_MODULE}.* tensors")+ values = {key: value.detach().to(torch.bfloat16).contiguous() for key, value in tensors.items()}+ save_file(values, str(path), metadata={"schema": H3_GEOMETRY_WEIGHTS_SCHEMA})+++__all__ = [+ "FASTVIDEO_STUDENT_STATE_KEY",+ "H3_GEOMETRY_WEIGHTS_SCHEMA",+ "decode_png_rgb16",+ "read_fastvideo_controlnet",+ "read_fastvideo_coordinates",+ "render_member_bytes",+ "save_controlnet_weights",+]diff --git a/src/solarwm/backends/minimax_h3/flow.py b/src/solarwm/backends/minimax_h3/flow.pynew file mode 100644index 0000000..f6ffb25--- /dev/null+++ b/src/solarwm/backends/minimax_h3/flow.py@@ -0,0 +1,174 @@+"""MiniMax-H3 data-ward rectified-flow arithmetic.++The public helpers use NumPy so their signs, schedules, and reconstruction+identity can be tested without a GPU runtime.+"""++from __future__ import annotations++from dataclasses import dataclass++from collections.abc import Sequence++import numpy as np+++@dataclass(frozen=True)+class H3FlowSchedule:+ """Shifted sigma grid and model timesteps; terminal sigma has no timestep."""++ sigmas: np.ndarray+ timesteps: np.ndarray+++def _pair(clean: object, noise: object) -> tuple[np.ndarray, np.ndarray]:+ clean_array = np.asarray(clean)+ noise_array = np.asarray(noise)+ if clean_array.shape != noise_array.shape:+ raise ValueError(+ f"clean/noise shapes must match, got {clean_array.shape} and {noise_array.shape}"+ )+ if not np.issubdtype(clean_array.dtype, np.floating):+ raise TypeError("clean sample must use a floating dtype")+ if not np.issubdtype(noise_array.dtype, np.floating):+ raise TypeError("noise sample must use a floating dtype")+ return clean_array, noise_array+++def _broadcast_time(value: object, sample: np.ndarray) -> np.ndarray:+ result = np.asarray(value, dtype=sample.dtype)+ while result.ndim < sample.ndim:+ result = np.expand_dims(result, axis=-1)+ return result+++def scale_noise(clean_sample: object, noise: object, timestep: object) -> np.ndarray:+ """Apply ``x_t = t*x0 + (1-t)*noise`` where ``t=1`` is clean."""++ clean_array, noise_array = _pair(clean_sample, noise)+ time = _broadcast_time(timestep, clean_array)+ return time * clean_array + (1.0 - time) * noise_array+++def data_velocity_target(clean_sample: object, noise: object) -> np.ndarray:+ """Return H3's data-ward velocity target, ``x0 - noise``."""++ clean_array, noise_array = _pair(clean_sample, noise)+ return clean_array - noise_array+++def predict_clean_sample(+ noisy_sample: object,+ velocity: object,+ timestep: object,+) -> np.ndarray:+ """Recover ``x0 = x_t + (1-t)*v`` under the H3 velocity convention."""++ noisy_array = np.asarray(noisy_sample)+ velocity_array = np.asarray(velocity)+ if noisy_array.shape != velocity_array.shape:+ raise ValueError(+ "noisy sample and velocity must have equal shapes, "+ f"got {noisy_array.shape} and {velocity_array.shape}"+ )+ sigma = 1.0 - _broadcast_time(timestep, noisy_array)+ return noisy_array + sigma * velocity_array+++def shift_sigma(sigma: object, shift: float) -> np.ndarray:+ """Apply H3's exponential shift ``s*sigma/(1+(s-1)*sigma)``."""++ if not np.isfinite(shift) or shift <= 0:+ raise ValueError(f"shift must be finite and positive, got {shift}")+ values = np.asarray(sigma)+ return shift * values / (1.0 + (shift - 1.0) * values)+++def sample_shifted_timestep(+ shape: int | tuple[int, ...],+ *,+ shift: float,+ generator: np.random.Generator,+ dtype: np.dtype | type = np.float32,+) -> np.ndarray:+ """Sample ``t=1-shift_sigma(U(0,1))`` from an explicit RNG."""++ normalized = (shape,) if isinstance(shape, int) else tuple(shape)+ if not normalized or any(int(size) <= 0 for size in normalized):+ raise ValueError(f"shape must be positive, got {shape!r}")+ if not isinstance(generator, np.random.Generator):+ raise TypeError("generator must be an explicit numpy.random.Generator")+ base_sigma = generator.random(normalized, dtype=np.dtype(dtype))+ return 1.0 - shift_sigma(base_sigma, shift)+++def make_shifted_schedule(+ num_inference_steps: int,+ *,+ shift: float = 12.0,+) -> H3FlowSchedule:+ """Build the exact descending shifted grid, including terminal sigma zero."""++ if num_inference_steps < 2:+ raise ValueError("num_inference_steps must be at least 2")+ base = np.linspace(1.0, 0.0, int(num_inference_steps), dtype=np.float32)+ shifted = np.asarray(shift_sigma(base, shift), dtype=np.float32)+ keep = np.concatenate((np.asarray([True]), shifted[1:] != shifted[:-1]))+ sigmas = shifted[keep]+ return H3FlowSchedule(sigmas=sigmas, timesteps=1.0 - sigmas[:-1])+++def explicit_schedule(sigmas: Sequence[float]) -> H3FlowSchedule:+ """A schedule given as its sigma list (a released few-step adapter's own steps), descending from at most 1 to 0."""++ values = np.asarray([float(x) for x in sigmas], dtype=np.float32)+ if values.size < 2 or values[0] > 1.0 or values[-1] != 0.0 or np.any(np.diff(values) >= 0):+ raise ValueError(f"a schedule descends strictly from at most 1 to exactly 0, got {list(sigmas)}")+ return H3FlowSchedule(sigmas=values, timesteps=1.0 - values[:-1])+++def euler_step(+ sample: object,+ velocity: object,+ timestep: object,+ sigma: object,+ sigma_next: object,+) -> np.ndarray:+ """Take one deterministic H3 Euler step using float32 accumulation."""++ sample_array = np.asarray(sample)+ velocity_array = np.asarray(velocity)+ if sample_array.shape != velocity_array.shape:+ raise ValueError("sample and velocity shapes must match")+ dtype_name = sample_array.dtype.name+ compute_dtype = (+ np.dtype(np.float32) if dtype_name in {"float16", "bfloat16"} else sample_array.dtype+ )+ sigma_array = np.asarray(sigma, dtype=compute_dtype)+ sigma_next_array = np.asarray(sigma_next, dtype=compute_dtype)+ if np.any(sigma_array == 0):+ raise ValueError("Euler step cannot start from sigma zero")+ denoised = predict_clean_sample(sample_array, velocity_array, timestep)+ ratio = sigma_next_array / sigma_array+ result = ratio * sample_array.astype(compute_dtype) + (1.0 - ratio) * denoised.astype(+ compute_dtype+ )+ return result.astype(sample_array.dtype, copy=False)+++add_noise = scale_noise+velocity_target = data_velocity_target+++__all__ = [+ "H3FlowSchedule",+ "add_noise",+ "data_velocity_target",+ "euler_step",+ "make_shifted_schedule",+ "predict_clean_sample",+ "sample_shifted_timestep",+ "scale_noise",+ "shift_sigma",+ "velocity_target",+]diff --git a/src/solarwm/backends/minimax_h3/fsdp.py b/src/solarwm/backends/minimax_h3/fsdp.pynew file mode 100644index 0000000..48eb93f--- /dev/null+++ b/src/solarwm/backends/minimax_h3/fsdp.py@@ -0,0 +1,220 @@+"""FULL_SHARD and activation-checkpointing setup for H3."""++from __future__ import annotations++import functools+import math+from collections.abc import Iterable+from datetime import timedelta+from typing import Any++import torch+import torch.distributed as dist+from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import (+ CheckpointImpl,+ apply_activation_checkpointing,+ checkpoint_wrapper,+)+from torch.utils.checkpoint import CheckpointPolicy, create_selective_checkpoint_contexts+from torch.distributed.fsdp import (+ BackwardPrefetch,+ MixedPrecision,+ ShardingStrategy,+)+from torch.distributed.fsdp import (+ FullyShardedDataParallel as FSDP,+)++from .distributed import get_dp_group, init_sequence_parallel+++def initialize_distributed(*, sp_size: int, local_rank: int) -> tuple[int, int]:+ """Initialize NCCL and H3's SP/DP process groups under torchrun."""++ import os++ world = int(os.environ.get("WORLD_SIZE", "1"))+ rank = int(os.environ.get("RANK", "0"))+ torch.cuda.set_device(local_rank)+ if world > 1 and not dist.is_initialized():+ dist.init_process_group(backend="nccl", init_method="env://", timeout=timedelta(hours=1))+ init_sequence_parallel(sp_size)+ return rank, world+++def wrap_h3_fsdp(+ model: torch.nn.Module,+ *,+ local_rank: int,+ transformer_block_cls: type[torch.nn.Module],+ fp32_units: Iterable[torch.nn.Module],+ ignored_parameters: Iterable[torch.nn.Parameter],+ activation_checkpointing: bool = True,+ frozen_base_shard_size: int | None = None,+ saved_attention_blocks: int = 0,+) -> torch.nn.Module:+ """Wrap 50 H3 blocks and the six FP32 leaf owners over logical DP.++ Every block is recomputed in backward; the first ``saved_attention_blocks`` keep their attention+ output from the forward instead of running attention a second time (about 0.74 GB per block per+ rank at 158 frames with the structure stream), which is most of a block's recompute cost."""++ if not dist.is_initialized() or dist.get_world_size() == 1:+ if activation_checkpointing and hasattr(model, "enable_gradient_checkpointing"):+ model.enable_gradient_checkpointing()+ return model.to(torch.device("cuda", local_rank))+ standalone = tuple(fp32_units)+ standalone_ids = {id(module) for module in standalone}+ if len(standalone_ids) not in {6, 8}:+ raise ValueError("H3 FSDP requires six FP32 leaf owners, plus two for AnyFlow")+ ignored = tuple(ignored_parameters)+ if len({id(parameter) for parameter in ignored}) != len(ignored):+ raise ValueError("H3 FSDP ignored parameters contain duplicates")++ def policy(module: torch.nn.Module, recurse: bool, nonwrapped_numel: int) -> bool:+ del nonwrapped_numel+ if recurse:+ return True+ return isinstance(module, transformer_block_cls) or id(module) in standalone_ids++ process_group = get_dp_group()+ strategy = ShardingStrategy.FULL_SHARD+ if frozen_base_shard_size is not None:+ process_group = frozen_base_groups(frozen_base_shard_size)+ strategy = ShardingStrategy.HYBRID_SHARD+ wrapped = FSDP(+ model,+ auto_wrap_policy=policy,+ sharding_strategy=strategy,+ mixed_precision=MixedPrecision(+ param_dtype=None, reduce_dtype=torch.float32, buffer_dtype=None+ ),+ cpu_offload=None,+ device_id=local_rank,+ use_orig_params=True,+ forward_prefetch=True,+ backward_prefetch=BackwardPrefetch.BACKWARD_PRE,+ sync_module_states=False,+ process_group=process_group,+ limit_all_gathers=True,+ ignored_states=ignored or None,+ )+ if activation_checkpointing:+ blocks_in_order = [m for m in wrapped.modules() if isinstance(m, transformer_block_cls)]+ keep = {id(m) for m in blocks_in_order[:saved_attention_blocks]}+ apply_activation_checkpointing(+ wrapped,+ checkpoint_wrapper_fn=lambda module: block_checkpoint_wrapper(+ module, save_attention=id(module) in keep+ ),+ check_fn=lambda module: isinstance(module, transformer_block_cls),+ )+ blocks = sum(isinstance(module, transformer_block_cls) for module in wrapped.modules())+ if blocks != 50:+ raise RuntimeError(f"H3 activation checkpointing expected 50 blocks, got {blocks}")+ return wrapped+++def _attention_ops() -> frozenset:+ """The attention kernels a block's forward can call: torch's own, and flash-attn's when it is loaded."""+ names = (+ "_scaled_dot_product_cudnn_attention",+ "_scaled_dot_product_flash_attention",+ "_scaled_dot_product_efficient_attention",+ "_scaled_dot_product_flash_attention_for_cpu",+ )+ ops = {getattr(torch.ops.aten, name).default for name in names if hasattr(torch.ops.aten, name)}+ flash = torch.ops.flash_attn+ for name in ("_flash_attn_forward", "_flash_attn_varlen_forward"):+ if hasattr(flash, name):+ ops.add(getattr(flash, name).default)+ return frozenset(ops)+++def keep_attention(ctx: Any, op: Any, *args: Any, **kwargs: Any) -> CheckpointPolicy:+ """Selective-checkpoint policy: keep what attention returns, recompute everything else."""+ del ctx, args, kwargs+ return CheckpointPolicy.MUST_SAVE if op in _attention_ops() else CheckpointPolicy.PREFER_RECOMPUTE+++def block_checkpoint_wrapper(module: torch.nn.Module, *, save_attention: bool) -> torch.nn.Module:+ """A block recomputed in backward, optionally without running its attention again."""+ if not save_attention:+ return checkpoint_wrapper(module, checkpoint_impl=CheckpointImpl.NO_REENTRANT)+ return checkpoint_wrapper(+ module,+ checkpoint_impl=CheckpointImpl.NO_REENTRANT,+ context_fn=functools.partial(create_selective_checkpoint_contexts, keep_attention),+ )++[email protected]_grad()+def finite_clip_norm(+ parameters: Iterable[torch.nn.Parameter], max_norm: float+) -> tuple[bool, float]:+ """Clip replicated LoRA gradients with an overflow-safe global 2-norm."""++ max_norm = float(max_norm)+ if not math.isfinite(max_norm) or max_norm <= 0.0:+ raise ValueError("max_norm must be finite and positive")+ values = tuple(parameter for parameter in parameters if parameter.grad is not None)+ if not values:+ raise RuntimeError("H3 optimizer step has no LoRA gradients")+ device = values[0].grad.device+ squared = torch.zeros((), device=device, dtype=torch.float64)+ for parameter in values:+ gradient = parameter.grad+ if gradient.is_sparse:+ raise RuntimeError("H3 does not support sparse LoRA gradients")+ detached = gradient.detach()+ norm_dtype = torch.float64 if detached.dtype == torch.float64 else torch.float32+ local_scale = detached.abs().amax().double()+ safe_scale = torch.where(local_scale > 0, local_scale, torch.ones_like(local_scale))+ scaled_norm = torch.linalg.vector_norm(+ detached / safe_scale.to(dtype=detached.dtype),+ ord=2,+ dtype=norm_dtype,+ )+ squared.add_((local_scale * scaled_norm.double()).square())+ norm = squared.sqrt()+ finite = bool(torch.isfinite(norm).item())+ if finite:+ coefficient = min(1.0, max_norm / (float(norm.item()) + 1.0e-6))+ if coefficient < 1.0:+ for parameter in values:+ parameter.grad.mul_(coefficient)+ return finite, float(norm.float().item())+++__all__ = ["finite_clip_norm", "initialize_distributed", "wrap_h3_fsdp"]+++_FROZEN_BASE_GROUPS: dict[tuple[int, int], tuple[object, object]] = {}+++def frozen_base_groups(shard_size: int) -> tuple[object, object]:+ """Shard only the frozen base over physical node GPUs; LoRA keeps SP/DP."""+ import os++ world, rank = dist.get_world_size(), dist.get_rank()+ local_world = int(os.environ["LOCAL_WORLD_SIZE"])+ if shard_size != local_world or world % shard_size:+ raise ValueError("H3 frozen-base shards must be exactly one complete physical node")+ key = (world, shard_size)+ if key not in _FROZEN_BASE_GROUPS:+ shard, replica = None, None+ for first in range(0, world, shard_size):+ ranks = list(range(first, first + shard_size))+ group = dist.new_group(ranks)+ if rank in ranks:+ shard = group+ for offset in range(shard_size):+ ranks = list(range(offset, world, shard_size))+ group = dist.new_group(ranks)+ if rank in ranks:+ replica = group+ if shard is None or replica is None:+ raise RuntimeError("H3 frozen-base process groups are incomplete")+ _FROZEN_BASE_GROUPS[key] = (shard, replica)+ return _FROZEN_BASE_GROUPS[key]diff --git a/src/solarwm/backends/minimax_h3/full_inference.py b/src/solarwm/backends/minimax_h3/full_inference.pynew file mode 100644index 0000000..579c6f8--- /dev/null+++ b/src/solarwm/backends/minimax_h3/full_inference.py@@ -0,0 +1,733 @@+"""Bucket-native, source-length H3 Stage2 SGF evaluation with SP8.++Preparation encodes only the caption/first-image condition. Generation uses+the existing student raw-KV W6 sampler and the official H3 temporal VAE.+"""++from __future__ import annotations++import gc+import json+import os+import subprocess+import time+from dataclasses import asdict+from pathlib import Path++from .full_length import exported_fps, full_length_geometry+from .source_length_cache import (+ atomic_json,+ preparation_identity,+ prepared_case_receipt,+ result_identity,+ source_files,+ verify_completed_result,+)+from .source_length_cache import (+ file_sha256 as sha,+)+++def selected_rows(args):+ from solarwm.data.index import validate_relative_key++ rows = json.loads(Path(args.plan).read_text())+ if not isinstance(rows, list) or not rows:+ raise ValueError("Full-length inference plan must be a nonempty JSON row array")+ keys = set()+ for row in rows:+ for field in ("key", "dataset"):+ value = validate_relative_key(row[field])+ if "/" in value:+ raise ValueError("Case key and dataset must each be one path component")+ if row["key"] in keys:+ raise ValueError("Duplicate case key in inference plan")+ keys.add(row["key"])+ if row.get("input_kind", "wds") not in {"wds", "demo"}:+ raise ValueError("Unsupported source-length input_kind")+ if int(row["num_frames"]) > 960 and not args.config["inference"].get(+ "stream_decode", False+ ):+ raise ValueError("Videos longer than 960 frames require inference.stream_decode=true")+ full_length_geometry(int(row["num_frames"]))+ exported_fps(float(row["fps"]), args.fps_policy)+ return rows+++def encode_source_render(video_vae, render, geometry, transform, *, device):+ """Encode a whole-source render member to ``[24, rollout_latents, 48, 84]`` latents.++ The render is stretched like the first image and padded like the decode+ (repeating the last frame to ``17n+5``), so render latent ``k`` drives rollout+ latent ``k``; the latent tail beyond the decode repeats like the cameras.+ """++ from .controlnet import pad_control_latents+ from .official_codec import encode_geometry_clips+ from .raw_data import decode_geometry_render++ frames = int(geometry.source_frames)++ def read(start, stop):+ indices = [min(index, frames - 1) for index in range(start, stop)]+ return decode_geometry_render(render, indices, transform)++ latents = encode_geometry_clips(video_vae, read, int(geometry.decoded_frames), device=device)+ if int(latents.shape[1]) != int(geometry.decode_latents):+ raise ValueError("Render latents differ from the decoded video length")+ return pad_control_latents(latents[None], int(geometry.rollout_latents))[0].contiguous()+++def prepare(args):+ import numpy as np+ import torch+ from decord import VideoReader, cpu+ from diffusers.modular_pipelines.minimax_h3.encoders import encode_vae_condition+ from PIL import Image+ from safetensors.torch import save_file++ from .controlnet_contract import geometry_member_field, geometry_profile_from_model_config+ from .official_codec import encode_joint_prompt_condition+ from .optional import load_conditioners+ from .raw_data import _normalise_K+ from .torch_prope import _invert_se3++ control_profile = geometry_profile_from_model_config(args.config["model"])+ rank = int(os.environ.get("LOCAL_RANK", "0"))+ world = int(os.environ.get("LOCAL_WORLD_SIZE", "1"))+ torch.set_num_threads(2)+ device = torch.device("cuda", rank)+ torch.cuda.set_device(device)+ pending = []+ for row in selected_rows(args)[rank::world]:+ case = Path(args.work_dir) / "cases" / row["key"]+ if (case / "READY.json").is_file():+ prepared_case_receipt(case, row, args)+ continue+ pending.append(row)+ if not pending:+ return+ components = load_conditioners(+ args.config["model"], device=device, audio_vae=False, schedulers=False+ )+ print(f"PREPARE_READY rank={rank} samples={len(pending)}", flush=True)+ for row in pending:+ if row.get("input_kind") == "demo":+ if control_profile is not None:+ raise ValueError(+ "Geometry ControlNet runs need dataset rows with render members, "+ f"not demo input {row['key']!r}"+ )+ from .demo_conditions import prepare_demo++ prepare_demo(row, args, components, device)+ gc.collect()+ torch.cuda.empty_cache()+ continue+ started = time.perf_counter()+ case = source_files(row, args)+ reader = VideoReader(str(case / "source.mp4"), ctx=cpu(), num_threads=2)+ frames = len(reader)+ if frames != int(row["num_frames"]):+ raise ValueError(f"Source frame count mismatch: {frames} != {row['num_frames']}")+ fps = float(reader.get_avg_fps())+ if abs(fps - float(row["fps"])) > 0.02:+ raise ValueError(f"Source FPS mismatch: {fps} != {row['fps']}")+ first = reader[0].asnumpy()+ height, width = first.shape[:2]+ if (height, width) != (row["height"], row["width"]):+ raise ValueError("Source dimensions differ from index")+ image = Image.fromarray(first).resize((1344, 768), Image.Resampling.LANCZOS)+ image.save(case / "first.png")+ del reader, first+ geometry = full_length_geometry(frames)+ with np.load(case / "source.camera.npz", allow_pickle=False) as data:+ if "c2w" not in data:+ raise ValueError("Standalone test source must contain authoritative c2w")+ c2w = np.asarray(data["c2w"]).copy()+ if c2w.shape != (frames, 4, 4) or not np.isfinite(c2w).all():+ raise ValueError("Invalid full camera trajectory")+ np.testing.assert_allclose(c2w[:, 3], np.broadcast_to([0, 0, 0, 1], (frames, 4)), atol=1e-5)+ transform = dict(+ source_w=width,+ source_h=height,+ resized_w=1344,+ resized_h=768,+ target_w=1344,+ target_h=768,+ crop_left=0,+ crop_top=0,+ )+ source_K = np.load(case / "source.intrinsics.npy", allow_pickle=False)+ # Full test-set evaluation preserves every source sample. With the+ # checkpoint's wan_fixed contract, attention replaces source focal+ # values before PRoPE; training's source focal-range filter is unused.+ # Still validate source shape, temporal alignment and finite values.+ K = _normalise_K(+ source_K,+ tuple(range(frames)),+ transform,+ enforce_focal_guard=args.config["model"].get("camera_intrinsics_mode") != "wan_fixed",+ )+ poses = torch.from_numpy(c2w).float()+ relative = torch.matmul(_invert_se3(poses[:1]), poses)+ relative[0] = torch.eye(4)+ views = _invert_se3(relative).contiguous()+ idx = torch.tensor(geometry.camera_indices)+ with torch.inference_mode():+ prompt, tags = encode_joint_prompt_condition(+ image,+ row["caption"],+ processor=components.processor,+ tokenizer=components.tokenizer,+ text_encoder=components.text_encoder,+ device=device,+ )+ pixels = (+ torch.from_numpy(np.asarray(image).copy())+ .permute(2, 0, 1)[None, :, None]+ .to(device)+ )+ anchor = encode_vae_condition(+ components.video_vae,+ pixels,+ (0.485, 0.456, 0.406),+ (0.229, 0.224, 0.225),+ encode_seed=42,+ )+ values = dict(+ prompt_embeds=prompt.cpu().contiguous(),+ text_token_tags=tags.cpu().contiguous(),+ anchor_latents=anchor.detach().cpu().contiguous(),+ viewmats=views[idx].contiguous(),+ K=torch.from_numpy(K)[idx].contiguous(),+ )+ if control_profile is not None:+ for key in control_profile.tensor_keys():+ field = geometry_member_field(key)+ values[key] = encode_source_render(+ components.video_vae,+ case / f"source.{field.removesuffix('_member')}.npz",+ geometry,+ transform,+ device=device,+ )+ if not all(bool(torch.isfinite(t).all()) for t in values.values()):+ raise FloatingPointError("Non-finite encoded condition")+ target = case / "condition.safetensors"+ save_file(values, str(target.with_suffix(".tmp")))+ target.with_suffix(".tmp").replace(target)+ torch.cuda.synchronize()+ atomic_json(+ case / "READY.json",+ dict(+ schema="solarwm.h3-full-condition.v1",+ row=row,+ preparation_identity=preparation_identity(row, args),+ geometry=asdict(geometry),+ condition_sha256=sha(target),+ source_sha256=sha(case / "source.mp4"),+ preparation_seconds=time.perf_counter() - started,+ source_fps=fps,+ full_camera_max_relative_translation=float(views[:, :3, 3].norm(dim=-1).max()),+ image_resize="PIL_LANCZOS_stretch_1344x768",+ model_fps=24,+ camera_convention="relative_w2c+normalized_K",+ source_frame_policy="all_consecutive_no_resampling",+ ),+ )+ print(+ f"PREPARED rank={rank} key={row['key']} frames={frames}"+ f" seconds={time.perf_counter() - started:.3f}",+ flush=True,+ )+ del values, anchor, pixels, prompt, tags+ gc.collect()+ torch.cuda.empty_cache()+++def decode_write(vae, generated, geometry, output, fps, device, *, streaming=False, progress=None):+ """Use official temporal overlap blending; postprocess one CPU frame at a time."""+ import imageio.v2 as imageio+ import torch++ if streaming:+ from .stream_decode import decoded_chunks++ chunks = decoded_chunks(vae, generated[:, :, : geometry.decode_latents], device=device)+ else:+ z = generated[:, :, : geometry.decode_latents].float()+ mean = torch.tensor(vae.config.latents_mean, device=device).view(1, -1, 1, 1, 1)+ std = torch.tensor(vae.config.latents_std, device=device).view(1, -1, 1, 1, 1)+ with torch.no_grad(), torch.autocast("cuda", dtype=torch.float16):+ pixels = vae.decode(z * std + mean, return_dict=False)[0]+ if pixels.shape[2] != geometry.decoded_frames:+ raise ValueError("VAE output frame count differs")+ chunks = (pixels.cpu(),)++ def frames():+ count = 0+ for chunk in chunks:+ for frame in chunk[0].unbind(1):+ if count < geometry.source_frames:+ yield count, frame+ count += 1+ if progress is not None:+ progress(count)+ if count != geometry.decoded_frames:+ raise ValueError(f"VAE returned {count} frames, expected {geometry.decoded_frames}")++ part = output.with_name("." + output.stem + ".tmp.mp4")+ pixel_mean = torch.tensor((0.485, 0.456, 0.406)).view(3, 1, 1)+ pixel_std = torch.tensor((0.229, 0.224, 0.225)).view(3, 1, 1)+ story_indices = sorted(+ set(round((geometry.source_frames - 1) * p) for p in (0, 0.1, 0.25, 0.5, 0.75, 0.9, 1))+ )+ story = []+ luminance = []+ with imageio.get_writer(+ str(part),+ fps=fps,+ codec="libx264",+ quality=8,+ macro_block_size=1,+ ffmpeg_params=["-threads", "2", "-pix_fmt", "yuv420p"],+ ) as writer:+ for i, decoded in frames():+ frame = (decoded.float() * pixel_std + pixel_mean).clamp(0, 1)+ if not bool(torch.isfinite(frame).all()):+ raise FloatingPointError("Non-finite decoded frame")+ array = frame.mul(255).round().byte().permute(1, 2, 0).numpy()+ writer.append_data(array)+ if i in story_indices:+ from PIL import Image++ story.append(Image.fromarray(array).resize((448, 256)))+ luminance.append(float(array.mean()))+ part.replace(output)+ from PIL import Image, ImageDraw++ sheet = Image.new("RGB", (448 * len(story), 280), "#202020")+ draw = ImageDraw.Draw(sheet)+ for j, (im, index) in enumerate(zip(story, story_indices, strict=True)):+ sheet.paste(im, (448 * j, 24))+ draw.text((448 * j + 6, 5), f"{index / fps:.2f}s / frame {index}", fill="white")+ sheet.save(output.with_suffix(".storyboard.jpg"), quality=90)+ return dict(sampled_frames=story_indices, mean_pixel_value=luminance)+++def generate(args):+ import torch+ import torch.distributed as dist+ from safetensors.torch import load_file++ from .artifacts import load_silence_latents+ from .controlnet_contract import geometry_profile_from_model_config, geometry_tensor_key+ from .distributed import broadcast_sp_tensor as broadcast_sequence_parallel_tensor+ from .layout import patchify_video+ from .lora import inject_h3_lora+ from .optional import load_conditioners, load_transformer+ from .runtime import _base_model_load_receipt+ from .sgf_rollout import H3SGFInputs, h3_sgf_rollout+ from .torch_flow import sample_shifted_timestep, scale_noise+ from .weights import load_initial_weights++ torch.set_num_threads(2)+ local = int(os.environ["LOCAL_RANK"])+ rank, world = dist.get_rank(), dist.get_world_size()+ if world != 8:+ raise ValueError("Full test-set worker requires exactly eight GPUs, SP8")+ device = torch.device("cuda", local)+ cfg = args.config+ control_profile = geometry_profile_from_model_config(cfg["model"])+ control_keys = (+ ()+ if control_profile is None+ else (+ *((name, geometry_tensor_key(name)) for name in control_profile.modalities),+ *(+ (f"{name}_wide", geometry_tensor_key(name, wide=True))+ for name in control_profile.wide_modalities+ ),+ )+ )+ torch.manual_seed(42)+ modules = load_transformer(cfg["model"], device=device)+ base = _base_model_load_receipt(cfg["model"], modules.transformer)+ model, lora = inject_h3_lora(modules.transformer, cfg["model"]["adapter"], base_identity=base)+ weights_id = load_initial_weights(+ dict(path=args.checkpoint, stage="stage2", weight_source=args.weight_source), lora+ )+ step = int(weights_id.rsplit("step=", 1)[1])+ if step != args.expected_step:+ raise ValueError(f"Expected checkpoint step {args.expected_step}, found {step}")+ model.eval().requires_grad_(False)+ silence, _ = load_silence_latents(args.silence, pixel_frames=158)+ silence = (+ silence.to(device).float().unsqueeze(0).permute(0, 1, 3, 2).reshape(1, -1, 32).contiguous()+ )+ vae = (+ load_conditioners(+ cfg["model"], device=device, qwen=False, audio_vae=False, schedulers=False+ ).video_vae+ if rank == 0+ else None+ )+ if vae is not None:+ vae.set_attention_backend("native")+ print(+ f"MODEL_READY rank={rank} weight_source={args.weight_source} step={step} sp=8"+ f" memory_gib={torch.cuda.memory_allocated() / 2**30:.2f}",+ flush=True,+ )+ checkpoint_receipt_sha256 = sha(Path(args.checkpoint) / "COMPLETE.json")+ silence_sha256 = sha(Path(args.silence))+ root = Path(args.output_dir)+ root.mkdir(parents=True, exist_ok=True)+ try:+ for row in selected_rows(args):+ dest = root / row["dataset"]+ dest.mkdir(parents=True, exist_ok=True)+ manifest = dest / (row["key"] + ".json")+ noise_seed = int(+ row.get("validation_noise_seed", row.get("validation_seed", args.noise_seed))+ )+ started = time.perf_counter()+ case = Path(args.work_dir) / "cases" / row["key"]+ condition_receipt = prepared_case_receipt(case, row, args)+ identity = result_identity(+ args,+ condition_receipt,+ checkpoint_sha256=checkpoint_receipt_sha256,+ silence_sha256=silence_sha256,+ seed=noise_seed,+ )+ if manifest.is_file():+ verify_completed_result(+ manifest,+ expected_identity=identity,+ save_latents=bool(cfg["inference"].get("save_latents", False)),+ )+ continue+ values = load_file(str(case / "condition.safetensors"), device="cpu")+ from types import SimpleNamespace++ prompt, tags, anchor = (+ values["prompt_embeds"],+ values["text_token_tags"],+ values["anchor_latents"],+ )+ if prompt.ndim == 2:+ prompt = prompt.unsqueeze(0)+ if anchor.ndim == 4:+ anchor = anchor.unsqueeze(0)+ if (+ prompt.ndim != 3+ or prompt.shape[0] != 1+ or prompt.shape[-1] != 5120+ or tuple(tags.shape) != (prompt.shape[1],)+ or tuple(anchor.shape) != (1, 24, 1, 48, 84)+ ):+ raise ValueError("Invalid H3 prompt, tags or anchor shape")+ if not set(tags.tolist()) <= {0, 1} or not all(+ bool(torch.isfinite(t).all()) for t in values.values()+ ):+ raise ValueError("Invalid H3 condition values")+ condition = SimpleNamespace(+ prompt_embeds=prompt, text_token_tags=tags, anchor_latents=anchor+ )+ geometry = full_length_geometry(int(row["num_frames"]))+ rng = torch.Generator(device=device).manual_seed(noise_seed)++ def random_like(value, *, rng=rng):+ out = torch.randn(value.shape, device=device, dtype=torch.float32, generator=rng)+ broadcast_sequence_parallel_tensor(out)+ return out++ anchor = condition.anchor_latents.to(device).float()+ anchor = scale_noise(anchor, random_like(anchor), 0.999)+ audio_t = sample_shifted_timestep(1, 3.0, device, torch.float32, generator=rng).reshape(+ ()+ )+ broadcast_sequence_parallel_tensor(audio_t)+ audio = scale_noise(silence, random_like(silence), audio_t)+ views, intrinsics = (+ values["viewmats"].unsqueeze(0).to(device),+ values["K"].unsqueeze(0).to(device),+ )+ views = torch.cat((views[:, :1], views), 1)+ intrinsics = torch.cat((intrinsics[:, :1], intrinsics), 1)+ geometry_latents = None+ if control_profile is not None:+ geometry_latents = {}+ for name, key in control_keys:+ value = values[key]+ if tuple(value.shape) != (24, geometry.rollout_latents, 48, 84):+ raise ValueError(f"Invalid geometry condition shape for {key}")+ value = value.to(device).float().unsqueeze(0).contiguous()+ broadcast_sequence_parallel_tensor(value)+ geometry_latents[name] = value+ inputs = H3SGFInputs(+ condition.prompt_embeds.to(device).bfloat16(),+ condition.text_token_tags.to(device),+ patchify_video(anchor),+ audio,+ audio_t,+ views,+ intrinsics,+ 48,+ 84,+ 0.999,+ True,+ geometry=control_profile,+ geometry_latents=geometry_latents,+ )+ noise = random_like(torch.empty(1, 24, geometry.rollout_latents, 48, 84, device=device))+ dist.barrier()+ torch.cuda.synchronize()+ generation_start = time.perf_counter()++ def progress(chunk, *, row=row, geometry=geometry, generation_start=generation_start):+ if rank == 0:+ torch.cuda.synchronize()+ print(+ f"CHUNK key={row['key']} chunk={chunk}/{geometry.rollout_latents // 5}"+ f" seconds={time.perf_counter() - generation_start:.3f}",+ flush=True,+ )++ rollout = h3_sgf_rollout(+ student=model,+ inputs=inputs,+ noise=noise,+ exit_index=3,+ generator=rng,+ video_shift=12.0,+ progress=progress,+ inference_full_length=True,+ output_device="cpu" if cfg["inference"].get("stream_decode", False) else None,+ )+ torch.cuda.synchronize()+ dist.barrier()+ generation_seconds = time.perf_counter() - generation_start+ streaming = bool(cfg["inference"].get("stream_decode", False))++ def decode_progress(_count):+ heartbeat = torch.ones((), device=device, dtype=torch.int32)+ dist.broadcast(heartbeat, src=0)++ if rank != 0 and streaming:+ # Hour-long VAE/encoding can exceed the process-group timeout.+ # Participate in each bounded temporal chunk's progress event.+ while True:+ event = torch.empty((), device=device, dtype=torch.int32)+ dist.broadcast(event, src=0)+ if int(event) == 0:+ break+ if int(event) < 0:+ raise RuntimeError("Rank zero failed during streaming video publication")+ if rank == 0:+ try:+ if cfg["inference"].get("save_latents", False):+ from safetensors.torch import save_file++ save_file(+ {+ "generated_latents": rollout.cache_target[+ :, :, : geometry.decode_latents+ ]+ .float()+ .cpu()+ .contiguous()+ },+ str(dest / (row["key"] + ".latents.safetensors")),+ )+ fps = exported_fps(float(row["fps"]), args.fps_policy)+ output = dest / (row["key"] + ".mp4")+ decode_start = time.perf_counter()+ visual = decode_write(+ vae,+ rollout.cache_target,+ geometry,+ output,+ fps,+ device,+ streaming=streaming,+ progress=decode_progress if streaming else None,+ )+ decode_seconds = time.perf_counter() - decode_start+ import imageio_ffmpeg++ ffmpeg = imageio_ffmpeg.get_ffmpeg_exe()+ compare = output.with_name(output.stem + ".compare.mp4")+ part = compare.with_name("." + compare.stem + ".tmp.mp4")+ if row.get("input_kind") != "demo":+ subprocess.run(+ [+ ffmpeg,+ "-nostdin",+ "-v",+ "error",+ "-y",+ "-i",+ str(case / "source.mp4"),+ "-i",+ str(output),+ "-filter_complex",+ f"[0:v]setpts=N/({fps}*TB),scale=1344:768:flags=lanczos,setsar=1[a];[1:v]setsar=1[b];[a][b]hstack",+ "-frames:v",+ str(geometry.source_frames),+ "-r",+ str(fps),+ "-an",+ "-c:v",+ "libx264",+ "-crf",+ "20",+ "-preset",+ "fast",+ "-threads",+ "2",+ str(part),+ ],+ check=True,+ )+ part.replace(compare)+ from decord import VideoReader, cpu++ for path in (output,) if row.get("input_kind") == "demo" else (output, compare):+ actual = VideoReader(str(path), ctx=cpu(), num_threads=2)+ if (+ len(actual) != geometry.source_frames+ or abs(actual.get_avg_fps() - fps) > 0.02+ ):+ raise ValueError("Written output frame count or FPS differs")+ actual[len(actual) - 1]+ del actual+ elapsed = time.perf_counter() - started+ atomic_json(+ manifest,+ dict(+ schema="solarwm.h3-full-test-result.v1",+ source_row=row,+ result_identity=identity,+ step=step,+ weights_id=weights_id,+ noise_seed=noise_seed,+ checkpoint_receipt_sha256=checkpoint_receipt_sha256,+ condition_sha256=condition_receipt["condition_sha256"],+ weights_source=args.weight_source,+ sp_size=8,+ world_size=8,+ geometry=asdict(geometry),+ fps=fps,+ model_fps=24,+ fps_policy=args.fps_policy,+ generation_seconds=generation_seconds,+ generation_fps=geometry.source_frames / generation_seconds,+ decode_write_seconds=decode_seconds,+ inference_end_to_end_seconds=elapsed,+ preparation_seconds=condition_receipt["preparation_seconds"],+ end_to_end_seconds=elapsed + condition_receipt["preparation_seconds"],+ end_to_end_fps=geometry.source_frames+ / (elapsed + condition_receipt["preparation_seconds"]),+ timing_excludes="model loading",+ nfe=4,+ forwards_per_chunk=5,+ window_chunks=6,+ chunk_latents=5,+ context_noise=0,+ camera_translation_transform="logd4",+ camera_intrinsics_mode="wan_fixed",+ student_rope_mode="sliding_local",+ kv_cache="raw_before_rope_and_prope",+ audio_condition_policy="fixed_noised_encoded_158f_silence_per_rollout",+ tail_camera_policy="repeat_last_available_latent_only_for_codec_tail",+ output_sha256=sha(output),+ compare_sha256=sha(compare)+ if row.get("input_kind") != "demo"+ else None,+ stream_decode=bool(cfg["inference"].get("stream_decode", False)),+ storyboard_sha256=sha(output.with_suffix(".storyboard.jpg")),+ latents_sha256=(+ sha(dest / (row["key"] + ".latents.safetensors"))+ if cfg["inference"].get("save_latents", False)+ else None+ ),+ visual_samples=visual,+ gpu_peak_memory_gib=torch.cuda.max_memory_allocated() / 2**30,+ ),+ )+ print("RESULT " + json.dumps(json.loads(manifest.read_text())), flush=True)+ finally:+ if streaming:+ import sys++ event = torch.tensor(+ 0 if sys.exc_info()[0] is None else -1, device=device, dtype=torch.int32+ )+ dist.broadcast(event, src=0)+ dist.barrier()+ del rollout, inputs, noise, values, anchor, audio, condition+ gc.collect()+ torch.cuda.empty_cache()+ finally:+ dist.destroy_process_group()+++def run_source_length_inference(config):+ """Public ``solarwm infer`` route: DP conditioning, then one SP8 rollout group."""+ from types import SimpleNamespace++ import torch+ import torch.distributed as dist++ from .fsdp import initialize_distributed++ inference = config["inference"]+ args = SimpleNamespace(+ config=config,+ plan=inference["plan"],+ work_dir=inference["work_dir"],+ dataset_root=inference["dataset_root"],+ model_dir=config["model"]["checkpoint_path"],+ checkpoint=config["checkpoint"]["resume_from"],+ weight_source=config["checkpoint"]["weight_source"],+ silence=config["data"]["silence_latents_path"],+ output_dir=config["runtime"]["output_dir"],+ expected_step=int(inference["expected_step"]),+ fps_policy=inference["fps_policy"],+ noise_seed=int(config["validation"]["noise_seed"]),+ )+ if int(os.environ.get("RANK", "0")) == 0:+ (Path(args.output_dir) / "WORKER_COMPLETE.json").unlink(missing_ok=True)+ rows = selected_rows(args)+ if int(os.environ.get("WORLD_SIZE", 0)) != 8 or int(os.environ.get("LOCAL_WORLD_SIZE", 0)) != 8:+ raise ValueError(+ "Source-length inference requires a single node with eight torchrun workers"+ )+ local = int(os.environ["LOCAL_RANK"])+ initialize_distributed(sp_size=8, local_rank=local)+ phase = inference.get("phase", "all")+ if phase in {"prepare", "all"}:+ prepare(args)+ gc.collect()+ torch.cuda.empty_cache()+ dist.barrier()+ if phase == "prepare":+ dist.destroy_process_group()+ return 0+ generate(args)+ if local == 0:+ atomic_json(+ Path(args.output_dir) / "WORKER_COMPLETE.json",+ {+ "schema": "solarwm.h3-source-length-worker.v1",+ "case_keys": [row["key"] for row in rows],+ "step": args.expected_step,+ "weight_source": args.weight_source,+ "entrypoint": "python -m solarwm infer",+ "config": config,+ },+ )+ return 0diff --git a/src/solarwm/backends/minimax_h3/full_length.py b/src/solarwm/backends/minimax_h3/full_length.pynew file mode 100644index 0000000..8955b79--- /dev/null+++ b/src/solarwm/backends/minimax_h3/full_length.py@@ -0,0 +1,45 @@+"""Source-length geometry for H3 SGF evaluation, independent of GPU libraries."""++from dataclasses import dataclass+++@dataclass(frozen=True)+class FullLengthGeometry:+ source_frames: int+ decoded_frames: int+ decode_latents: int+ rollout_latents: int+ camera_indices: tuple[int, ...]+ tail_camera_rows: int+++def full_length_geometry(source_frames: int) -> FullLengthGeometry:+ if isinstance(source_frames, bool) or not isinstance(source_frames, int) or source_frames < 22:+ raise ValueError("H3 evaluation requires at least 22 consecutive source frames")+ n = (source_frames - 5 + 16) // 17+ decode_latents = 5 * n + 2+ rollout_latents = (decode_latents + 4) // 5 * 5+ cadence = (1, 4, 4, 4, 4)+ indices = [0]+ tail = 0+ source_index = 0+ for index in range(rollout_latents - 1):+ source_index += cadence[index % 5]+ if source_index < source_frames:+ indices.append(source_index)+ else:+ indices.append(indices[-1])+ tail += 1+ return FullLengthGeometry(+ source_frames, 17 * n + 5, decode_latents, rollout_latents, tuple(indices), tail+ )+++def exported_fps(source_fps: float, policy: str) -> float:+ import math++ if not math.isfinite(source_fps) or source_fps <= 0:+ raise ValueError("Source FPS must be finite and positive")+ if policy not in {"source", "h3"}:+ raise ValueError("FPS policy must be source or h3")+ return float(source_fps) if policy == "source" else 24.0diff --git a/src/solarwm/backends/minimax_h3/geometry.py b/src/solarwm/backends/minimax_h3/geometry.pynew file mode 100644index 0000000..c2e881d--- /dev/null+++ b/src/solarwm/backends/minimax_h3/geometry.py@@ -0,0 +1,276 @@+"""MiniMax-H3 video geometry and native three-axis position grids.++This module is intentionally NumPy-only. Importing the H3 plugin must not import+PyTorch, Diffusers, or materialize the 33B transformer.+"""++from __future__ import annotations++import math+from dataclasses import dataclass++import numpy as np+++@dataclass(frozen=True)+class H3Geometry:+ """MiniMax-H3 VisualVAE and transformer geometry."""++ pixel_frames_per_chunk: int = 17+ latent_frames_per_chunk: int = 5+ latent_prefix_frames: int = 2+ vae_spatial_compression: int = 16+ transformer_patch: tuple[int, int, int] = (1, 2, 2)+ canvas_multiple: int = 32+ semantic_fps: int = 24+ audio_latents_per_second: int = 40+ audio_channels: int = 2+ rope_frame_rescale: float = 5.0 / 3.0+ rope_frames_per_latent: tuple[int, ...] = (1, 4, 4, 4, 4)+++DEFAULT_GEOMETRY = H3Geometry()+++@dataclass(frozen=True)+class H3Stage0p5Geometry:+ """The supported Stage0.5 SP2 geometry."""++ pixel_frames: int = 158+ encoded_latents: int = 47+ height: int = 768+ width: int = 1344+ latent_channels: int = 24+ latent_height: int = 48+ latent_width: int = 84+ patch_size: tuple[int, int, int] = (1, 2, 2)++ @property+ def rows_per_latent(self) -> int:+ patch_t, patch_h, patch_w = self.patch_size+ if patch_t != 1:+ raise ValueError("MiniMax-H3 requires temporal patch size 1")+ return (self.latent_height // patch_h) * (self.latent_width // patch_w)+++STABLE_STAGE0P5_GEOMETRY = H3Stage0p5Geometry()+++def align_pixel_frames(+ num_frames: int,+ geometry: H3Geometry = DEFAULT_GEOMETRY,+) -> int:+ """Round up to the next valid ``17*n + 5`` VisualVAE length."""++ if num_frames < 1:+ raise ValueError(f"num_frames must be positive, got {num_frames}")+ frames = int(num_frames)+ while frames % geometry.pixel_frames_per_chunk != geometry.latent_frames_per_chunk:+ frames += 1+ return frames+++def pixel_frames_to_latent_frames(+ num_frames: int,+ geometry: H3Geometry = DEFAULT_GEOMETRY,+) -> int:+ """Map an exact ``17*n + 5`` pixel length to ``5*n + 2`` latents."""++ fpc = geometry.pixel_frames_per_chunk+ lpc = geometry.latent_frames_per_chunk+ if num_frames < lpc or num_frames % fpc != lpc:+ raise ValueError(f"num_frames must have form {fpc}*n+{lpc}, got {num_frames}")+ return (num_frames - lpc) // fpc * lpc + geometry.latent_prefix_frames+++def latent_frames_to_pixel_frames(+ num_latent_frames: int,+ geometry: H3Geometry = DEFAULT_GEOMETRY,+) -> int:+ """Invert :func:`pixel_frames_to_latent_frames` on exact lengths."""++ prefix = geometry.latent_prefix_frames+ lpc = geometry.latent_frames_per_chunk+ if num_latent_frames < prefix or (num_latent_frames - prefix) % lpc:+ raise ValueError(+ f"num_latent_frames must have form {lpc}*n+{prefix}, got {num_latent_frames}"+ )+ chunks = (num_latent_frames - prefix) // lpc+ return chunks * geometry.pixel_frames_per_chunk + lpc+++def latent_aligned_pixel_indices(+ num_frames: int,+ geometry: H3Geometry = DEFAULT_GEOMETRY,+) -> np.ndarray:+ """Map encoded latent positions to exact source-pixel frame indices.++ H3 repeats offsets ``[0,1,5,9,13]`` every 17 input frames and retains+ in-range offsets from the final partial period. For 158 frames this+ returns 47 entries and never retimes the source clip.+ """++ expected_latents = pixel_frames_to_latent_frames(num_frames, geometry)+ offsets = np.asarray((0, 1, 5, 9, 13), dtype=np.int64)+ indices: list[np.ndarray] = []+ for base in range(0, num_frames, geometry.pixel_frames_per_chunk):+ period = base + offsets+ indices.append(period[period < num_frames])+ result = np.concatenate(indices)+ if result.shape != (expected_latents,):+ raise RuntimeError("H3 pixel/latent alignment is internally inconsistent")+ return result+++def audio_latents_for_video(+ num_frames: int,+ *,+ fps: float = DEFAULT_GEOMETRY.semantic_fps,+ latents_per_second: int = DEFAULT_GEOMETRY.audio_latents_per_second,+) -> int:+ """Return per-channel audio latents on H3's rounded semantic timeline."""++ if num_frames < 1 or fps <= 0 or latents_per_second <= 0:+ raise ValueError("num_frames, fps, and latents_per_second must be positive")+ return round(num_frames / fps * latents_per_second)+++def validate_canvas(height: int, width: int, *, multiple: int = 32) -> tuple[int, int]:+ """Reject rather than round a canvas outside the checkpoint contract."""++ if height <= 0 or width <= 0:+ raise ValueError(f"height and width must be positive, got {(height, width)}")+ if multiple <= 0 or height % multiple or width % multiple:+ raise ValueError(+ f"height and width must both be multiples of {multiple}, got {(height, width)}"+ )+ return int(height), int(width)+++def validate_stage0p5_geometry(+ *,+ pixel_frames: int,+ encoded_latents: int,+ height: int,+ width: int,+ latent_channels: int,+ latent_height: int,+ latent_width: int,+) -> H3Stage0p5Geometry:+ """Validate the 158f/47-latent 768x1344 profile."""++ expected = STABLE_STAGE0P5_GEOMETRY+ observed = H3Stage0p5Geometry(+ pixel_frames=int(pixel_frames),+ encoded_latents=int(encoded_latents),+ height=int(height),+ width=int(width),+ latent_channels=int(latent_channels),+ latent_height=int(latent_height),+ latent_width=int(latent_width),+ )+ if observed != expected:+ raise ValueError(+ "MiniMax-H3 Stage0.5 SP2 supports only "+ "158f -> 47 latents at 768x1344 with [24,47,48,84] latents; "+ f"got {observed}"+ )+ if pixel_frames_to_latent_frames(observed.pixel_frames) != observed.encoded_latents:+ raise ValueError("pixel/latent temporal geometry is internally inconsistent")+ validate_canvas(observed.height, observed.width)+ if observed.height // DEFAULT_GEOMETRY.vae_spatial_compression != observed.latent_height:+ raise ValueError("height does not match VisualVAE spatial compression")+ if observed.width // DEFAULT_GEOMETRY.vae_spatial_compression != observed.latent_width:+ raise ValueError("width does not match VisualVAE spatial compression")+ if observed.rows_per_latent != 1008:+ raise ValueError("H3 transformer geometry must have 1008 rows per latent")+ return observed+++def spatial_position_axis(dim: int, patch: int, sqrt_area: float) -> np.ndarray:+ """Build the provider-compatible float64 endpoint-excluded spatial axis."""++ if dim <= 0 or patch <= 0 or dim % patch:+ raise ValueError(f"dim={dim} must be positive and divisible by patch={patch}")+ ratio = dim / sqrt_area+ left = (1.0 - ratio) / 2.0+ return np.linspace(left, left + ratio, dim // patch, endpoint=False) * 32+++def frame_position_grid(+ latent_height: int,+ latent_width: int,+ patch_h: int = 2,+ patch_w: int = 2,+) -> tuple[np.ndarray, np.ndarray]:+ """Return flattened ``(h,w)`` coordinates and the width axis for a frame."""++ sqrt_area = math.sqrt(latent_height * latent_width)+ height_grid = spatial_position_axis(latent_height, patch_h, sqrt_area)+ width_grid = spatial_position_axis(latent_width, patch_w, sqrt_area)+ grid_h, grid_w = np.meshgrid(height_grid, width_grid, indexing="ij")+ return np.stack((grid_h.reshape(-1), grid_w.reshape(-1)), axis=-1), width_grid+++def temporal_position_grid(num_latent_frames: int, origin: float = 0.0) -> np.ndarray:+ """Return H3's non-uniform ``5/3*(1,4,4,4,4)`` rotary-time grid."""++ if num_latent_frames < 0:+ raise ValueError(f"num_latent_frames must be non-negative, got {num_latent_frames}")+ spans = np.asarray(+ [+ DEFAULT_GEOMETRY.rope_frame_rescale+ * DEFAULT_GEOMETRY.rope_frames_per_latent[+ index % len(DEFAULT_GEOMETRY.rope_frames_per_latent)+ ]+ for index in range(num_latent_frames)+ ],+ dtype=np.float64,+ )+ if num_latent_frames == 0:+ return spans+ result = np.empty(num_latent_frames, dtype=np.float64)+ result[0] = float(origin)+ if num_latent_frames > 1:+ result[1:] = float(origin) + np.cumsum(spans[:-1])+ return result+++def native_video_position_grid(+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ *,+ patch_size: tuple[int, int, int] = (1, 2, 2),+ origin: float = 0.0,+) -> np.ndarray:+ """Return flattened native H3 ``(t,h,w)`` IDs in frame-major order."""++ patch_t, patch_h, patch_w = patch_size+ if patch_t != 1:+ raise ValueError("H3 MM-RoPE layout requires temporal patch_size=1")+ frame_grid, _ = frame_position_grid(latent_height, latent_width, patch_h, patch_w)+ times = temporal_position_grid(num_latent_frames, origin)+ result = np.empty((num_latent_frames, frame_grid.shape[0], 3), dtype=np.float64)+ result[:, :, 0] = times[:, None]+ result[:, :, 1:] = frame_grid[None]+ return result.reshape(-1, 3)+++__all__ = [+ "DEFAULT_GEOMETRY",+ "STABLE_STAGE0P5_GEOMETRY",+ "H3Geometry",+ "H3Stage0p5Geometry",+ "align_pixel_frames",+ "audio_latents_for_video",+ "frame_position_grid",+ "latent_aligned_pixel_indices",+ "latent_frames_to_pixel_frames",+ "native_video_position_grid",+ "pixel_frames_to_latent_frames",+ "spatial_position_axis",+ "temporal_position_grid",+ "validate_canvas",+ "validate_stage0p5_geometry",+]diff --git a/src/solarwm/backends/minimax_h3/inference.py b/src/solarwm/backends/minimax_h3/inference.pynew file mode 100644index 0000000..cc4ca20--- /dev/null+++ b/src/solarwm/backends/minimax_h3/inference.py@@ -0,0 +1,241 @@+"""MiniMax-H3 generation packaging and shared-engine inference adapter."""++from __future__ import annotations++import hashlib+import tempfile+from collections.abc import Mapping+from pathlib import Path+from typing import Any++from solarwm.inference import GeneratedSample, InferenceCase, encode_compare_mp4++from .artifacts import H3ArtifactBatch+from .stage0p5 import H3Stage0p5Core+++def camera_fingerprint(batch: H3ArtifactBatch) -> str:+ digest = hashlib.blake2s()+ digest.update(batch.camera_viewmats.detach().cpu().numpy().tobytes())+ digest.update(batch.camera_K.detach().cpu().numpy().tobytes())+ if batch.rollout_camera_viewmats is not None:+ digest.update(batch.rollout_camera_viewmats.detach().cpu().numpy().tobytes())+ digest.update(batch.rollout_camera_K.detach().cpu().numpy().tobytes())+ if batch.geometry_latents:+ # A re-rendered geometry video is a different case even with the same+ # camera; frozen validation plans must not silently reuse the old one.+ import torch++ for key in sorted(batch.geometry_latents):+ value = batch.geometry_latents[key].detach().cpu().contiguous()+ digest.update(key.encode("utf-8"))+ digest.update(value.view(torch.uint8).numpy().tobytes())+ return digest.hexdigest()+++def decoded_pixel_frames(latent_frames: int) -> int:+ """The VisualVAE writes ``17n+5`` pixel frames for the ``5n+2`` latent frames it was given."""++ if latent_frames < 2 or (latent_frames - 2) % 5:+ raise RuntimeError(f"H3 latents hold {latent_frames} frames, which is not 5n+2")+ return 17 * ((latent_frames - 2) // 5) + 5+++def decode_h3_latents(video_vae: Any, latents: Any, *, device: Any) -> Any:+ """Run official latent de-normalization and the ``5n+2`` -> ``17n+5`` VisualVAE decode."""++ import torch++ mean = torch.tensor(video_vae.config.latents_mean, device=device).view(1, -1, 1, 1, 1)+ std = torch.tensor(video_vae.config.latents_std, device=device).view(1, -1, 1, 1, 1)+ decoder_input = latents.to(device=device, dtype=torch.float32) * std + mean+ with (+ torch.no_grad(),+ torch.autocast(+ device_type=device.type,+ dtype=torch.float16,+ enabled=device.type == "cuda",+ ),+ ):+ decoded = video_vae.decode(decoder_input, return_dict=False)[0]+ pixel_mean = torch.tensor((0.485, 0.456, 0.406), device=device).view(1, -1, 1, 1, 1)+ pixel_std = torch.tensor((0.229, 0.224, 0.225), device=device).view(1, -1, 1, 1, 1)+ decoded = (decoded.float() * pixel_std + pixel_mean).clamp(0, 1)+ expected = (+ 1,+ 3,+ decoded_pixel_frames(int(latents.shape[-3])),+ int(latents.shape[-2]) * 16,+ int(latents.shape[-1]) * 16,+ )+ if tuple(decoded.shape) != expected:+ raise RuntimeError(+ f"H3 VisualVAE decoded {tuple(decoded.shape)}, expected {list(expected)}"+ )+ return decoded+++def _mp4_bytes(decoded: Any, *, fps: int = 24) -> bytes:+ import imageio.v2 as imageio+ import numpy as np+ import torch++ frames = (+ decoded[0].permute(1, 2, 3, 0).mul(255).round().clamp(0, 255).to(torch.uint8).cpu().numpy()+ )+ with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as temporary:+ path = Path(temporary.name)+ try:+ imageio.mimwrite(+ str(path),+ [np.asarray(frame) for frame in frames],+ fps=int(fps),+ quality=8,+ macro_block_size=1,+ ffmpeg_params=["-pix_fmt", "yuv420p", "-movflags", "+faststart"],+ )+ return path.read_bytes()+ finally:+ path.unlink(missing_ok=True)+++class H3InferenceAdapter:+ family = "minimax_h3"++ def __init__(+ self,+ *,+ core: H3Stage0p5Core,+ video_vae: Any,+ batches: Mapping[str, H3ArtifactBatch],+ device: Any,+ num_inference_steps: int,+ ) -> None:+ self.core = core+ self.video_vae = video_vae+ self.batches = dict(batches)+ self.device = device+ self.num_inference_steps = int(num_inference_steps)++ def generate(self, case: InferenceCase, *, weights_id: str) -> GeneratedSample:+ try:+ batch = self.batches[case.sample_id]+ except KeyError as exc:+ raise ValueError(f"no H3 artifact batch for case {case.sample_id!r}") from exc+ if camera_fingerprint(batch) != case.camera_fingerprint:+ raise ValueError("H3 inference case camera identity differs")+ latents = self.core.generate(+ batch,+ noise_seed=case.noise_seed,+ num_inference_steps=self.num_inference_steps,+ )+ return package_generated(+ latents,+ video_vae=self.video_vae,+ device=self.device,+ weights_id=weights_id,+ num_inference_steps=self.num_inference_steps,+ reference_latents=batch.target_latents,+ )+++def package_generated(+ latents: Any,+ *,+ video_vae: Any,+ device: Any,+ weights_id: str,+ num_inference_steps: int,+ reference_latents: Any | None = None,+ sample_solver: str = "shifted-euler-data-ward",+) -> GeneratedSample:+ """Decode/package already-generated latents on the single artifact writer."""++ import torch+ from safetensors.torch import save++ decoded = decode_h3_latents(video_vae, latents, device=device)+ ground_truth = None+ if reference_latents is not None:+ reference = reference_latents+ if reference.ndim == 4:+ reference = reference.unsqueeze(0)+ ground_truth = decode_h3_latents(video_vae, reference, device=device)+ metrics = _finite_generation_metrics(+ latents=latents,+ decoded=decoded,+ reference_decoded=ground_truth,+ )+ artifacts = {+ "generated.safetensors": save(+ {"latents": latents.detach().to("cpu", dtype=torch.bfloat16)}+ ),+ "generated.mp4": _mp4_bytes(decoded),+ }+ if ground_truth is not None:+ artifacts["compare.mp4"] = encode_compare_mp4(+ ground_truth,+ decoded,+ fps=24,+ layout="bcthw",+ value_range="zero_one",+ )+ return GeneratedSample(+ artifacts=artifacts,+ shape=tuple(int(value) for value in decoded.shape),+ dtype="float32",+ metrics=metrics,+ provenance={+ "weights_id": weights_id,+ "preencode_version": "h3.158f.v1",+ "sampler": sample_solver,+ "solver": sample_solver,+ "num_inference_steps": int(num_inference_steps),+ "num_sigma_points": int(num_inference_steps),+ "video_shift": 12.0,+ "audio_shift": 3.0,+ },+ )+++def _finite_generation_metrics(+ *,+ latents: Any,+ decoded: Any,+ reference_decoded: Any | None,+) -> dict[str, float]:+ """Fail before publication unless generated latent/decoded tensors are finite."""++ import torch++ values = {+ "latent": latents.detach(),+ "decoded": decoded.detach(),+ }+ if reference_decoded is not None:+ values["reference_decoded"] = reference_decoded.detach()+ for name, value in values.items():+ if not bool(torch.isfinite(value).all().item()):+ raise RuntimeError(f"H3 inference {name} contains NaN or Inf")+ metrics = {+ "finite_fraction": 1.0,+ "latent_finite_fraction": 1.0,+ "decoded_finite_fraction": 1.0,+ "latent_min": float(values["latent"].min().item()),+ "latent_max": float(values["latent"].max().item()),+ "latent_mean": float(values["latent"].mean(dtype=torch.float32).item()),+ "decoded_min": float(values["decoded"].min().item()),+ "decoded_max": float(values["decoded"].max().item()),+ "decoded_mean": float(values["decoded"].mean(dtype=torch.float32).item()),+ }+ if reference_decoded is not None:+ metrics["reference_decoded_finite_fraction"] = 1.0+ return metrics+++__all__ = [+ "H3InferenceAdapter",+ "camera_fingerprint",+ "decode_h3_latents",+ "package_generated",+]diff --git a/src/solarwm/backends/minimax_h3/layout.py b/src/solarwm/backends/minimax_h3/layout.pynew file mode 100644index 0000000..12c3103--- /dev/null+++ b/src/solarwm/backends/minimax_h3/layout.py@@ -0,0 +1,755 @@+"""Dependency-light MiniMax-H3 packed-row layouts.++The Stage0.5 document is ordered as::++ [Qwen joint image/text | VisualVAE anchor | visual observations | audio | target video]++A visual observation is any image or video latent, of any size and length, and+carries no type. Only its placement says how it relates to the clip:++* *aligned* with a list of target latent frames: its frames share those frames'+ rotary times and camera poses, and its spatial grid is laid over the target's+ canvas whatever its resolution or aspect ratio. A depth, pose or edge video, the+ source of an edit, a keyframe, a low-resolution preview are all this case. An+ optional box per frame lays it over a part of the canvas instead: a face crop+ that follows a face, a detail patch, an object pasted at a location.+* *free* (no frames): it is not a view of the clip. It sits before the clip on the+ rotary time axis, in order, on its own spatial grid, and never takes part in+ camera PRoPE. A subject, an object, a scene, a style, a motion clip. The clock+ is the one of the released ``ref2va`` layout: it starts where the text ends, an+ image advances it by one slot, a video by its own rotary span, and the clip+ (audio, anchors and target alike) starts where the free observations left it.++Qwen vision rows may carry ``VIDEO_TAG`` while still belonging to the text+segment. Camera PRoPE must therefore use ``camera_video_indices`` and never+infer camera rows from token tags.+"""++from __future__ import annotations++from dataclasses import dataclass, fields, replace++import numpy as np++from .geometry import frame_position_grid, temporal_position_grid++VIDEO_TAG = 0+TEXT_TAG = 1+AUDIO_TAG = 2++PADDING_ROLE = -1+CONDITION_ROLE = 0+AUDIO_ROLE = 1+CLEAN_VIDEO_ROLE = 2+NOISY_VIDEO_ROLE = 3+TARGET_AUDIO_ROLE = AUDIO_ROLE+TARGET_VIDEO_ROLE = NOISY_VIDEO_ROLE+++@dataclass(frozen=True)+class H3VisualObservationSpec:+ """Shape and placement of one visual observation; ``frames=None`` is a free one."""++ num_frames: int+ height: int+ width: int+ frames: tuple[int, ...] | None = None+ # ``(x0, y0, x1, y1)`` in the target's normalized canvas, one per frame or one for all.+ boxes: tuple[tuple[float, float, float, float], ...] | None = None+++def _canvas_grid(+ item: H3VisualObservationSpec,+ box: tuple[float, float, float, float],+ target_height: int,+ target_width: int,+ patch_h: int,+ patch_w: int,+) -> np.ndarray:+ """Spatial positions of one observation frame laid over ``box`` of the target canvas."""++ x0, y0, x1, y1 = (float(value) for value in box)+ if not (0.0 <= x0 < x1 <= 1.0 and 0.0 <= y0 < y1 <= 1.0):+ raise ValueError(f"visual observation box {box} must lie inside the unit canvas")+ sqrt_area = np.sqrt(target_height * target_width)+ axes = []+ for size, low, high, count in (+ (target_height, y0, y1, item.height // patch_h),+ (target_width, x0, x1, item.width // patch_w),+ ):+ ratio = size / sqrt_area+ left = (1.0 - ratio) / 2.0+ steps = low + (high - low) * np.arange(count, dtype=np.float64) / count+ axes.append((left + ratio * steps) * 32)+ grid_h, grid_w = np.meshgrid(axes[0], axes[1], indexing="ij")+ return np.stack((grid_h.reshape(-1), grid_w.reshape(-1)), axis=-1)+++def _indices(start: int, stop: int) -> np.ndarray:+ return np.arange(start, stop, dtype=np.int64)+++def _empty_indices() -> np.ndarray:+ return np.empty((0,), dtype=np.int64)+++@dataclass(frozen=True)+class H3PackedLayout:+ """All structural arrays for one packed H3 attention document."""++ position_ids: np.ndarray+ token_tags: np.ndarray+ video_indices: np.ndarray+ audio_indices: np.ndarray+ text_indices: np.ndarray+ condition_indices: np.ndarray+ target_indices: np.ndarray+ condition_video_indices: np.ndarray+ audio_condition_indices: np.ndarray+ clean_video_indices: np.ndarray+ noisy_video_indices: np.ndarray+ target_video_indices: np.ndarray+ target_audio_indices: np.ndarray+ camera_video_indices: np.ndarray+ camera_frame_ids: np.ndarray+ row_roles: np.ndarray+ target_video_chunk_ids: np.ndarray+ num_condition_video_rows: int+ num_condition_audio_rows: int+ num_clean_video_rows: int+ num_noisy_video_rows: int+ rows_per_video_frame: int+ latent_height: int+ latent_width: int+ patch_size: tuple[int, int, int]+ stage: str+ window_chunks: int | None = None+ chunk_latent_frames: int | None = None+ # Visual observations are the tail of ``condition_video_indices``; row counts per item.+ observation_rows: tuple[int, ...] = ()++ @property+ def num_observation_rows(self) -> int:+ return int(sum(self.observation_rows))++ @property+ def observation_video_indices(self) -> np.ndarray:+ return self.condition_video_indices[+ self.num_condition_video_rows - self.num_observation_rows :+ ]++ @property+ def sequence_length(self) -> int:+ return int(self.position_ids.shape[0])++ @property+ def target_video_output_slice(self) -> slice:+ """Select target rows from an output already filtered to video rows."""++ start = self.num_condition_video_rows + self.num_clean_video_rows+ return slice(start, start + self.num_noisy_video_rows)++ @property+ def noisy_video_output_slice(self) -> slice:+ return self.target_video_output_slice++ @property+ def clean_video_output_slice(self) -> slice:+ start = self.num_condition_video_rows+ return slice(start, start + self.num_clean_video_rows)++ @property+ def condition_video_output_slice(self) -> slice:+ return slice(0, self.num_condition_video_rows)++ @property+ def camera_indices(self) -> np.ndarray:+ return self.camera_video_indices++ def to(self, device: object) -> H3PackedLayout:+ """Transfer structural arrays to torch without changing the public layout."""+ import torch++ values = {}+ for item in fields(self):+ value = getattr(self, item.name)+ if isinstance(value, np.ndarray):+ values[item.name] = torch.from_numpy(value).to(device)+ elif isinstance(value, torch.Tensor):+ values[item.name] = value.to(device)+ return replace(self, **values)++ def transformer_kwargs(self) -> dict[str, np.ndarray]:+ """Return the five native structural arguments expected by H3."""++ return {+ "position_ids": self.position_ids,+ "token_tags": self.token_tags,+ "video_indices": self.video_indices,+ "audio_indices": self.audio_indices,+ "text_indices": self.text_indices,+ }+++def _validate_layout_inputs(+ text_token_tags: object,+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ num_audio_latents: int,+ patch_size: tuple[int, int, int],+ audio_channels: int,+) -> tuple[np.ndarray, int]:+ tags = np.asarray(text_token_tags, dtype=np.int64)+ if tags.ndim != 1:+ raise ValueError(f"text_token_tags must be one-dimensional, got {tags.shape}")+ patch_t, patch_h, patch_w = patch_size+ if patch_t != 1 or latent_height % patch_h or latent_width % patch_w:+ raise ValueError(+ f"latent grid {(num_latent_frames, latent_height, latent_width)} "+ f"is incompatible with patch {patch_size}"+ )+ if num_latent_frames <= 0 or num_audio_latents < 0 or audio_channels <= 0:+ raise ValueError("latent frame counts/channels are invalid")+ if audio_channels != 2 and num_audio_latents:+ raise ValueError("native H3 audio packing requires two channel-major channels")+ return tags, (latent_height // patch_h) * (latent_width // patch_w)+++def _base_layout(+ text_token_tags: object,+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ num_audio_latents: int,+ *,+ patch_size: tuple[int, int, int],+ audio_channels: int,+ keyframe_anchors: tuple[str, ...],+ stage: str,+ chunk_latent_frames: int | None,+ window_chunks: int | None,+ observations: tuple[H3VisualObservationSpec, ...] = (),+) -> H3PackedLayout:+ tags, rows_per_frame = _validate_layout_inputs(+ text_token_tags,+ num_latent_frames,+ latent_height,+ latent_width,+ num_audio_latents,+ patch_size,+ audio_channels,+ )+ _, patch_h, patch_w = patch_size+ num_text = int(tags.size)+ observation_rows = []+ for item in observations:+ if item.num_frames <= 0 or item.height <= 0 or item.width <= 0:+ raise ValueError(f"visual observation {item} is empty")+ if item.height % patch_h or item.width % patch_w:+ raise ValueError(f"visual observation grid {item} does not fit {patch_size}")+ if item.frames is not None and (+ len(item.frames) != item.num_frames+ or min(item.frames) < 0+ or max(item.frames) >= num_latent_frames+ ):+ raise ValueError(f"visual observation {item} is not aligned with the target frames")+ if item.boxes is not None and (+ item.frames is None or len(item.boxes) not in (1, item.num_frames)+ ):+ raise ValueError(f"visual observation {item} needs frames and one box per frame")+ observation_rows.append(+ item.num_frames * (item.height // patch_h) * (item.width // patch_w)+ )+ num_anchor_video = len(keyframe_anchors) * rows_per_frame+ num_condition_video = num_anchor_video + sum(observation_rows)+ num_audio = num_audio_latents * audio_channels+ num_target_video = num_latent_frames * rows_per_frame+ condition_start = num_text+ audio_start = condition_start + num_condition_video+ target_start = audio_start + num_audio+ sequence_length = target_start + num_target_video++ position_ids = np.zeros((sequence_length, 3), dtype=np.float64)+ position_ids[:num_text, 0] = np.arange(num_text, dtype=np.float64)+ frame_grid, width_grid = frame_position_grid(latent_height, latent_width, patch_h, patch_w)++ # Free observations advance the rotary clock ahead of the clip, like the released ref2va layout.+ origin = float(num_text)+ free_times: list[np.ndarray | None] = []+ for item in observations:+ if item.frames is not None:+ free_times.append(None)+ elif item.num_frames == 1:+ free_times.append(np.asarray([origin]))+ origin += 1.0+ else:+ free_times.append(temporal_position_grid(item.num_frames, origin))+ origin += float(np.diff(temporal_position_grid(item.num_frames + 1, 0.0)).sum())++ anchor_frame_ids: list[int] = []+ temporal = temporal_position_grid(num_latent_frames, origin)+ for index, anchor in enumerate(keyframe_anchors):+ if anchor == "first":+ anchor_time = origin+ anchor_frame_ids.append(0)+ elif anchor == "last":+ anchor_time = float(+ temporal_position_grid(num_latent_frames + 1, origin)[-1] - 5.0 / 3.0+ )+ anchor_frame_ids.append(num_latent_frames - 1)+ else:+ raise ValueError(f"keyframe anchor must be 'first' or 'last', got {anchor!r}")+ start = condition_start + index * rows_per_frame+ rows = slice(start, start + rows_per_frame)+ position_ids[rows, 0] = anchor_time+ position_ids[rows, 1:] = frame_grid++ # Aligned observations take their frames' rotary times and the target's canvas; free+ # ones take their slot on the clock and keep their own spatial grid.+ start = condition_start + num_anchor_video+ observation_camera_frames: list[np.ndarray] = []+ observation_camera_rows: list[np.ndarray] = []+ for item, count, free in zip(observations, observation_rows, free_times, strict=True):+ per_frame = count // item.num_frames+ if free is not None:+ frame_times = free+ else:+ frame_times = temporal[np.asarray(item.frames, dtype=np.int64)]+ observation_camera_rows.append(_indices(start, start + count))+ observation_camera_frames.append(+ np.repeat(np.asarray(item.frames, dtype=np.int64), per_frame)+ )+ rows = slice(start, start + count)+ position_ids[rows, 0] = np.repeat(frame_times, per_frame)+ if item.frames is None:+ grid = frame_position_grid(item.height, item.width, patch_h, patch_w)[0]+ position_ids[rows, 1:] = np.tile(grid, (item.num_frames, 1))+ else:+ boxes = item.boxes or ((0.0, 0.0, 1.0, 1.0),)+ grids = [+ _canvas_grid(item, box, latent_height, latent_width, patch_h, patch_w)+ for box in boxes+ ]+ position_ids[rows, 1:] = (+ np.tile(grids[0], (item.num_frames, 1))+ if len(grids) == 1+ else np.concatenate(grids, axis=0)+ )+ start += count++ if num_audio:+ audio_rows = slice(audio_start, target_start)+ audio_time = origin + np.arange(num_audio_latents, dtype=np.float64)+ position_ids[audio_rows, 0] = np.tile(audio_time, audio_channels)+ position_ids[audio_rows, 2] = np.concatenate(+ (+ np.full(num_audio_latents, width_grid[0], dtype=np.float64),+ np.full(num_audio_latents, width_grid[-1], dtype=np.float64),+ )+ )++ target_positions = np.empty((num_latent_frames, rows_per_frame, 3), dtype=np.float64)+ target_positions[:, :, 0] = temporal[:, None]+ target_positions[:, :, 1:] = frame_grid[None]+ position_ids[target_start:] = target_positions.reshape(-1, 3)++ text_indices = _indices(0, num_text)+ condition_video_indices = _indices(condition_start, audio_start)+ audio_indices = _indices(audio_start, target_start)+ target_video_indices = _indices(target_start, sequence_length)+ video_indices = np.concatenate((condition_video_indices, target_video_indices))++ token_tags = np.empty(sequence_length, dtype=np.int64)+ token_tags[text_indices] = tags+ token_tags[video_indices] = VIDEO_TAG+ token_tags[audio_indices] = AUDIO_TAG+ row_roles = np.full(sequence_length, CONDITION_ROLE, dtype=np.int64)+ row_roles[audio_indices] = AUDIO_ROLE+ row_roles[target_video_indices] = NOISY_VIDEO_ROLE++ condition_indices = np.concatenate((text_indices, condition_video_indices))+ target_indices = np.concatenate((audio_indices, target_video_indices))+ target_chunk_ids = np.full(sequence_length, -1, dtype=np.int64)+ if chunk_latent_frames is not None:+ if chunk_latent_frames <= 0 or num_latent_frames % chunk_latent_frames:+ raise ValueError(+ f"target length {num_latent_frames} must be divisible by "+ f"chunk_latent_frames={chunk_latent_frames}"+ )+ frame_chunks = np.arange(num_latent_frames, dtype=np.int64) // chunk_latent_frames+ target_chunk_ids[target_video_indices] = np.repeat(frame_chunks, rows_per_frame)++ condition_camera_frames = (+ np.repeat(np.asarray(anchor_frame_ids, dtype=np.int64), rows_per_frame)+ if anchor_frame_ids+ else _empty_indices()+ )+ target_camera_frames = np.repeat(np.arange(num_latent_frames, dtype=np.int64), rows_per_frame)+ # Aligned observations are views along the trajectory; free ones are not.+ camera_video_indices = np.concatenate(+ (+ condition_video_indices[:num_anchor_video],+ *observation_camera_rows,+ target_video_indices,+ )+ )+ camera_frame_ids = np.concatenate(+ (condition_camera_frames, *observation_camera_frames, target_camera_frames)+ )++ return H3PackedLayout(+ position_ids=position_ids,+ token_tags=token_tags,+ video_indices=video_indices,+ audio_indices=audio_indices,+ text_indices=text_indices,+ condition_indices=condition_indices,+ target_indices=target_indices,+ condition_video_indices=condition_video_indices,+ audio_condition_indices=_empty_indices(),+ clean_video_indices=_empty_indices(),+ noisy_video_indices=target_video_indices,+ target_video_indices=target_video_indices,+ target_audio_indices=audio_indices,+ camera_video_indices=camera_video_indices,+ camera_frame_ids=camera_frame_ids,+ row_roles=row_roles,+ target_video_chunk_ids=target_chunk_ids,+ num_condition_video_rows=num_condition_video,+ num_condition_audio_rows=0,+ num_clean_video_rows=0,+ num_noisy_video_rows=num_target_video,+ rows_per_video_frame=rows_per_frame,+ latent_height=latent_height,+ latent_width=latent_width,+ patch_size=patch_size,+ stage=stage,+ window_chunks=window_chunks,+ chunk_latent_frames=chunk_latent_frames,+ observation_rows=tuple(observation_rows),+ )+++def build_stage0p5_layout(+ text_token_tags: object,+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ num_audio_latents: int = 0,+ *,+ patch_size: tuple[int, int, int] = (1, 2, 2),+ audio_channels: int = 2,+ keyframe_anchors: tuple[str, ...] = ("first",),+ observations: tuple[H3VisualObservationSpec, ...] = (),+) -> H3PackedLayout:+ """Build Stage0.5's full-attention packed document."""++ return _base_layout(+ text_token_tags,+ num_latent_frames,+ latent_height,+ latent_width,+ num_audio_latents,+ patch_size=patch_size,+ audio_channels=audio_channels,+ keyframe_anchors=keyframe_anchors,+ stage="stage0p5",+ chunk_latent_frames=None,+ window_chunks=None,+ observations=observations,+ )+++def build_stage1_layout(+ text_token_tags: object,+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ num_audio_latents: int = 0,+ *,+ patch_size: tuple[int, int, int] = (1, 2, 2),+ audio_channels: int = 2,+ keyframe_anchors: tuple[str, ...] = ("first",),+ chunk_latent_frames: int = 5,+ window_chunks: int = 6,+) -> H3PackedLayout:+ """Build a six-chunk Stage1 row layout."""++ if window_chunks <= 0:+ raise ValueError(f"window_chunks must be positive, got {window_chunks}")+ base = _base_layout(+ text_token_tags,+ num_latent_frames,+ latent_height,+ latent_width,+ num_audio_latents,+ patch_size=patch_size,+ audio_channels=audio_channels,+ keyframe_anchors=keyframe_anchors,+ stage="stage1",+ chunk_latent_frames=chunk_latent_frames,+ window_chunks=window_chunks,+ )+ target_positions = base.position_ids[base.target_video_indices]+ target_start = int(base.target_video_indices[0])+ num_target_rows = int(base.target_video_indices.size)+ clean_start = target_start+ noisy_start = clean_start + num_target_rows+ sequence_length = noisy_start + num_target_rows++ position_ids = np.concatenate(+ (base.position_ids[:target_start], target_positions, target_positions), axis=0+ )+ clean_indices = _indices(clean_start, noisy_start)+ noisy_indices = _indices(noisy_start, sequence_length)+ video_indices = np.concatenate((base.condition_video_indices, clean_indices, noisy_indices))+ token_tags = np.empty(sequence_length, dtype=np.int64)+ token_tags[base.text_indices] = base.token_tags[base.text_indices]+ token_tags[base.audio_indices] = AUDIO_TAG+ token_tags[video_indices] = VIDEO_TAG+ row_roles = np.full(sequence_length, CONDITION_ROLE, dtype=np.int64)+ row_roles[base.audio_indices] = AUDIO_ROLE+ row_roles[clean_indices] = CLEAN_VIDEO_ROLE+ row_roles[noisy_indices] = NOISY_VIDEO_ROLE++ frame_chunks = np.repeat(+ np.arange(num_latent_frames, dtype=np.int64) // chunk_latent_frames,+ base.rows_per_video_frame,+ )+ chunk_ids = np.full(sequence_length, -1, dtype=np.int64)+ chunk_ids[clean_indices] = frame_chunks+ chunk_ids[noisy_indices] = frame_chunks+ target_camera_frames = np.repeat(+ np.arange(num_latent_frames, dtype=np.int64), base.rows_per_video_frame+ )+ condition_camera_frames = base.camera_frame_ids[: base.num_condition_video_rows]+ camera_indices = np.concatenate((base.condition_video_indices, clean_indices, noisy_indices))+ camera_frames = np.concatenate(+ (condition_camera_frames, target_camera_frames, target_camera_frames)+ )+ condition_indices = np.concatenate((base.text_indices, base.condition_video_indices))+ target_indices = np.concatenate((base.audio_indices, noisy_indices))++ return H3PackedLayout(+ position_ids=position_ids,+ token_tags=token_tags,+ video_indices=video_indices,+ audio_indices=base.audio_indices,+ text_indices=base.text_indices,+ condition_indices=condition_indices,+ target_indices=target_indices,+ condition_video_indices=base.condition_video_indices,+ audio_condition_indices=_empty_indices(),+ clean_video_indices=clean_indices,+ noisy_video_indices=noisy_indices,+ target_video_indices=noisy_indices,+ target_audio_indices=base.audio_indices,+ camera_video_indices=camera_indices,+ camera_frame_ids=camera_frames,+ row_roles=row_roles,+ target_video_chunk_ids=chunk_ids,+ num_condition_video_rows=base.num_condition_video_rows,+ num_condition_audio_rows=0,+ num_clean_video_rows=num_target_rows,+ num_noisy_video_rows=num_target_rows,+ rows_per_video_frame=base.rows_per_video_frame,+ latent_height=latent_height,+ latent_width=latent_width,+ patch_size=patch_size,+ stage="stage1",+ window_chunks=window_chunks,+ chunk_latent_frames=chunk_latent_frames,+ )+++def build_row_timesteps(+ layout: H3PackedLayout,+ video_timestep: object,+ audio_timestep: object,+ *,+ text_timestep: object | None = None,+ condition_video_timestep: float = 0.999,+ clean_video_timestep: float = 1.0,+ observation_timesteps: object | None = None,+) -> tuple[np.ndarray, np.ndarray]:+ """Return sorted distinct times and each row's index into that array.++ ``observation_timesteps`` gives one known degree per visual observation; they+ default to the condition level.+ """++ video_values = np.asarray(video_timestep, dtype=np.float32).reshape(-1)+ if layout.stage == "stage0p5":+ rows = int(layout.noisy_video_indices.size)+ if video_values.size == 1:+ noisy_values = np.full(rows, video_values[0], dtype=np.float32)+ default_text = float(video_values[0])+ else:+ # A known-degree field: one time per latent frame or per target row.+ if video_values.size == rows // layout.rows_per_video_frame:+ noisy_values = np.repeat(video_values, layout.rows_per_video_frame)+ elif video_values.size == rows:+ noisy_values = video_values+ else:+ raise ValueError("Stage0.5 video_timestep must be scalar, per-frame, or per-row")+ if text_timestep is None:+ raise ValueError("a Stage0.5 known-degree field requires an explicit text_timestep")+ default_text = 0.0+ elif layout.stage == "stage1":+ chunk_frames = int(layout.chunk_latent_frames or 0)+ num_frames = layout.noisy_video_indices.size // layout.rows_per_video_frame+ num_chunks = num_frames // chunk_frames+ if video_values.size == 1:+ frame_values = np.full(num_frames, video_values[0], dtype=np.float32)+ elif video_values.size == num_chunks:+ frame_values = np.repeat(video_values, chunk_frames)+ elif video_values.size == num_frames:+ frame_values = video_values+ else:+ raise ValueError("Stage1 video_timestep must be scalar, per-chunk, or per-frame")+ noisy_values = np.repeat(frame_values, layout.rows_per_video_frame)+ default_text = 1.0+ else:+ raise ValueError(f"unsupported layout stage {layout.stage!r}")++ def scalar(value: object, name: str) -> float:+ array = np.asarray(value, dtype=np.float32)+ if array.size != 1:+ raise ValueError(f"{name} must be scalar, got shape {array.shape}")+ return float(array.reshape(()))++ row_times = np.zeros(layout.sequence_length, dtype=np.float32)+ row_times[layout.text_indices] = scalar(+ default_text if text_timestep is None else text_timestep, "text_timestep"+ )+ row_times[layout.condition_video_indices] = scalar(+ condition_video_timestep, "condition_video_timestep"+ )+ if observation_timesteps is not None:+ degrees = np.asarray(observation_timesteps, dtype=np.float32).reshape(-1)+ if degrees.size != len(layout.observation_rows):+ raise ValueError("observation_timesteps needs one degree per visual observation")+ row_times[layout.observation_video_indices] = np.repeat(+ degrees, np.asarray(layout.observation_rows, dtype=np.int64)+ )+ row_times[layout.audio_indices] = scalar(audio_timestep, "audio_timestep")+ row_times[layout.clean_video_indices] = scalar(clean_video_timestep, "clean_video_timestep")+ row_times[layout.noisy_video_indices] = noisy_values+ return np.unique(row_times, return_inverse=True)+++def patchify_video(latents: object, patch_size: tuple[int, int, int] = (1, 2, 2)) -> object:+ """Pack ``[B,C,T,H,W]`` latents into ``[B,N,C*pt*ph*pw]`` rows."""++ is_torch = type(latents).__module__.startswith("torch")+ if is_torch:+ import torch++ if not isinstance(latents, torch.Tensor):+ raise TypeError("unrecognized torch-like latent value")+ values = latents+ else:+ values = np.asarray(latents)+ if values.ndim != 5:+ raise ValueError(f"latents must be [B,C,T,H,W], got {values.shape}")+ patch_t, patch_h, patch_w = patch_size+ batch, channels, frames, height, width = values.shape+ if frames % patch_t or height % patch_h or width % patch_w:+ raise ValueError(f"latents shape {values.shape} is not divisible by {patch_size}")+ rows = values.reshape(+ batch,+ channels,+ frames // patch_t,+ patch_t,+ height // patch_h,+ patch_h,+ width // patch_w,+ patch_w,+ )+ if is_torch:+ rows = rows.permute(0, 2, 4, 6, 1, 3, 5, 7)+ return rows.reshape(batch, -1, channels * patch_t * patch_h * patch_w).contiguous()+ rows = rows.transpose(0, 2, 4, 6, 1, 3, 5, 7)+ return np.ascontiguousarray(rows.reshape(batch, -1, channels * patch_t * patch_h * patch_w))+++def unpatchify_video(+ rows: object,+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ *,+ patch_size: tuple[int, int, int] = (1, 2, 2),+ channels: int | None = None,+) -> object:+ """Invert :func:`patchify_video` into ``[B,C,T,H,W]`` latents."""++ is_torch = type(rows).__module__.startswith("torch")+ if is_torch:+ import torch++ if not isinstance(rows, torch.Tensor):+ raise TypeError("unrecognized torch-like row value")+ values = rows+ else:+ values = np.asarray(rows)+ if values.ndim != 3:+ raise ValueError(f"rows must be [B,N,D], got {values.shape}")+ patch_t, patch_h, patch_w = patch_size+ if num_latent_frames % patch_t or latent_height % patch_h or latent_width % patch_w:+ raise ValueError("requested latent grid is not divisible by patch_size")+ patch_volume = patch_t * patch_h * patch_w+ if channels is None:+ if values.shape[-1] % patch_volume:+ raise ValueError("row width is not divisible by patch volume")+ channels = values.shape[-1] // patch_volume+ expected_rows = (+ (num_latent_frames // patch_t) * (latent_height // patch_h) * (latent_width // patch_w)+ )+ if values.shape[1:] != (expected_rows, channels * patch_volume):+ raise ValueError(+ f"rows shape {values.shape} does not match expected "+ f"N={expected_rows}, D={channels * patch_volume}"+ )+ batch = values.shape[0]+ latents = values.reshape(+ batch,+ num_latent_frames // patch_t,+ latent_height // patch_h,+ latent_width // patch_w,+ channels,+ patch_t,+ patch_h,+ patch_w,+ )+ if is_torch:+ latents = latents.permute(0, 4, 1, 5, 2, 6, 3, 7)+ return latents.reshape(+ batch, channels, num_latent_frames, latent_height, latent_width+ ).contiguous()+ latents = latents.transpose(0, 4, 1, 5, 2, 6, 3, 7)+ return np.ascontiguousarray(+ latents.reshape(batch, channels, num_latent_frames, latent_height, latent_width)+ )+++__all__ = [+ "AUDIO_ROLE",+ "AUDIO_TAG",+ "CLEAN_VIDEO_ROLE",+ "CONDITION_ROLE",+ "NOISY_VIDEO_ROLE",+ "PADDING_ROLE",+ "TARGET_AUDIO_ROLE",+ "TARGET_VIDEO_ROLE",+ "TEXT_TAG",+ "VIDEO_TAG",+ "H3PackedLayout",+ "H3VisualObservationSpec",+ "build_row_timesteps",+ "build_stage0p5_layout",+ "build_stage1_layout",+ "patchify_video",+ "unpatchify_video",+]diff --git a/src/solarwm/backends/minimax_h3/lora.py b/src/solarwm/backends/minimax_h3/lora.pynew file mode 100644index 0000000..23a6ab2--- /dev/null+++ b/src/solarwm/backends/minimax_h3/lora.py@@ -0,0 +1,492 @@+"""Exact PEFT LoRA-384 target and checkpoint contract for MiniMax-H3."""++from __future__ import annotations++import json+import re+from collections import OrderedDict+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++from solarwm.errors import BackendContractError++H3_LORA_TARGET_COUNT = 312+H3_LORA_TRAINABLE_PARAMETERS = 2_075_394_048+H3_LORA_SUFFIXES = (+ "attn.to_q",+ "attn.to_k",+ "attn.to_v",+ "attn.to_out.0",+ "ff.net.0.proj",+ "ff.net.2",+)+++def _indexed_prefixes(+ modules: Mapping[str, Any], pattern: str, expected: int, label: str+) -> tuple[str, ...]:+ expression = re.compile(pattern)+ found: dict[int, str] = {}+ for name in modules:+ match = expression.fullmatch(name)+ if match:+ index = int(match.group(1))+ if index in found:+ raise BackendContractError(f"duplicate {label} block index {index}")+ found[index] = name+ required = set(range(expected))+ if set(found) != required:+ raise BackendContractError(+ f"{label} topology differs: missing={sorted(required - set(found))} "+ f"extra={sorted(set(found) - required)}"+ )+ return tuple(found[index] for index in range(expected))+++def discover_h3_lora_targets(model: Any) -> tuple[str, ...]:+ """Discover and audit all 50 main + 2 refiner block QKVO/FFN linears."""++ try:+ import torch+ except ImportError as exc: # pragma: no cover - optional local dependency+ raise BackendContractError("PyTorch is required for H3 LoRA discovery") from exc+ modules = dict(model.named_modules())+ main = _indexed_prefixes(+ modules,+ r"(?:.*\.)?transformer_blocks\.(\d+)",+ 50,+ "MiniMax-H3 main transformer",+ )+ refiner = _indexed_prefixes(+ modules,+ r"(?:.*\.)?token_refiner\.refiner_blocks\.(\d+)",+ 2,+ "MiniMax-H3 token refiner",+ )+ blocks = main + refiner+ for block in blocks:+ attention = modules.get(f"{block}.attn")+ if attention is None or getattr(attention, "fused_projections", None) is not False:+ raise BackendContractError(f"H3 LoRA requires split Q/K/V projections at {block!r}")+ targets = tuple(sorted(f"{block}.{suffix}" for block in blocks for suffix in H3_LORA_SUFFIXES))+ if len(targets) != H3_LORA_TARGET_COUNT or len(set(targets)) != len(targets):+ raise AssertionError("internal H3 LoRA target-count error")+ invalid = [+ name+ for name in targets+ if name not in modules or not isinstance(modules[name], torch.nn.Linear)+ ]+ if invalid:+ raise BackendContractError(+ f"H3 LoRA expected nn.Linear targets; first invalid={invalid[:8]}"+ )+ return targets+++@dataclass+class H3LoRARuntime:+ """Replicated adapter state: the LoRA pairs plus an optional geometry ControlNet.++ ``parameter_by_key`` is the complete saved/FSDP-ignored set. The LoRA fields+ of :meth:`metadata` and :attr:`parameter_count` describe the LoRA alone, so a+ run without the ControlNet keeps its released identity unchanged.+ """++ model: Any+ targets: tuple[str, ...]+ parameter_by_key: OrderedDict[str, Any]+ peft_config: Any+ peft_module: Any+ base_identity: Mapping[str, Any]+ rank: int+ alpha: int+ geometry_keys: tuple[str, ...] = ()+ geometry_profile: Any = None++ @property+ def parameters(self) -> tuple[Any, ...]:+ return tuple(self.parameter_by_key.values())++ @property+ def lora_keys(self) -> tuple[str, ...]:+ geometry = set(self.geometry_keys)+ return tuple(key for key in self.parameter_by_key if key not in geometry)++ @property+ def lora_parameters(self) -> tuple[Any, ...]:+ return tuple(self.parameter_by_key[key] for key in self.lora_keys)++ @property+ def geometry_parameters(self) -> tuple[Any, ...]:+ return tuple(self.parameter_by_key[key] for key in self.geometry_keys)++ @property+ def trainable_parameters(self) -> tuple[Any, ...]:+ """What the optimizer, gradient sync and EMA own."""++ return tuple(parameter for parameter in self.parameters if parameter.requires_grad)++ @property+ def parameter_count(self) -> int:+ return sum(parameter.numel() for parameter in self.lora_parameters)++ def state_dict(self) -> OrderedDict[str, Any]:+ return OrderedDict(+ (key, parameter.detach()) for key, parameter in self.parameter_by_key.items()+ )++ def load_state_dict(self, values: Mapping[str, Any], *, broadcast: bool = True) -> None:+ import torch+ import torch.distributed as dist++ if set(values) != set(self.parameter_by_key):+ raise BackendContractError(+ "H3 LoRA checkpoint keys differ: "+ f"missing={sorted(set(self.parameter_by_key) - set(values))[:8]} "+ f"extra={sorted(set(values) - set(self.parameter_by_key))[:8]}"+ )+ with torch.no_grad():+ for key, parameter in self.parameter_by_key.items():+ value = values[key]+ if tuple(value.shape) != tuple(parameter.shape):+ raise BackendContractError(f"H3 LoRA shape differs for {key!r}")+ if value.dtype != parameter.dtype:+ raise BackendContractError(f"H3 LoRA dtype differs for {key!r}")+ parameter.copy_(value.to(device=parameter.device))+ if broadcast and dist.is_available() and dist.is_initialized():+ dist.broadcast(parameter, src=0)++ @staticmethod+ def _descriptors(values: Mapping[str, Any]) -> dict[str, Any]:+ return {+ "state_keys": list(values),+ "state_shapes": {key: list(parameter.shape) for key, parameter in values.items()},+ "state_dtypes": {+ key: str(parameter.dtype).removeprefix("torch.")+ for key, parameter in values.items()+ },+ }++ def metadata(self) -> dict[str, Any]:+ lora = OrderedDict((key, self.parameter_by_key[key]) for key in self.lora_keys)+ values = {+ "schema": "solarwm.minimax-h3-lora.v1",+ "peft_version": str(self.peft_module.__version__),+ "rank": self.rank,+ "alpha": self.alpha,+ "target_count": len(self.targets),+ "target_modules": list(self.targets),+ **self._descriptors(lora),+ "trainable_parameters": self.parameter_count,+ "base_identity": json.loads(json.dumps(dict(self.base_identity), sort_keys=True)),+ }+ if self.geometry_profile is not None:+ geometry = OrderedDict((key, self.parameter_by_key[key]) for key in self.geometry_keys)+ values["geometry_controlnet"] = {+ "architecture": self.geometry_profile.architecture(),+ "lora_trainable": bool(self.geometry_profile.lora_trainable),+ "parameters": sum(parameter.numel() for parameter in geometry.values()),+ **self._descriptors(geometry),+ }+ return values+++def inject_h3_lora(+ model: Any,+ adapter_cfg: Mapping[str, Any],+ *,+ base_identity: Mapping[str, Any],+ ranks: tuple[int, ...] = (384,),+) -> tuple[Any, H3LoRARuntime]:+ """Inject standard BF16 PEFT LoRA and verify its realized topology/size.++ The run's adapter is rank 384; ``ranks`` admits another for an auxiliary role (a DMD critic+ at rank 128, as LongLive-Plug trains it), whose size is checked at that rank.+ """++ try:+ import peft+ import torch+ import torch.distributed.tensor+ except ImportError as exc: # pragma: no cover - optional runtime dependency+ raise BackendContractError("H3 LoRA requires peft==0.20.0") from exc++ rank = int(adapter_cfg.get("rank", 0))+ alpha = int(adapter_cfg.get("alpha", 0))+ if rank not in ranks or alpha != rank:+ raise BackendContractError(f"H3 adapter requires rank=alpha in {list(ranks)}")+ targets = discover_h3_lora_targets(model)+ configuration = peft.LoraConfig(+ r=rank,+ lora_alpha=alpha,+ lora_dropout=float(adapter_cfg.get("dropout", 0.0)),+ target_modules=list(targets),+ bias="none",+ init_lora_weights=True,+ )+ wrapped = peft.get_peft_model(+ model,+ configuration,+ adapter_name="default",+ autocast_adapter_dtype=False,+ )+ realized = tuple(sorted(str(name) for name in wrapped.base_model.targeted_module_names))+ if realized != targets:+ raise BackendContractError(+ "PEFT realized a different H3 topology: "+ f"missing={sorted(set(targets) - set(realized))[:8]} "+ f"extra={sorted(set(realized) - set(targets))[:8]}"+ )+ wrapped.peft_config["default"].target_modules = set(targets)+ for parameter in wrapped.parameters():+ if parameter.requires_grad:+ parameter.data = parameter.data.to(torch.bfloat16)+ keep_vars = wrapped.state_dict(keep_vars=True)+ standard = peft.get_peft_model_state_dict(+ wrapped,+ state_dict=keep_vars,+ adapter_name="default",+ save_embedding_layers=False,+ )+ parameter_by_key: OrderedDict[str, Any] = OrderedDict()+ for key in sorted(standard):+ value = standard[key]+ if not isinstance(value, torch.nn.Parameter):+ raise BackendContractError(f"PEFT state {key!r} is not a Parameter")+ parameter_by_key[key] = value+ trainable = tuple(parameter for parameter in wrapped.parameters() if parameter.requires_grad)+ if {id(value) for value in trainable} != {id(value) for value in parameter_by_key.values()}:+ raise BackendContractError("all and only H3 LoRA parameters must be trainable")+ if len(parameter_by_key) != 2 * H3_LORA_TARGET_COUNT:+ raise BackendContractError("H3 LoRA must expose one A/B tensor pair per target")+ runtime = H3LoRARuntime(+ model=wrapped,+ targets=targets,+ parameter_by_key=parameter_by_key,+ peft_config=wrapped.peft_config["default"],+ peft_module=peft,+ base_identity=dict(base_identity),+ rank=rank,+ alpha=alpha,+ )+ expected = int(+ adapter_cfg.get("expected_trainable_parameters", H3_LORA_TRAINABLE_PARAMETERS * rank // 384)+ )+ if runtime.parameter_count != expected:+ raise BackendContractError(+ f"H3 LoRA trainable parameters={runtime.parameter_count:,}, expected={expected:,}"+ )+ if any(parameter.dtype != torch.bfloat16 for parameter in runtime.parameters):+ raise BackendContractError("all H3 LoRA parameters must remain BF16")+ _attach_geometry_controlnet(wrapped, runtime)+ return wrapped, runtime+++def _attach_geometry_controlnet(wrapped: Any, runtime: H3LoRARuntime) -> None:+ """Add the ControlNet to the adapter set after the LoRA-only audits passed.++ PEFT froze the trunk with the rest of the base model. It trains in full,+ shares LoRA's replicated/ignored-by-FSDP ownership, and is keyed exactly as+ its EMA shadow names canonicalize, so every stage transition treats it like+ one more adapter tensor.+ """++ from .controlnet_contract import H3_GEOMETRY_MODULE, H3_GEOMETRY_STATE_PREFIX++ controlnet = getattr(wrapped.base_model.model, H3_GEOMETRY_MODULE, None)+ if controlnet is None:+ return+ profile = controlnet.profile+ parameters = OrderedDict(+ (f"{H3_GEOMETRY_STATE_PREFIX}{name}", parameter)+ for name, parameter in sorted(controlnet.named_parameters())+ )+ if not parameters:+ raise BackendContractError("H3 geometry ControlNet exposes no parameters")+ if set(parameters) & set(runtime.parameter_by_key):+ raise BackendContractError("H3 geometry ControlNet keys collide with LoRA keys")+ try:+ import torch+ except ImportError as exc: # pragma: no cover - optional runtime dependency+ raise BackendContractError("H3 geometry ControlNet requires PyTorch") from exc+ for key, parameter in parameters.items():+ if parameter.dtype != torch.bfloat16:+ raise BackendContractError(f"H3 geometry ControlNet parameter {key!r} must be BF16")+ parameter.requires_grad_(True)+ if not profile.lora_trainable:+ for parameter in runtime.parameters:+ parameter.requires_grad_(False)+ runtime.parameter_by_key.update(parameters)+ runtime.geometry_keys = tuple(parameters)+ runtime.geometry_profile = profile+++__all__ = [+ "H3_LORA_SUFFIXES",+ "H3_LORA_TARGET_COUNT",+ "H3_LORA_TRAINABLE_PARAMETERS",+ "H3LoRARuntime",+ "discover_h3_lora_targets",+ "inject_h3_lora",+]+++def merge_lora_delta(+ model: Any, state: Mapping[str, Any], *, alpha: float | None, rank: int | None, scale: float = 1.0+) -> int:+ """Add a released LoRA's ``scale * alpha/rank * B @ A`` into the weights it names.++ ``alpha=None``: a release that names no alpha means alpha = rank on every layer, whatever each layer's rank+ (InstantX's H3 Turbo has rank 64 and 16 layers) - every pair adds ``scale * B @ A``.++ For a LoRA trained on the released H3 and carried onto a model derived from it (+ LongLive-Plug's few-step and guidance LoRAs). It is merged before this run's own adapter is+ injected, so the adapter sits on top of it. Returns the number of weights changed.+ """++ import torch++ modules = dict(model.named_modules())+ pairs: dict[str, dict[str, Any]] = {}+ # "<module>.diff" / "<module>.diff_b": whole-tensor deltas added to that module's weight / bias after the pairs+ # (FastVideo's FastH3 LoRA also carries its embedders and output projections this way)+ deltas = {key: value for key, value in state.items() if re.fullmatch(r".+\.diff(_b)?", key)}+ state = {key: value for key, value in state.items() if key not in deltas}+ for key, value in state.items():+ match = re.fullmatch(r"(.+)\.lora_([AB])\.weight", key)+ if match is None:+ raise BackendContractError(f"LoRA key {key!r} is not <module>.lora_A/B.weight")+ pairs.setdefault(match.group(1), {})[match.group(2)] = value+ factor = float(scale) if alpha is None else float(scale) * float(alpha) / float(rank)+ with torch.no_grad():+ for name, pair in sorted(pairs.items()):+ target = modules.get(name)+ if target is None or not hasattr(target, "weight"):+ raise BackendContractError(f"LoRA names {name}, which the model lacks")+ if set(pair) != {"A", "B"}:+ raise BackendContractError(f"LoRA for {name} lacks half of its pair")+ weight = target.weight+ down, up = pair["A"], pair["B"]+ pair_rank = down.shape[0] if alpha is None else rank+ if (+ down.shape[0] != pair_rank+ or up.shape[1] != pair_rank+ or (up.shape[0], down.shape[1]) != tuple(weight.shape)+ ):+ raise BackendContractError(+ f"LoRA for {name} has shape {tuple(up.shape)} @ {tuple(down.shape)}, "+ f"not the weight's {tuple(weight.shape)}"+ )+ delta = up.to(weight.device, torch.float32) @ down.to(weight.device, torch.float32)+ weight.copy_((weight.float() + factor * delta).to(weight.dtype))+ for key, value in sorted(deltas.items()):+ name, attr = (key[: -len(".diff_b")], "bias") if key.endswith(".diff_b") else (key[: -len(".diff")], "weight")+ target = getattr(modules.get(name), attr, None)+ if target is None:+ raise BackendContractError(f"delta {key!r} names {name}.{attr}, which the model lacks")+ if tuple(value.shape) != tuple(target.shape):+ raise BackendContractError(+ f"delta {key!r} has shape {tuple(value.shape)}, not the {attr}'s {tuple(target.shape)}"+ )+ target.copy_((target.float() + float(scale) * value.to(target.device, torch.float32)).to(target.dtype))+ return len(pairs) + len({key.rsplit(".", 1)[0] for key in deltas})+++def _released_pairs(state: Mapping[str, Any]) -> dict[str, dict[str, Any]]:+ pairs: dict[str, dict[str, Any]] = {}+ for key, value in state.items():+ match = re.fullmatch(r"(.+)\.lora_([AB])\.weight", key)+ if match is None:+ raise BackendContractError(f"LoRA key {key!r} is not <module>.lora_A/B.weight")+ pairs.setdefault(match.group(1), {})[match.group(2)] = value+ for name, pair in pairs.items():+ if set(pair) != {"A", "B"}:+ raise BackendContractError(f"LoRA for {name} lacks half of its pair")+ return pairs+++def released_lora_state(path: str) -> dict[str, Any]:+ """A released LoRA's ``<module>.lora_A/B.weight`` tensors, from a .safetensors file or from a+ training export that keeps them under ``student_lora`` (LongLive-Plug's few-step H3 LoRA)."""++ import torch++ if str(path).endswith(".safetensors"):+ from safetensors.torch import load_file++ return load_file(str(path))+ state = torch.load(str(path), map_location="cpu", weights_only=True)+ return state.get("student_lora", state)+++def adapter_state_as_released(state: Mapping[str, Any]) -> dict[str, Any]:+ """A saved run adapter (PEFT keys, maybe with the control trunk) as ``<module>.lora_A/B``.++ So a trained adapter can be merged into a base the way a released LoRA is+ (:func:`merge_lora_delta`), e.g. to make a frozen control LoRA part of a student's base.+ """++ prefix = "base_model.model."+ return {+ key.removeprefix(prefix): value+ for key, value in state.items()+ if re.fullmatch(r".+\.lora_[AB]\.weight", key)+ }+++def seed_lora_from_released(+ live: Mapping[str, Any],+ released: Mapping[str, Any],+ *,+ live_scale: float,+ released_scale: float,+ offset: int = 0,+) -> int:+ """Start a live (wider) LoRA from a released one, its update unchanged (DMD).++ ``offset`` places the released pair at rank slots ``[offset, offset + rank)``, keeping the slots+ before it, so two LoRAs stack into one whose update is their sum (control + four-step LoRA, trained+ jointly).++ The released pair fills the leading rank slots - ``B`` rescaled by+ ``released_scale / live_scale`` so ``live_scale * B @ A`` equals the released update - and the+ spare slots keep their fresh ``A`` with a zero ``B``: the same update, with room to grow.+ ``live`` is keyed like the run's adapter (PEFT ``base_model.model.`` prefix allowed).+ Returns the number of modules seeded.+ """++ import torch++ by_module: dict[str, dict[str, Any]] = {}+ for key, value in live.items():+ match = re.fullmatch(r"(?:base_model\.model\.)?(.+)\.lora_([AB])\.weight", key)+ if match is not None:+ by_module.setdefault(match.group(1), {})[match.group(2)] = value+ factor = float(released_scale) / float(live_scale)+ pairs = _released_pairs(released)+ with torch.no_grad():+ for name, pair in sorted(pairs.items()):+ target = by_module.get(name)+ if target is None or set(target) != {"A", "B"}:+ raise BackendContractError(+ f"released LoRA names {name}, which the live LoRA lacks"+ )+ down, up = pair["A"], pair["B"]+ rank = int(down.shape[0])+ stop = int(offset) + rank+ if stop > int(target["A"].shape[0]):+ raise BackendContractError(+ f"released rank {rank} at offset {offset} exceeds the live rank "+ f"{int(target['A'].shape[0])} at {name}"+ )+ if (+ tuple(down.shape[1:]) != tuple(target["A"].shape[1:])+ or up.shape[0] != target["B"].shape[0]+ ):+ raise BackendContractError(f"released LoRA shape differs at {name}")+ a, b = target["A"], target["B"]+ a[offset:stop].copy_(down.to(a.device, a.dtype))+ b[:, offset:].zero_()+ b[:, offset:stop].copy_((factor * up.float()).to(b.device, b.dtype))+ return len(pairs)diff --git a/src/solarwm/backends/minimax_h3/mask.py b/src/solarwm/backends/minimax_h3/mask.pynew file mode 100644index 0000000..9643bc0--- /dev/null+++ b/src/solarwm/backends/minimax_h3/mask.py@@ -0,0 +1,114 @@+"""MiniMax-H3 attention visibility contracts without framework imports."""++from __future__ import annotations++from dataclasses import dataclass++import numpy as np++from .layout import (+ AUDIO_ROLE,+ CLEAN_VIDEO_ROLE,+ CONDITION_ROLE,+ NOISY_VIDEO_ROLE,+ H3PackedLayout,+)+++@dataclass(frozen=True)+class FullAttentionMask:+ """A non-materialized all-visible square mask."""++ sequence_length: int++ @property+ def shape(self) -> tuple[int, int]:+ return self.sequence_length, self.sequence_length++ def allows(self, query_index: int, key_index: int) -> bool:+ if not 0 <= query_index < self.sequence_length:+ raise IndexError(query_index)+ if not 0 <= key_index < self.sequence_length:+ raise IndexError(key_index)+ return True+++def stage0p5_full_attention(layout: H3PackedLayout) -> FullAttentionMask:+ """Describe Stage0.5 full attention without allocating an ``S**2`` array."""++ if layout.stage != "stage0p5":+ raise ValueError("Stage0.5 full attention requires a Stage0.5 layout")+ return FullAttentionMask(layout.sequence_length)+++def stage1_window_allows(+ query_role: int,+ query_chunk: int,+ key_role: int,+ key_chunk: int,+ *,+ window_chunks: int = 6,+) -> bool:+ """Return the Stage1 six-chunk visibility decision."""++ if window_chunks <= 0:+ raise ValueError(f"window_chunks must be positive, got {window_chunks}")+ if query_role == CONDITION_ROLE:+ return key_role == CONDITION_ROLE+ if query_role == AUDIO_ROLE:+ return key_role in (CONDITION_ROLE, AUDIO_ROLE)+ if query_role == CLEAN_VIDEO_ROLE:+ if key_role in (CONDITION_ROLE, AUDIO_ROLE):+ return True+ return key_role == CLEAN_VIDEO_ROLE and query_chunk - key_chunk >= 0+ if query_role == NOISY_VIDEO_ROLE:+ if key_role in (CONDITION_ROLE, AUDIO_ROLE):+ return True+ if key_role == CLEAN_VIDEO_ROLE:+ delta = query_chunk - key_chunk+ return 1 <= delta < window_chunks+ return key_role == NOISY_VIDEO_ROLE and query_chunk == key_chunk+ return False+++def build_stage1_window_mask(layout: H3PackedLayout) -> np.ndarray:+ """Materialize a dense boolean six-chunk mask."""++ if layout.stage != "stage1" or layout.window_chunks is None:+ raise ValueError("six-chunk mask requires a Stage1 layout")+ if not layout.clean_video_indices.size or not layout.noisy_video_indices.size:+ raise ValueError("Stage1 mask requires explicit clean and noisy video rows")+ roles = layout.row_roles+ chunks = layout.target_video_chunk_ids+ q_role, k_role = roles[:, None], roles[None, :]+ delta = chunks[:, None] - chunks[None, :]+ condition_q = q_role == CONDITION_ROLE+ condition_k = k_role == CONDITION_ROLE+ audio_q = q_role == AUDIO_ROLE+ audio_k = k_role == AUDIO_ROLE+ clean_q = q_role == CLEAN_VIDEO_ROLE+ clean_k = k_role == CLEAN_VIDEO_ROLE+ noisy_q = q_role == NOISY_VIDEO_ROLE+ noisy_k = k_role == NOISY_VIDEO_ROLE+ common_k = condition_k | audio_k+ return (+ (condition_q & condition_k)+ | (audio_q & common_k)+ | (clean_q & (common_k | (clean_k & (delta >= 0))))+ | (+ noisy_q+ & (+ common_k+ | (clean_k & (delta >= 1) & (delta < int(layout.window_chunks)))+ | (noisy_k & (delta == 0))+ )+ )+ )+++__all__ = [+ "FullAttentionMask",+ "build_stage1_window_mask",+ "stage0p5_full_attention",+ "stage1_window_allows",+]diff --git a/src/solarwm/backends/minimax_h3/model.py b/src/solarwm/backends/minimax_h3/model.pynew file mode 100644index 0000000..46a58b8--- /dev/null+++ b/src/solarwm/backends/minimax_h3/model.py@@ -0,0 +1,810 @@+"""Diffusers MiniMax-H3 adapter with W6 masks and token-selective fused PRoPE.++The default FM adapter preserves upstream checkpoint/state-dict names. AnyFlow+target-time conditioning is an explicit extension enabled after strict loading.+The adapter relies on the released diffusers H3 block layout and fails loudly+if that dependency is unavailable.+"""++from __future__ import annotations++from collections.abc import Callable+from dataclasses import dataclass+from typing import Any++try:+ import torch+ from diffusers.models.attention_dispatch import dispatch_attention_fn+ from diffusers.models.transformers.transformer_minimax_h3 import (+ MINIMAX_H3_MODALITY_NUM,+ MiniMaxH3AttnProcessor,+ MiniMaxH3Transformer3DModel,+ MiniMaxH3TransformerBlock,+ MiniMaxH3TransformerOutput,+ _apply_rotary_emb,+ )+ from diffusers.utils import apply_lora_scale+ from torch.nn.attention.flex_attention import (+ flex_attention as _native_flex_attention,+ )+except (ImportError, ModuleNotFoundError) as exc: # pragma: no cover - dependency environment+ raise ImportError(+ "The SolarWM H3 adapter requires a diffusers build containing "+ "MiniMaxH3Transformer3DModel (and its matching torch dependency)."+ ) from exc++from .anyflow_conditioning import H3AnyFlowConditioningMixin+from .camera import fixed_intrinsics_like+from .diffusers_compat import patch_minimax_h3_parameter_dtype+from .distributed import (+ get_sp_size,+ is_sequence_parallel_enabled,+ prepare_flex_attention_input,+ validate_packed_attention_mask,+)+from .distributed import (+ sequence_all_gather as sequence_model_parallel_all_gather,+)+from .distributed import (+ sequence_all_to_all as sequence_model_parallel_all_to_all_4D,+)+from .distributed import (+ shard_stage0p5_packed_sequence as shard_packed_sequence,+)+from .distributed import (+ state as get_sequence_parallel_state,+)++get_parameter_dtype = patch_minimax_h3_parameter_dtype()+++FusedPrope = Callable[+ ..., tuple[torch.Tensor, torch.Tensor, torch.Tensor, Callable[[torch.Tensor], torch.Tensor]]+]+++def _inductor_flex_attention(+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ block_mask: Any,+) -> torch.Tensor:+ """A fail-closed graph boundary around the native FlexAttention HOP."""++ # If Dynamo hits a cache limit, a graph break, suppress_errors, or a+ # disabled compile context, it may try to run this Python function eagerly.+ # Raw torch 2.6 FlexAttention then uses a dense composite implementation;+ # for H3 that would allocate hundreds of GiB. Never permit that fallback.+ if not torch.compiler.is_dynamo_compiling():+ raise RuntimeError(+ "H3 Stage1 FlexAttention escaped its Inductor graph; refusing the dense eager fallback"+ )+ return _native_flex_attention(+ query,+ key,+ value,+ block_mask=block_mask,+ )+++def _training_flex_attention(query, key, value, block_mask):+ return _inductor_flex_attention(query, key, value, block_mask)+++def _validation_flex_attention(query, key, value, block_mask):+ return _inductor_flex_attention(query, key, value, block_mask)+++def _compile_flex_attention(entry):+ return torch.compile(+ entry,+ backend="inductor",+ fullgraph=True,+ dynamic=True,+ # PyTorch 2.6 training needs the no-cudagraphs mode for Flex backward.+ mode="max-autotune-no-cudagraphs",+ )+++# Dynamo budgets recompilations per Python code object. Validation introduces+# shorter rollout shapes and lazily registered decoder dependencies; these+# must not consume the training entry's bounded specialization budget.+_compiled_training_flex_attention = _compile_flex_attention(_training_flex_attention)+_compiled_validation_flex_attention = _compile_flex_attention(_validation_flex_attention)+++def _compiled_h3_flex_attention(query, key, value, block_mask, *, training=True):+ # Detached AnyFlow target forwards are still training. Grad mode cannot+ # distinguish them from validation; use the attention module's mode.+ compiled = (+ _compiled_training_flex_attention if training else _compiled_validation_flex_attention+ )+ return compiled(query, key, value, block_mask)+++@dataclass(frozen=True)+class H3AttentionControl:+ """Non-parameter payload carried through the upstream block forward."""++ attention_mask: Any+ fused_prope: bool | FusedPrope | None+ prope_token_indices: torch.Tensor | None+ prope_frame_ids: torch.Tensor | None+ cam_viewmats: torch.Tensor | None+ cam_K: torch.Tensor | None+ prope_kwargs: dict[str, Any] | None+ sgf_attention: Any = None+ sequence_lengths: tuple[int, ...] = ()+ # "wan_fixed" pins every clip to one field of view, which is what the released camera training+ # assumed; "sample" lets a clip bring its own focal length, and change it as it goes.+ camera_intrinsics: str = "wan_fixed"+++def _is_flex_block_mask(mask: Any) -> bool:+ """Recognise torch FlexAttention BlockMask without importing it eagerly."""++ return (+ mask is not None and type(mask).__name__ == "BlockMask" and hasattr(mask, "kv_num_blocks")+ )+++def _default_prope() -> FusedPrope:+ """Resolve H3's shared head-sliced/logd4 PRoPE runtime."""++ try:+ from .torch_prope import prope_qkv+ except (ImportError, ModuleNotFoundError) as exc:+ raise ImportError(+ "fused PRoPE was requested, but h3.torch_prope.prope_qkv is unavailable"+ ) from exc+ return prope_qkv+++def _normalise_camera_rows(+ tensor: torch.Tensor,+ *,+ name: str,+ batch_size: int,+ num_selected_rows: int,+ frame_ids: torch.Tensor | None,+ matrix_size: int,+) -> torch.Tensor:+ """Accept either token-aligned or frame-aligned camera tensors."""++ if tensor.ndim != 4 or tuple(tensor.shape[-2:]) != (matrix_size, matrix_size):+ raise ValueError(+ f"{name} must be [B,N,{matrix_size},{matrix_size}], got {tuple(tensor.shape)}"+ )+ if tensor.shape[0] not in (1, batch_size):+ raise ValueError(+ f"{name} batch {tensor.shape[0]} cannot serve attention batch {batch_size}"+ )+ if tensor.shape[0] == 1 and batch_size != 1:+ tensor = tensor.expand(batch_size, -1, -1, -1)+ if tensor.shape[1] == num_selected_rows:+ return tensor+ if frame_ids is None:+ raise ValueError(+ f"{name} has {tensor.shape[1]} camera frames "+ f"for {num_selected_rows} selected token rows; "+ "pass prope_frame_ids to expand frame-aligned cameras"+ )+ frame_ids = frame_ids.to(device=tensor.device, dtype=torch.long)+ if frame_ids.ndim != 1 or frame_ids.numel() != num_selected_rows:+ raise ValueError(+ f"prope_frame_ids must have {num_selected_rows} entries, got {tuple(frame_ids.shape)}"+ )+ if frame_ids.numel() and (int(frame_ids.min()) < 0 or int(frame_ids.max()) >= tensor.shape[1]):+ raise ValueError(f"prope_frame_ids addresses outside {name}'s {tensor.shape[1]} frames")+ return tensor.index_select(1, frame_ids)+++class SolarMiniMaxH3AttnProcessor(MiniMaxH3AttnProcessor):+ """Upstream-compatible processor that unwraps SolarWM attention controls."""++ def __call__(+ self,+ attn: Any,+ hidden_states: torch.Tensor,+ rotary_emb: tuple[torch.Tensor, torch.Tensor] | None = None,+ attention_mask: Any = None,+ ) -> torch.Tensor:+ if not isinstance(attention_mask, H3AttentionControl):+ return super().__call__(attn, hidden_states, rotary_emb, attention_mask)+ if rotary_emb is None:+ raise ValueError("SolarWM H3 block attention requires native MM-RoPE")+ control = attention_mask+ if control.sequence_lengths:+ # Activation recomputation may follow a score-model forward with+ # a different packed length. Restore this document's SP layout.+ get_sequence_parallel_state().sequence_lengths = control.sequence_lengths+ return SolarMiniMaxH3Transformer3DModel._attention_forward(+ attn,+ hidden_states,+ rotary_emb,+ control.attention_mask,+ fused_prope=control.fused_prope,+ prope_token_indices=control.prope_token_indices,+ prope_frame_ids=control.prope_frame_ids,+ cam_viewmats=control.cam_viewmats,+ cam_K=control.cam_K,+ prope_kwargs=control.prope_kwargs,+ sgf_attention=control.sgf_attention,+ camera_intrinsics=control.camera_intrinsics,+ )+++class SolarMiniMaxH3Transformer3DModel(H3AnyFlowConditioningMixin, MiniMaxH3Transformer3DModel):+ """Upstream H3 attention plumbing with optional AnyFlow time conditioning."""++ def _install_solar_processors(self) -> None:+ """Install non-module processors without changing model state keys."""++ # Processors are plain callables, not nn.Modules: replacing them adds no+ # state_dict keys. Installing before FSDP wrapping keeps every block's+ # ordinary forward boundary intact for sharding/checkpoint hooks.+ for layer_index, block in enumerate(self.transformer_blocks):+ block.attn.set_processor(SolarMiniMaxH3AttnProcessor())+ block.attn.h3_layer_index = layer_index++ @classmethod+ def from_config(+ cls,+ config: Any = None,+ return_unused_kwargs: bool = False,+ **kwargs: Any,+ ) -> Any:+ """Construct with the inherited upstream init signature, then adapt attention."""++ result = super().from_config(+ config,+ return_unused_kwargs=return_unused_kwargs,+ **kwargs,+ )+ if return_unused_kwargs:+ model, unused_kwargs = result+ model._install_solar_processors()+ return model, unused_kwargs+ result._install_solar_processors()+ return result++ @property+ def uses_geometry_controlnet(self) -> bool:+ return hasattr(self, "geometry_controlnet")++ def enable_geometry_controlnet(self, profile: Any, *, device: Any) -> bool:+ """Attach the geometry ControlNet after the upstream checkpoint strict-loaded.++ Call before PEFT/FSDP wrapping. The trunk is built on ``meta`` and seeded+ per parameter name, so it consumes no global RNG and never alters the+ base state dict. Returns whether the trunk was created.+ """++ import math++ from .controlnet import build_geometry_controlnet++ if self.uses_geometry_controlnet:+ if self.geometry_controlnet.profile != profile:+ raise ValueError("H3 geometry ControlNet is already enabled with another profile")+ return False+ config = self.config+ module = build_geometry_controlnet(+ profile,+ device=device,+ hidden_size=int(config.hidden_size),+ time_embed_dim=int(config.time_embed_dim),+ num_layers=int(config.num_layers),+ video_patch_dim=int(config.in_channels) * math.prod(tuple(config.patch_size)),+ norm_eps=float(config.norm_eps),+ qk_norm_eps=float(config.qk_norm_eps),+ rotary_channels=2 * 3 * int(config.rope_freq_dim),+ )+ module.attention_backend = getattr(+ self.transformer_blocks[0].attn.processor, "_attention_backend", None+ )+ self.geometry_controlnet = module+ return True++ @classmethod+ def strict_from_pretrained(cls, pretrained_model_name_or_path: str, **kwargs: Any):+ """Load a diffusers H3 checkpoint and reject every key/shape mismatch."""++ kwargs.pop("output_loading_info", None)+ model, loading_info = cls.from_pretrained(+ pretrained_model_name_or_path,+ output_loading_info=True,+ **kwargs,+ )+ problems = {+ name: loading_info.get(name)+ for name in ("missing_keys", "unexpected_keys", "mismatched_keys", "error_msgs")+ if loading_info.get(name)+ }+ if problems:+ raise RuntimeError(f"MiniMax-H3 checkpoint did not strict-load: {problems}")+ return model++ @staticmethod+ def _prepare_prope(+ *,+ query: torch.Tensor,+ key: torch.Tensor,+ value: torch.Tensor,+ fused_prope: bool | FusedPrope | None,+ prope_token_indices: torch.Tensor | None,+ prope_frame_ids: torch.Tensor | None,+ cam_viewmats: torch.Tensor | None,+ cam_K: torch.Tensor | None,+ prope_kwargs: dict[str, Any] | None,+ camera_intrinsics: str = "wan_fixed",+ ) -> tuple[+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ Callable[[torch.Tensor], torch.Tensor] | None,+ torch.Tensor | None,+ ]:+ cameras_present = cam_viewmats is not None or cam_K is not None+ if fused_prope is False:+ return query, key, value, None, None+ if not cameras_present and fused_prope is not True and not callable(fused_prope):+ return query, key, value, None, None+ if cam_viewmats is None or cam_K is None:+ raise ValueError("fused PRoPE requires both cam_viewmats and cam_K")+ if prope_token_indices is None:+ raise ValueError(+ "fused PRoPE requires explicit prope_token_indices; "+ "token_tags cannot distinguish Qwen vision rows"+ )++ indices = prope_token_indices.to(device=query.device, dtype=torch.long)+ if indices.ndim != 1:+ raise ValueError(+ f"prope_token_indices must be one-dimensional, got {tuple(indices.shape)}"+ )+ if indices.numel() == 0:+ return query, key, value, None, indices+ if int(indices.min()) < 0 or int(indices.max()) >= query.shape[1]:+ raise ValueError("prope_token_indices contains an out-of-range packed row")+ if torch.unique(indices).numel() != indices.numel():+ raise ValueError("prope_token_indices must not contain duplicates")++ viewmats = _normalise_camera_rows(+ cam_viewmats,+ name="cam_viewmats",+ batch_size=query.shape[0],+ num_selected_rows=indices.numel(),+ frame_ids=prope_frame_ids,+ matrix_size=4,+ ).to(device=query.device)+ intrinsics = _normalise_camera_rows(+ cam_K,+ name="cam_K",+ batch_size=query.shape[0],+ num_selected_rows=indices.numel(),+ frame_ids=prope_frame_ids,+ matrix_size=3,+ ).to(device=query.device)+ # Match the established Wan camera-training convention: the artifact K is loaded and+ # shape-validated above, and only the PRoPE runtime value is replaced. A run that asks for+ # "sample" keeps the clip's own focal length instead, which is what a zoom is made of.+ if camera_intrinsics != "sample":+ intrinsics = fixed_intrinsics_like(intrinsics)+ if callable(fused_prope):+ selected_q = query.index_select(1, indices).transpose(1, 2)+ selected_k = key.index_select(1, indices).transpose(1, 2)+ selected_v = value.index_select(1, indices).transpose(1, 2)+ selected_q, selected_k, selected_v, apply_output = fused_prope(+ selected_q,+ selected_k,+ selected_v,+ viewmats=viewmats,+ Ks=intrinsics,+ **(prope_kwargs or {}),+ )+ query = query.index_copy(1, indices, selected_q.transpose(1, 2))+ key = key.index_copy(1, indices, selected_k.transpose(1, 2))+ value = value.index_copy(1, indices, selected_v.transpose(1, 2))+ return query, key, value, apply_output, indices+ else:+ # Einsum requires camera matrices and q/k/v to share a dtype. Cast+ # only the small camera tensors. Identity matrices on non-camera+ # rows make the operation token-selective without materialising+ # selected q/k/v plus three full-sequence index_copy results.+ dtype = query.dtype+ batch, sequence = query.shape[:2]+ full_viewmats = (+ torch.eye(4, device=query.device, dtype=dtype)+ .view(1, 1, 4, 4)+ .expand(batch, sequence, 4, 4)+ .clone()+ )+ full_intrinsics = (+ torch.eye(3, device=query.device, dtype=dtype)+ .view(1, 1, 3, 3)+ .expand(batch, sequence, 3, 3)+ .clone()+ )+ full_viewmats = full_viewmats.index_copy(1, indices, viewmats.to(dtype))+ full_intrinsics = full_intrinsics.index_copy(1, indices, intrinsics.to(dtype))+ query, key, value, apply_output = _default_prope()(+ query.transpose(1, 2),+ key.transpose(1, 2),+ value.transpose(1, 2),+ viewmats=full_viewmats,+ Ks=full_intrinsics,+ **(prope_kwargs or {}),+ )+ return (+ query.transpose(1, 2),+ key.transpose(1, 2),+ value.transpose(1, 2),+ apply_output,+ None,+ )++ @staticmethod+ def _sgf_attention_forward(attn, query, key, value, control):+ from .sgf_attention import sgf_attention as attend_sgf++ sp_enabled = is_sequence_parallel_enabled()+ if sp_enabled:+ if attn.heads % get_sp_size():+ raise ValueError("H3 SGF heads must be divisible by SP size")+ query = sequence_model_parallel_all_to_all_4D(query, scatter_dim=2, gather_dim=1)+ key = sequence_model_parallel_all_to_all_4D(key, scatter_dim=2, gather_dim=1)+ value = sequence_model_parallel_all_to_all_4D(value, scatter_dim=2, gather_dim=1)++ def segment_attention(q, k, v):+ # Each segment contains exactly its visible keys. Flash+ # attention needs no token-level mask or dense score tensor.+ from flash_attn import flash_attn_func++ return flash_attn_func(+ q.contiguous(), k.contiguous(), v.contiguous(), dropout_p=0.0, causal=False+ )++ attended = attend_sgf(+ query,+ key,+ value,+ control=control,+ layer=attn.h3_layer_index,+ apply_rotary=_apply_rotary_emb,+ attend=segment_attention,+ )+ if sp_enabled:+ attended = sequence_model_parallel_all_to_all_4D(attended, scatter_dim=1, gather_dim=2)+ attended = attended.flatten(2, 3).type_as(query)+ return attn.to_out[1](attn.to_out[0](attended))++ @classmethod+ def _attention_forward(+ cls,+ attn: Any,+ hidden_states: torch.Tensor,+ rotary_emb: tuple[torch.Tensor, torch.Tensor],+ attention_mask: Any,+ *,+ fused_prope: bool | FusedPrope | None,+ prope_token_indices: torch.Tensor | None,+ prope_frame_ids: torch.Tensor | None,+ cam_viewmats: torch.Tensor | None,+ cam_K: torch.Tensor | None,+ prope_kwargs: dict[str, Any] | None,+ sgf_attention: Any = None,+ camera_intrinsics: str = "wan_fixed",+ ) -> torch.Tensor:+ if attn.fused_projections:+ query, key, value = attn.to_qkv(hidden_states).chunk(3, dim=-1)+ else:+ query = attn.to_q(hidden_states)+ key = attn.to_k(hidden_states)+ value = attn.to_v(hidden_states)+ query = query.unflatten(-1, (attn.heads, -1))+ key = key.unflatten(-1, (attn.heads, -1))+ value = value.unflatten(-1, (attn.heads, -1))+ query = attn.norm_q(query)+ key = attn.norm_k(key)++ if sgf_attention is not None:+ return cls._sgf_attention_forward(attn, query, key, value, sgf_attention)++ # H3 native MM-RoPE first, then token-selective camera PRoPE.+ query = _apply_rotary_emb(query, *rotary_emb)+ key = _apply_rotary_emb(key, *rotary_emb)+ query, key, value, apply_output, prope_indices = cls._prepare_prope(+ query=query,+ key=key,+ value=value,+ fused_prope=fused_prope,+ prope_token_indices=prope_token_indices,+ prope_frame_ids=prope_frame_ids,+ cam_viewmats=cam_viewmats,+ cam_K=cam_K,+ prope_kwargs=prope_kwargs,+ camera_intrinsics=camera_intrinsics,+ )++ sp_enabled = is_sequence_parallel_enabled()+ if sp_enabled:+ # Shard the already-packed heterogeneous document in both stages.+ # Ulysses exchanges local sequence for local heads only after both+ # native MM-RoPE and token-selective camera PRoPE have been applied,+ # so the full-sequence attention is numerically the same operation+ # as SP1. Stage1 retains its full, head-broadcast W6 BlockMask:+ # after this exchange each rank owns every row and fewer heads.+ sp_size = get_sp_size()+ if attn.heads % sp_size:+ raise RuntimeError(+ f"MiniMax-H3 attention heads={attn.heads} must be divisible "+ f"by sp_size={sp_size}"+ )+ query = sequence_model_parallel_all_to_all_4D(query, scatter_dim=2, gather_dim=1)+ key = sequence_model_parallel_all_to_all_4D(key, scatter_dim=2, gather_dim=1)+ value = sequence_model_parallel_all_to_all_4D(value, scatter_dim=2, gather_dim=1)++ if type(attention_mask).__name__ == "H3SegmentPlan":+ # Block-causal omni rendering (omni_causal.py): each query segment over exactly its visible keys.+ from .omni_causal import segment_attention++ attended = segment_attention(query, key, value, attention_mask, attn.h3_layer_index)+ elif _is_flex_block_mask(attention_mask):+ expected_shape = (int(query.shape[1]), int(key.shape[1]))+ if tuple(attention_mask.seq_lengths) != expected_shape:+ raise ValueError(+ "H3 FlexAttention BlockMask/QK shape mismatch: "+ f"mask={tuple(attention_mask.seq_lengths)} qk={expected_shape}"+ )+ if (+ query.device.type != "cuda"+ or key.device != query.device+ or value.device != query.device+ ):+ raise RuntimeError("H3 Stage1 FlexAttention requires Q/K/V on one CUDA device")+ # Release the sequence-major references as each head-major input+ # is prepared; retain native Flex and its exact global mask.+ query = prepare_flex_attention_input(query)+ key = prepare_flex_attention_input(key)+ value = prepare_flex_attention_input(value)+ attended = _compiled_h3_flex_attention(+ query,+ key,+ value,+ attention_mask,+ training=attn.training,+ ).transpose(1, 2)+ else:+ attended = dispatch_attention_fn(+ query,+ key,+ value,+ attn_mask=attention_mask,+ dropout_p=0.0,+ is_causal=False,+ backend=getattr(attn.processor, "_attention_backend", None),+ parallel_config=getattr(attn.processor, "_parallel_config", None),+ )++ if sp_enabled:+ attended = sequence_model_parallel_all_to_all_4D(attended, scatter_dim=1, gather_dim=2)++ if apply_output is not None:+ if prope_indices is None:+ attended = apply_output(attended.transpose(1, 2)).transpose(1, 2)+ elif prope_indices.numel():+ selected = attended.index_select(1, prope_indices).transpose(1, 2)+ selected = apply_output(selected).transpose(1, 2)+ attended = attended.index_copy(1, prope_indices, selected)+ attended = attended.flatten(2, 3).type_as(query)+ attended = attn.to_out[0](attended)+ return attn.to_out[1](attended)++ @apply_lora_scale("attention_kwargs")+ def forward(+ self,+ hidden_states: torch.Tensor,+ audio_hidden_states: torch.Tensor,+ encoder_hidden_states: torch.Tensor,+ timestep: torch.Tensor,+ timestep_indices: torch.Tensor,+ token_tags: torch.Tensor,+ position_ids: torch.Tensor,+ video_indices: torch.Tensor,+ audio_indices: torch.Tensor,+ text_indices: torch.Tensor,+ attention_kwargs: dict[str, Any] | None = None,+ return_dict: bool = True,+ *,+ r_timestep: torch.Tensor | None = None,+ attention_mask: Any = None,+ stage1_block_mask: Any = None,+ fused_prope: bool | FusedPrope | None = None,+ prope_token_indices: torch.Tensor | None = None,+ prope_frame_ids: torch.Tensor | None = None,+ cam_viewmats: torch.Tensor | None = None,+ cam_K: torch.Tensor | None = None,+ prope_kwargs: dict[str, Any] | None = None,+ stage0p5_sequence_parallel: bool = False,+ packed_sequence_parallel: bool | None = None,+ sgf_attention: Any = None,+ geometry_control: Any = None,+ ) -> MiniMaxH3TransformerOutput | tuple[torch.Tensor, torch.Tensor]:+ """Run upstream H3 with an optional dense/Flex mask and fused PRoPE.++ ``stage1_block_mask`` is an explicit alias for a FlexAttention+ ``BlockMask``. Camera tensors may be token-aligned or frame-aligned; in+ the latter case pass the layout's ``camera_frame_ids``. ``cam_viewmats``+ follows SolarWM PRoPE's world-to-camera convention; callers holding+ camera-to-world matrices must invert them before this boundary.+ AnyFlow requires ``r_timestep`` with exactly ``timestep.shape``, in+ H3's native data-ward time coordinate (``1 - sigma``).+ ``packed_sequence_parallel`` enables the shared packed/Ulysses path;+ ``stage0p5_sequence_parallel`` remains a compatibility alias.+ ``geometry_control`` (an :class:`~.controlnet.H3GeometryControl`) drives+ the geometry ControlNet; ``None`` runs the unmodified block stack.+ """++ del attention_kwargs # LoRA scaling is handled by the decorator.+ if attention_mask is not None and stage1_block_mask is not None:+ raise ValueError("pass only one of attention_mask and stage1_block_mask")+ if stage1_block_mask is not None:+ attention_mask = stage1_block_mask+ if position_ids.ndim != 2 or position_ids.shape[-1] != 3:+ raise ValueError(f"position_ids must be [sequence,3], got {tuple(position_ids.shape)}")+ sequence_length = position_ids.shape[0]+ if token_tags.shape != (sequence_length,) or timestep_indices.shape != (sequence_length,):+ raise ValueError(+ "token_tags and timestep_indices must match the packed sequence length"+ )+ sp_enabled = is_sequence_parallel_enabled()+ if stage0p5_sequence_parallel and packed_sequence_parallel is False:+ raise ValueError("conflicting H3 sequence-parallel flags")+ requested_sp = (+ bool(stage0p5_sequence_parallel)+ if packed_sequence_parallel is None+ else bool(packed_sequence_parallel)+ )+ if sp_enabled != requested_sp:+ raise RuntimeError(+ "MiniMax-H3 SP runtime/forward contract mismatch: "+ f"runtime_sp={get_sp_size()} "+ f"packed_sequence_parallel={requested_sp}"+ )+ if sp_enabled:+ validate_packed_attention_mask(attention_mask, total_tokens=sequence_length)+ if sgf_attention is not None:+ if attention_mask is not None or fused_prope is not True:+ raise ValueError("H3 SGF requires its own segmented attention and fused PRoPE")+ from .sgf_attention import prepare_sgf_rotaries++ sgf_attention = prepare_sgf_rotaries(sgf_attention, self.rope)++ video_embeds = self.proj_in(hidden_states.to(get_parameter_dtype(self.proj_in)))+ audio_embeds = self.audio_proj_in(+ audio_hidden_states.to(get_parameter_dtype(self.audio_proj_in))+ )+ text_embeds = self.context_embedder(+ encoder_hidden_states.to(get_parameter_dtype(self.context_embedder))+ )+ text_embeds = self.token_refiner(text_embeds)+ packed = text_embeds.new_zeros(+ (text_embeds.shape[0], sequence_length, text_embeds.shape[-1])+ )+ packed = packed.index_copy(1, text_indices, text_embeds)+ packed = packed.index_copy(1, video_indices, video_embeds.to(text_embeds.dtype))+ packed = packed.index_copy(1, audio_indices, audio_embeds.to(text_embeds.dtype))++ temb = self._time_condition(timestep, r_timestep, parameter_dtype=get_parameter_dtype)+ adaln_indices = timestep_indices * MINIMAX_H3_MODALITY_NUM + token_tags+ if sp_enabled:+ shard = shard_packed_sequence(+ hidden_states=packed,+ position_ids=position_ids,+ token_tags=token_tags,+ timestep_indices=timestep_indices,+ prope_token_indices=prope_token_indices,+ prope_frame_ids=prope_frame_ids,+ camera_viewmats=cam_viewmats,+ camera_K=cam_K,+ )+ packed = shard.hidden_states+ position_ids = shard.position_ids+ token_tags = shard.token_tags+ timestep_indices = shard.timestep_indices+ adaln_indices = timestep_indices * MINIMAX_H3_MODALITY_NUM + token_tags+ prope_token_indices = shard.prope_token_indices+ prope_frame_ids = shard.prope_frame_ids+ cam_viewmats = shard.camera_viewmats+ cam_K = shard.camera_K+ rotary_emb = self.rope(position_ids)+ control_stream = None+ if geometry_control is not None:+ if not self.uses_geometry_controlnet:+ raise ValueError("geometry_control requires an enabled geometry ControlNet")+ # The control stream is embedded over every packed row, then sliced to+ # exactly the rows this rank's backbone shard owns.+ control_stream = self.geometry_controlnet.prepare(+ geometry_control,+ sequence_length=sequence_length,+ temb=temb,+ timestep_indices=timestep_indices,+ rope=self.rope,+ start=shard.start if sp_enabled else 0,+ stop=shard.stop if sp_enabled else sequence_length,+ sequence_lengths=shard.sequence_lengths if sp_enabled else (),+ )+ needs_control = (+ _is_flex_block_mask(attention_mask)+ or fused_prope is not None+ or cam_viewmats is not None+ or cam_K is not None+ or prope_token_indices is not None+ or prope_frame_ids is not None+ or prope_kwargs is not None+ )+ attention_control = (+ H3AttentionControl(+ attention_mask=attention_mask,+ fused_prope=fused_prope,+ prope_token_indices=prope_token_indices,+ prope_frame_ids=prope_frame_ids,+ cam_viewmats=cam_viewmats,+ cam_K=cam_K,+ prope_kwargs=prope_kwargs,+ sgf_attention=sgf_attention,+ sequence_lengths=shard.sequence_lengths if sp_enabled else (),+ camera_intrinsics=str(getattr(self, "camera_intrinsics_mode", "wan_fixed")),+ )+ if needs_control+ else attention_mask+ )+ for index, block in enumerate(self.transformer_blocks):+ if torch.is_grad_enabled() and self.gradient_checkpointing:++ def custom_forward(+ states: torch.Tensor, current_block: Any = block+ ) -> torch.Tensor:+ return current_block(states, temb, adaln_indices, rotary_emb, attention_control)++ packed = self._gradient_checkpointing_func(custom_forward, packed)+ else:+ packed = block(packed, temb, adaln_indices, rotary_emb, attention_control)+ if control_stream is not None:+ packed = self.geometry_controlnet.inject(index, packed, control_stream)++ packed = self.norm_out(packed, temb, timestep_indices).to(+ get_parameter_dtype(self.proj_out)+ )+ video_packed_output = self.proj_out(packed)+ audio_packed_output = self.audio_proj_out(packed)+ if sp_enabled:+ # Restore only the small output-head widths, not the 5,376-wide+ # residual stream. Every SP rank then computes the identical full+ # logical loss; the trainer scales it by 1/sp_size before backward.+ video_packed_output = sequence_model_parallel_all_gather(video_packed_output, dim=1)+ audio_packed_output = sequence_model_parallel_all_gather(audio_packed_output, dim=1)+ if (+ int(video_packed_output.shape[1]) != sequence_length+ or int(audio_packed_output.shape[1]) != sequence_length+ ):+ raise RuntimeError(+ "MiniMax-H3 SP output gather did not restore the full packed "+ f"sequence={sequence_length}: video={video_packed_output.shape[1]} "+ f"audio={audio_packed_output.shape[1]}"+ )+ video_output = video_packed_output.index_select(1, video_indices)+ audio_output = audio_packed_output.index_select(1, audio_indices)+ if not return_dict:+ return video_output, audio_output+ return MiniMaxH3TransformerOutput(sample=video_output, audio_sample=audio_output)+++__all__ = [+ "FusedPrope",+ "H3AttentionControl",+ "MiniMaxH3TransformerBlock",+ "SolarMiniMaxH3AttnProcessor",+ "SolarMiniMaxH3Transformer3DModel",+]diff --git a/src/solarwm/backends/minimax_h3/observations.py b/src/solarwm/backends/minimax_h3/observations.pynew file mode 100644index 0000000..c72470b--- /dev/null+++ b/src/solarwm/backends/minimax_h3/observations.py@@ -0,0 +1,361 @@+"""Observation plans: which inputs a sample carries and how well each target row is known.++Every target video row has a *known degree* in H3's data-ward time: ``1`` is a row+whose content is given, ``0`` a row that is pure noise. A row at degree ``t``+holds ``t * x0 + (1 - t) * noise``, so a condition is a target row placed high on+the same axis the denoiser already uses. One plan therefore describes ordinary+generation, first/last-frame and prefix conditioning, inpainting, outpainting,+partially trusted observations and staggered schedules. The flow-matching target+``x0 - noise`` does not depend on the degree, so a plan changes the noising and the+row timesteps, never the loss expression.++The helpers use NumPy with an explicit generator so every pattern can be tested+without a GPU runtime.+"""++from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass, field+from typing import Any++import numpy as np++PATTERNS = (+ "given",+ "shared",+ "known_frames",+ "known_region",+ "soft_frames",+ "soft_region",+ "confidence_map",+ "ramp",+)+_DROPOUTS = ("camera", "anchor", "control", "observation")+++@dataclass(frozen=True)+class H3ObservationProfile:+ """Sampling distribution over observation plans for one training run."""++ patterns: tuple[tuple[str, float], ...] = (("shared", 1.0),)+ hard_level: float = 0.999+ soft_min: float = 0.5+ levels: int = 1000+ dropout: tuple[tuple[str, float], ...] = field(default_factory=tuple)++ def dropout_probability(self, name: str) -> float:+ return dict(self.dropout).get(name, 0.0)++ def fingerprint(self) -> dict[str, Any]:+ """Checkpoint-visible description of the training distribution."""++ return {+ "patterns": {name: weight for name, weight in self.patterns},+ "hard_level": self.hard_level,+ "soft_min": self.soft_min,+ "levels": self.levels,+ "dropout": {name: value for name, value in self.dropout},+ }+++@dataclass(frozen=True)+class H3ObservationPlan:+ """One sample's known-degree field and the optional inputs it keeps."""++ pattern: str+ base_time: float+ row_times: np.ndarray+ loss_rows: np.ndarray+ use_camera: bool = True+ use_anchor: bool = True+ use_control: bool = True+ # One uniform draw per visual observation; an observation is kept above the threshold.+ observation_draws: np.ndarray = field(default_factory=lambda: np.ones(0))+ observation_dropout: float = 0.0++ def keeps_observation(self, index: int) -> bool:+ if index >= self.observation_draws.size:+ raise ValueError("a sample carries more visual observations than a plan covers")+ return bool(self.observation_draws[index] >= self.observation_dropout)++ @property+ def generated_fraction(self) -> float:+ return float(self.loss_rows.mean())+++def observation_profile_from_train_config(+ train: Mapping[str, Any],+) -> H3ObservationProfile | None:+ """Parse the optional ``train.observations`` block; ``None`` keeps the fixed recipe."""++ from solarwm.errors import ConfigurationError++ block = train.get("observations")+ if block is None:+ return None+ if not isinstance(block, Mapping):+ raise ConfigurationError("train.observations must be a mapping")+ unknown = sorted(set(block) - {"patterns", "hard_level", "soft_min", "levels", "dropout"})+ if unknown:+ raise ConfigurationError(f"train.observations has unsupported keys {unknown}")+ patterns = block.get("patterns", {"shared": 1.0})+ if not isinstance(patterns, Mapping) or not patterns:+ raise ConfigurationError("train.observations.patterns must be a non-empty mapping")+ weights: list[tuple[str, float]] = []+ for name in PATTERNS:+ if name not in patterns:+ continue+ weight = patterns[name]+ if isinstance(weight, bool) or not isinstance(weight, int | float) or weight < 0:+ raise ConfigurationError(+ f"train.observations.patterns.{name} must be a non-negative number"+ )+ if weight > 0:+ weights.append((name, float(weight)))+ extra = sorted(set(patterns) - set(PATTERNS))+ if extra:+ raise ConfigurationError(f"train.observations.patterns has unsupported patterns {extra}")+ if not weights:+ raise ConfigurationError("train.observations.patterns needs a positive weight")+ total = sum(weight for _, weight in weights)++ def unit(key: str, default: float) -> float:+ value = block.get(key, default)+ if isinstance(value, bool) or not isinstance(value, int | float):+ raise ConfigurationError(f"train.observations.{key} must be a number")+ return float(value)++ hard_level, soft_min = unit("hard_level", 0.999), unit("soft_min", 0.5)+ levels = block.get("levels", 1000)+ if isinstance(levels, bool) or not isinstance(levels, int) or levels < 2:+ raise ConfigurationError("train.observations.levels must be an integer of at least 2")+ if not 0.0 < soft_min < hard_level <= 1.0:+ raise ConfigurationError("train.observations needs 0 < soft_min < hard_level <= 1")+ if round(hard_level * levels) / levels != hard_level:+ raise ConfigurationError("train.observations.hard_level must lie on the level grid")+ dropout = block.get("dropout", {})+ if not isinstance(dropout, Mapping):+ raise ConfigurationError("train.observations.dropout must be a mapping")+ extra = sorted(set(dropout) - set(_DROPOUTS))+ if extra:+ raise ConfigurationError(f"train.observations.dropout has unsupported inputs {extra}")+ probabilities: list[tuple[str, float]] = []+ for name in _DROPOUTS:+ value = dropout.get(name, 0.0)+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value <= 1:+ raise ConfigurationError(f"train.observations.dropout.{name} must lie in [0, 1]")+ probabilities.append((name, float(value)))+ return H3ObservationProfile(+ patterns=tuple((name, weight / total) for name, weight in weights),+ hard_level=hard_level,+ soft_min=soft_min,+ levels=int(levels),+ dropout=tuple(probabilities),+ )+++@dataclass(frozen=True)+class H3InferenceObservations:+ """What a sampling call is given, beyond text: everything here is optional.++ ``row_degrees`` holds the known degree of every target row (``0`` rows are+ generated) and ``latents`` their ``[24,47,h,w]`` content. Rows with a positive+ degree stay fixed during sampling, so the rest is drawn conditionally on them.+ ``start_degrees`` marks rows that *start* from ``latents`` at that degree and are+ then denoised with the others: refining a rough result, or an observation with+ error ``s`` entered at ``1/(1+s)``.+ """++ latents: Any = None+ row_degrees: Any = None+ start_degrees: Any = None+ use_camera: bool = True+ use_anchor: bool = True+ use_visual_observations: bool = True+ control_gain: float = 1.0++ def __post_init__(self) -> None:+ fields = self.row_degrees is not None or self.start_degrees is not None+ if (self.latents is None) == fields:+ raise ValueError("in-place observations need latents and row or start degrees")+ if not 0.0 <= float(self.control_gain) <= 1.0:+ raise ValueError("control_gain must lie in [0, 1]")+++def _shift(value: np.ndarray | float, shift: float) -> np.ndarray:+ values = np.asarray(value, dtype=np.float64)+ return shift * values / (1.0 + (shift - 1.0) * values)+++def _unshift(value: float, shift: float) -> float:+ return float(value / (shift - (shift - 1.0) * value))+++def _frame_subset(frames: int, rng: np.random.Generator) -> np.ndarray:+ """Frames given as observations: prefix, suffix, first and last, or scattered keyframes."""++ known = np.zeros(frames, dtype=bool)+ kind = int(rng.integers(4))+ limit = max(1, frames // 2)+ if kind == 0:+ known[: int(rng.integers(1, limit + 1))] = True+ elif kind == 1:+ known[frames - int(rng.integers(1, limit + 1)) :] = True+ elif kind == 2:+ known[[0, frames - 1]] = True+ else:+ count = int(rng.integers(1, min(4, frames - 1) + 1))+ known[rng.choice(frames, size=count, replace=False)] = True+ if known.all():+ known[int(rng.integers(frames))] = False+ return known+++def _region(frames: int, height: int, width: int, rng: np.random.Generator) -> np.ndarray:+ """``[frames, height, width]`` mask of the rows to regenerate.++ A box covering 10-60% of the frame, regenerated inside (edit, inpaint, removal)+ or outside (outpaint), over the whole clip or a sub-span of it.+ """++ area = rng.uniform(0.1, 0.6)+ aspect = np.exp(rng.uniform(-0.7, 0.7))+ box_h = int(np.clip(round(np.sqrt(area / aspect) * height), 1, height))+ box_w = int(np.clip(round(np.sqrt(area * aspect) * width), 1, width))+ top = int(rng.integers(0, height - box_h + 1))+ left = int(rng.integers(0, width - box_w + 1))+ box = np.zeros((height, width), dtype=bool)+ box[top : top + box_h, left : left + box_w] = True+ if rng.random() < 0.25 and not box.all():+ box = ~box+ regenerate = np.zeros((frames, height, width), dtype=bool)+ if rng.random() < 0.5:+ regenerate[:] = box+ else:+ span = int(rng.integers(1, frames + 1))+ start = int(rng.integers(0, frames - span + 1))+ regenerate[start : start + span] = box+ return regenerate+++def _confidence(height: int, width: int, low: float, high: float, rng: np.random.Generator):+ """A smooth per-row confidence field with holes, as a reprojected memory would give."""++ coarse_h, coarse_w = max(2, height // 6), max(2, width // 6)+ coarse = rng.uniform(low, high, size=(coarse_h, coarse_w))+ rows = np.linspace(0, coarse_h - 1, height)+ cols = np.linspace(0, coarse_w - 1, width)+ r0, c0 = np.floor(rows).astype(int), np.floor(cols).astype(int)+ r1, c1 = np.minimum(r0 + 1, coarse_h - 1), np.minimum(c0 + 1, coarse_w - 1)+ fr, fc = (rows - r0)[:, None], (cols - c0)[None, :]+ smooth = (+ coarse[r0][:, c0] * (1 - fr) * (1 - fc)+ + coarse[r0][:, c1] * (1 - fr) * fc+ + coarse[r1][:, c0] * fr * (1 - fc)+ + coarse[r1][:, c1] * fr * fc+ )+ holes = rng.random((coarse_h, coarse_w)) < 0.2+ hole_rows = np.minimum((np.arange(height) * coarse_h) // height, coarse_h - 1)+ hole_cols = np.minimum((np.arange(width) * coarse_w) // width, coarse_w - 1)+ return smooth, holes[hole_rows][:, hole_cols]+++def sample_observation_plan(+ profile: H3ObservationProfile,+ *,+ frames: int,+ grid_height: int,+ grid_width: int,+ base_time: float,+ shift: float,+ rng: np.random.Generator,+ given: np.ndarray | None = None,+) -> H3ObservationPlan:+ """Draw one plan. ``base_time`` is the run's ordinary shifted video timestep.++ ``given`` is the data's own known-degree field for this sample, if it has one;+ the ``given`` pattern uses it and falls back to ``shared`` without one.+ """++ if not isinstance(rng, np.random.Generator):+ raise TypeError("rng must be an explicit numpy.random.Generator")+ if frames < 2 or grid_height < 1 or grid_width < 1:+ raise ValueError("observation plans need at least two frames and a non-empty grid")+ if not 0.0 <= base_time <= 1.0:+ raise ValueError(f"base_time must lie in [0, 1], got {base_time}")+ names = [name for name, _ in profile.patterns]+ weights = np.asarray([weight for _, weight in profile.patterns], dtype=np.float64)+ pattern = names[int(rng.choice(len(names), p=weights / weights.sum()))]+ hard = profile.hard_level+ base = min(float(base_time), hard)+ times = np.full((frames, grid_height, grid_width), base, dtype=np.float64)++ if pattern == "given":+ if given is None:+ pattern = "shared"+ else:+ field_ = np.asarray(given, dtype=np.float64).reshape(times.shape)+ if field_.min() < 0.0 or field_.max() > 1.0:+ raise ValueError("a given known-degree field must lie in [0, 1]")+ times = np.where(field_ > 0.0, np.maximum(np.minimum(field_, hard), base), base)+ if pattern in ("known_frames", "soft_frames"):+ level = hard if pattern == "known_frames" else rng.uniform(profile.soft_min, hard)+ times[_frame_subset(frames, rng)] = max(level, base)+ elif pattern in ("known_region", "soft_region"):+ level = hard if pattern == "known_region" else rng.uniform(profile.soft_min, hard)+ times[~_region(frames, grid_height, grid_width, rng)] = max(level, base)+ elif pattern == "confidence_map":+ smooth, holes = _confidence(grid_height, grid_width, profile.soft_min, hard, rng)+ observed = _frame_subset(frames, rng)+ times[observed] = np.where(holes, base, np.maximum(smooth, base))[None]+ elif pattern == "ramp":+ # Frames leave the noise one after another, as in a rolling schedule.+ sigma_u = _unshift(1.0 - base, shift)+ order = np.linspace(-0.5, 0.5, frames) * (1.0 if rng.random() < 0.5 else -1.0)+ staggered = np.clip(sigma_u + rng.uniform(0.2, 1.0) * order, 0.0, 1.0)+ times[:] = np.minimum(1.0 - _shift(staggered, shift), hard)[:, None, None]++ times = np.round(times * profile.levels) / profile.levels+ loss_rows = times < hard+ if not loss_rows.any():+ # A plan always leaves something to generate.+ times[-1] = np.round(base * profile.levels) / profile.levels+ times[-1] = np.minimum(times[-1], hard - 1.0 / profile.levels)+ loss_rows = times < hard+ return H3ObservationPlan(+ pattern=pattern,+ base_time=float(np.round(base * profile.levels) / profile.levels),+ row_times=times.reshape(-1).astype(np.float32),+ loss_rows=loss_rows.reshape(-1),+ use_camera=bool(rng.random() >= profile.dropout_probability("camera")),+ use_anchor=bool(rng.random() >= profile.dropout_probability("anchor")),+ use_control=bool(rng.random() >= profile.dropout_probability("control")),+ observation_draws=rng.random(64),+ observation_dropout=profile.dropout_probability("observation"),+ )+++def confidence_to_known_degree(error_std: Any) -> Any:+ """Known degree that matches an observation ``y = x0 + error_std * e``.++ ``t * y`` then has exactly the distribution of a row at degree ``t``, so the+ sampler draws from the posterior under that error model. An infinite error is+ a hole (degree zero).+ """++ values = np.asarray(error_std, dtype=np.float64)+ if np.any(values < 0):+ raise ValueError("observation error must be non-negative")+ return 1.0 / (1.0 + values)+++__all__ = [+ "PATTERNS",+ "H3InferenceObservations",+ "H3ObservationPlan",+ "H3ObservationProfile",+ "confidence_to_known_degree",+ "observation_profile_from_train_config",+ "sample_observation_plan",+]diff --git a/src/solarwm/backends/minimax_h3/official_codec.py b/src/solarwm/backends/minimax_h3/official_codec.pynew file mode 100644index 0000000..1b3fd38--- /dev/null+++ b/src/solarwm/backends/minimax_h3/official_codec.py@@ -0,0 +1,343 @@+"""Official Qwen/VisualVAE/AudioVAE codec shared by online and preencode paths."""++from __future__ import annotations++from collections.abc import Mapping+from typing import Any++import numpy as np++from solarwm.errors import DataContractError+from solarwm.preencode import EncoderContract, validate_encoded_tensors++from .artifacts import h3_encoder_contract, h3_silence_profile+from .codec import H3Codec, H3RawSample, validate_raw_sample++H3_TEXT_ENCODER_LAYER = 50+H3_VIDEO_VAE_SEED = 42+H3_GEOMETRY_PIXEL_SHAPE = (158, 768, 1344, 3)+H3_AUDIO_HOP_LENGTH = 800+H3_AUDIO_LATENTS = 263+H3_SILENCE_METADATA = {"artifact": "official-audiovae-encoded-stereo-zero-waveform"}+++def _pixels(value: Any, *, device: Any) -> Any:+ import torch++ tensor = value if isinstance(value, torch.Tensor) else torch.as_tensor(value)+ if tensor.ndim == 4 and tensor.shape[-1] == 3:+ tensor = tensor.permute(3, 0, 1, 2).unsqueeze(0)+ elif tensor.ndim == 5 and tensor.shape[1] == 3:+ pass+ else:+ raise DataContractError("H3 pixels must be [T,H,W,3] or [B,3,T,H,W]")+ if tuple(tensor.shape) != (1, 3, 158, 768, 1344):+ raise DataContractError(f"H3 pixels must be [1,3,158,768,1344], got {tuple(tensor.shape)}")+ return tensor.to(device=device, non_blocking=True).contiguous()+++def encode_geometry_clips(video_vae: Any, read_frames: Any, num_frames: int, *, device: Any) -> Any:+ """Encode a render of any length to normalized ``[24,T,H/16,W/16]`` BF16 latents.++ ``read_frames(start, stop)`` returns ``[n,H,W,3]`` pixels in ``[0, 1]``. The+ H3 video VAE encodes every ``clip_length`` window independently (padding the+ tail by repeating the last frame) and drops ``token_drop`` trailing latents,+ so streaming its clips equals encoding the whole render while holding one+ clip in memory. Renders share the RGB path's ImageNet normalization and+ latent mean/std, but take the posterior mode: a conditioning signal is one+ deterministic encoding, not a sample of the encoder noise.+ """++ import torch+ from diffusers.models.autoencoders.vae import DiagonalGaussianDistribution++ if num_frames < 2:+ raise DataContractError("H3 geometry renders need at least two frames")+ clip = int(video_vae.config.clip_length)+ drop = int(video_vae.config.token_drop)+ dtype = next(video_vae.encoder.parameters()).dtype+ pixel_mean = torch.tensor((0.485, 0.456, 0.406), device=device).view(1, -1, 1, 1, 1)+ pixel_std = torch.tensor((0.229, 0.224, 0.225), device=device).view(1, -1, 1, 1, 1)+ moments = []+ with torch.inference_mode():+ for start in range(0, num_frames, clip):+ frames = torch.as_tensor(+ np.asarray(read_frames(start, min(start + clip, num_frames))),+ dtype=torch.float32,+ )+ if frames.ndim != 4 or frames.shape[-1] != 3:+ raise DataContractError("H3 geometry render frames must be [n,H,W,3]")+ if frames.shape[0] < clip:+ tail = frames[-1:].expand(clip - frames.shape[0], -1, -1, -1)+ frames = torch.cat((frames, tail), dim=0)+ pixels = frames.permute(3, 0, 1, 2).unsqueeze(0).to(device, non_blocking=True)+ normalized = ((pixels - pixel_mean) / pixel_std).to(dtype)+ moments.append(video_vae._encode_clip(normalized).float().cpu())+ stacked = torch.cat(moments, dim=2)+ if drop:+ stacked = stacked[:, :, :-drop]+ latent = DiagonalGaussianDistribution(stacked).mode()+ mean = torch.tensor(video_vae.config.latents_mean).view(1, -1, 1, 1, 1)+ std = torch.tensor(video_vae.config.latents_std).view(1, -1, 1, 1, 1)+ return ((latent - mean) / std).to(torch.bfloat16)[0].contiguous()+++def encode_joint_prompt_condition(+ keyframe: Any, caption: str, *, processor: Any, tokenizer: Any, text_encoder: Any, device: Any+) -> tuple[Any, Any]:+ import torch+ from PIL import Image++ if not isinstance(keyframe, Image.Image):+ array = np.asarray(keyframe, dtype=np.uint8)+ keyframe = Image.fromarray(array, mode="RGB")+ vision = processor.image_processor(images=[keyframe], return_tensors="pt")+ grid = vision["image_grid_thw"]+ merge = processor.image_processor.merge_size**2+ image_tokens = int(grid[0].prod()) // merge+ label_ids = tokenizer("<Picture 1>: ", add_special_tokens=False)["input_ids"]+ vision_ids = (+ [tokenizer.convert_tokens_to_ids("<|vision_start|>")]+ + [tokenizer.convert_tokens_to_ids("<|image_pad|>")] * image_tokens+ + [tokenizer.convert_tokens_to_ids("<|vision_end|>")]+ )+ caption_ids = tokenizer(str(caption), add_special_tokens=False)["input_ids"]+ token_ids = label_ids + vision_ids + caption_ids+ tags = [1] * len(label_ids) + [0] * len(vision_ids) + [1] * len(caption_ids)+ if not token_ids:+ raise DataContractError("H3 Qwen presentation produced no tokens")+ input_ids = torch.tensor([token_ids], dtype=torch.long, device=device)+ token_types = torch.tensor(+ processor.create_mm_token_type_ids([token_ids]),+ dtype=torch.long,+ device=device,+ )+ outputs = text_encoder.model(+ input_ids=input_ids,+ attention_mask=torch.ones_like(input_ids),+ mm_token_type_ids=token_types,+ use_cache=False,+ output_hidden_states=True,+ pixel_values=vision["pixel_values"].to(device, text_encoder.dtype),+ image_grid_thw=grid.to(device),+ )+ if len(outputs.hidden_states) <= H3_TEXT_ENCODER_LAYER:+ raise DataContractError("H3 Qwen has no hidden state 50")+ hidden = outputs.hidden_states[H3_TEXT_ENCODER_LAYER][0].to(torch.bfloat16).contiguous().cpu()+ tags_tensor = torch.tensor(tags, dtype=torch.long)+ if tuple(hidden.shape) != (len(token_ids), 5120):+ raise DataContractError("H3 Qwen hidden state has the wrong shape")+ return hidden, tags_tensor+++class OfficialH3Codec(H3Codec):+ """Concrete official H3 encoder implementation."""++ def __init__(+ self,+ *,+ text_encoder: Any,+ tokenizer: Any,+ processor: Any,+ video_vae: Any,+ audio_vae: Any,+ device: Any,+ encoder_identity: str,+ geometry: Any = None,+ ) -> None:+ self.geometry = geometry+ self.text_encoder = text_encoder+ self.tokenizer = tokenizer+ self.processor = processor+ self.video_vae = video_vae+ self.audio_vae = audio_vae+ self.video_vae.set_attention_backend("native")+ self.audio_vae.set_attention_backend("native")+ self.device = device+ self.identity = str(encoder_identity).strip()+ if not self.identity:+ raise DataContractError("official H3 codec requires encoder identity")+ self._contract = h3_encoder_contract(encoder_identity=self.identity, geometry=geometry)+ self._silence: Any = None+ self._encoders_active = True++ @property+ def contract(self) -> EncoderContract:+ return self._contract++ def bind_silence_profile(self) -> None:+ """Freeze readable silence semantics into every row contract."""++ self._contract = h3_encoder_contract(+ encoder_identity=self.identity,+ silence_artifact_profile=h3_silence_profile(),+ geometry=self.geometry,+ )++ def silence_artifact_bytes(self) -> bytes:+ """Serialize the online/offline global silence artifact identically."""++ from safetensors.torch import save++ return save(+ {"silence_158f": self._silence_latents()},+ metadata=H3_SILENCE_METADATA,+ )++ def activate_encoders(self) -> None:+ """Move the per-sample video and prompt encoders onto their CUDA device."""++ if self._encoders_active:+ return+ self.text_encoder.to(self.device)+ self.video_vae.to(self.device)+ self._encoders_active = True++ def offload_encoders(self) -> None:+ """Release conditioner memory before the 33B training forward."""++ import torch++ self.text_encoder.to("cpu")+ self.video_vae.to("cpu")+ self.audio_vae.to("cpu")+ self._encoders_active = False+ if getattr(self.device, "type", str(self.device)) == "cuda":+ torch.cuda.empty_cache()++ def _video_latents(self, pixels: Any, *, seed: int) -> Any:+ import torch++ mean = torch.tensor(self.video_vae.config.latents_mean).view(1, -1, 1, 1, 1)+ std = torch.tensor(self.video_vae.config.latents_std).view(1, -1, 1, 1, 1)+ pixel_mean = torch.tensor((0.485, 0.456, 0.406), device=pixels.device).view(1, -1, 1, 1, 1)+ pixel_std = torch.tensor((0.229, 0.224, 0.225), device=pixels.device).view(1, -1, 1, 1, 1)+ normalized = (pixels.float().div(255.0) - pixel_mean) / pixel_std+ posterior = self.video_vae.encode(normalized, return_dict=False)[0]+ latent = posterior.sample(generator=torch.Generator().manual_seed(int(seed)))+ # This fp16 round before mean/std normalization is part of the required+ # cache identity, despite the final BF16 storage.+ latent = latent.to(torch.float16).float().cpu()+ return ((latent - mean) / std).to(torch.bfloat16).contiguous()++ def _geometry_latents(self, unit: Any) -> Any:+ """Encode one ``[158,768,1344,3]`` render in ``[0,1]`` to normalized latents."""++ shape = tuple(getattr(unit, "shape", ()))+ if shape != H3_GEOMETRY_PIXEL_SHAPE:+ raise DataContractError(+ f"H3 geometry render must be {list(H3_GEOMETRY_PIXEL_SHAPE)}, got {shape}"+ )+ return encode_geometry_clips(+ self.video_vae,+ lambda start, stop: unit[start:stop],+ int(shape[0]),+ device=self.device,+ )++ def _joint_prompt(self, keyframe: Any, caption: str) -> tuple[Any, Any]:+ return encode_joint_prompt_condition(+ keyframe,+ caption,+ processor=self.processor,+ tokenizer=self.tokenizer,+ text_encoder=self.text_encoder,+ device=self.device,+ )++ def _silence_latents(self) -> Any:+ import torch++ if self._silence is not None:+ return self._silence+ waveform = torch.zeros(+ (2, 1, H3_AUDIO_LATENTS * H3_AUDIO_HOP_LENGTH),+ dtype=torch.float32,+ device=self.device,+ )+ posterior = self.audio_vae.encode(waveform, return_dict=False)[0]+ latent = posterior.mode().float().cpu()+ mean = torch.tensor(self.audio_vae.config.latents_mean).view(1, -1, 1)+ std = torch.tensor(self.audio_vae.config.latents_std).view(1, -1, 1)+ self._silence = ((latent - mean) / std).to(torch.bfloat16).contiguous()+ if tuple(self._silence.shape) != (2, 32, H3_AUDIO_LATENTS):+ raise DataContractError("official AudioVAE silence is not [2,32,263]")+ return self._silence++ # H3Codec protocol -------------------------------------------------+ def encode_target_video(self, sample: H3RawSample) -> Any:+ validate_raw_sample(sample)+ return self._video_latents(+ _pixels(sample.frames, device=self.device), seed=H3_VIDEO_VAE_SEED+ )[0]++ def encode_visual_anchor(self, sample: H3RawSample) -> Any:+ validate_raw_sample(sample)+ pixels = _pixels(sample.frames, device=self.device)[:, :, :1]+ return self._video_latents(pixels, seed=H3_VIDEO_VAE_SEED)[0]++ def encode_joint_prompt(self, sample: H3RawSample) -> tuple[Any, Any]:+ validate_raw_sample(sample)+ return self._joint_prompt(np.asarray(sample.frames)[0], sample.caption)++ def encode_silence_audio(self, sample: H3RawSample) -> Any:+ validate_raw_sample(sample)+ return self._silence_latents()++ # offline preencode protocol --------------------------------------+ def encode(+ self,+ *,+ sample_id: str,+ pixels: Any,+ caption: str,+ camera: Any,+ seed: int,+ geometry: Mapping[str, Any] | None = None,+ ) -> Mapping[str, Any]:+ """Encode one sample; ``geometry`` maps artifact tensor keys to decoded renders."""++ import torch++ if not isinstance(camera, Mapping):+ raise DataContractError("H3 preencode camera must be a mapping")+ expected_geometry = set(self.geometry.tensor_keys()) if self.geometry is not None else set()+ if set(geometry or {}) != expected_geometry:+ raise DataContractError(+ f"H3 preencode geometry renders differ: got {sorted(geometry or {})}, "+ f"expected {sorted(expected_geometry)}"+ )+ c2w = torch.as_tensor(camera.get("camera_c2w"), dtype=torch.float32)+ K = torch.as_tensor(camera.get("camera_K"), dtype=torch.float32)+ source_indices = torch.as_tensor(camera.get("source_frame_indices"), dtype=torch.long)+ prepared = _pixels(pixels, device=self.device)+ with torch.inference_mode():+ target = self._video_latents(prepared, seed=int(seed))[0]+ anchor = self._video_latents(prepared[:, :, :1], seed=int(seed))[0]+ prompt, tags = self._joint_prompt(np.asarray(pixels)[0], str(caption))+ renders = {+ key: self._geometry_latents(value) for key, value in (geometry or {}).items()+ }+ values = {+ "target_latents": target,+ "anchor_latents": anchor,+ "prompt_embeds": prompt,+ "text_token_tags": tags,+ "source_frame_indices": source_indices.contiguous(),+ "camera_c2w": c2w.contiguous(),+ "camera_K": K.contiguous(),+ **renders,+ }+ validate_encoded_tensors(values, self.contract)+ return values+++__all__ = [+ "H3_AUDIO_HOP_LENGTH",+ "H3_AUDIO_LATENTS",+ "H3_SILENCE_METADATA",+ "H3_TEXT_ENCODER_LAYER",+ "H3_VIDEO_VAE_SEED",+ "OfficialH3Codec",+ "encode_geometry_clips",+]diff --git a/src/solarwm/backends/minimax_h3/omni_anyflow.py b/src/solarwm/backends/minimax_h3/omni_anyflow.pynew file mode 100644index 0000000..fdf1cb0--- /dev/null+++ b/src/solarwm/backends/minimax_h3/omni_anyflow.py@@ -0,0 +1,112 @@+"""AnyFlow flow-map distillation on omni samples, for either way of reading the condition.++The sample is prepared exactly as the ordinary omni recipe prepares it - the aligned observation+through the control trunk (a ControlNet run) or packed beside the target (a concatenation run),+free pictures through the prompt - and only the target rows move: every evaluation names where+they go (r), and everything else stays at its own time. The four AnyFlow evaluations share one+set of conditions; ``h3_anyflow_v15_loss`` holds the objective.+"""++from __future__ import annotations++import dataclasses+from typing import Any++from solarwm.training.anyflow import apply_timestep_shift, sample_anyflow_time_pairs++from .anyflow_loss import h3_anyflow_v15_loss+from .distributed import get_dp_group, get_dp_rank, get_dp_world_size+from .layout import patchify_video+from .stage0p5 import H3Stage0p5Core+++class H3OmniAnyFlowCore(H3Stage0p5Core):+ """Stage0.5 omni inputs with a flow-map objective and few-step sampling."""++ def __init__(+ self,+ model: Any,+ silence_latents: Any,+ device: Any,+ train: Any,+ *,+ geometry: Any = None,+ observations: Any = None,+ ) -> None:+ super().__init__(+ model,+ silence_latents,+ device,+ geometry=geometry,+ observations=observations,+ flow_map=True,+ )+ self.train = train++ def forward_loss(self, batch: Any, *, noise_seed: int | None) -> Any:+ import torch++ generator = (+ None+ if noise_seed is None+ else torch.Generator(device=self.device).manual_seed(int(noise_seed))+ )+ drawn: dict[str, Any] = {}++ def draw_time(draw: Any) -> Any:+ pairs = sample_anyflow_time_pairs(+ 1,+ logical_dp_rank=get_dp_rank(),+ logical_dp_world_size=get_dp_world_size(),+ diffusion_ratio=float(self.train["diffusion_ratio"]),+ consistency_ratio=float(self.train["consistency_ratio"]),+ generator=draw,+ device=self.device,+ )+ self._broadcast_logical_sample(*pairs)+ drawn["sigma_t"] = apply_timestep_shift(pairs.t, 12.0)+ drawn["sigma_r"] = apply_timestep_shift(pairs.r, 12.0)+ drawn["is_diffusion"] = pairs.is_diffusion+ return (1.0 - drawn["sigma_t"]).reshape(()).float()++ # A flow map carries every target row from one time to another; per-row known degrees+ # from the data (edit masks) would hold rows still, which the objective has no place for.+ inputs = self.prepare(dataclasses.replace(batch, known_degrees=None), generator, draw_time)+ # The plan's row times are not read here: the flow map carries every target row from+ # sigma_t itself. Only its pattern matters (a time at the clean end frees the plan's last+ # frame alone, still under the shared pattern, and must train like any other).+ plan = inputs.plan+ if plan is not None and plan.pattern != "shared":+ raise ValueError("flow-map distillation needs the shared observation pattern alone")+ condition_rows = inputs.condition_rows++ def velocity(sample: Any, t: Any, r: Any) -> Any:+ return self._forward(+ video_rows=torch.cat((*condition_rows, sample), dim=1),+ audio_rows=inputs.audio_rows,+ prompt=inputs.prompt,+ layout=inputs.layout,+ video_t=t.reshape(()),+ audio_t=inputs.audio_t,+ viewmats=inputs.viewmats,+ K=inputs.K,+ frame_ids=inputs.frame_ids,+ geometry_control=inputs.geometry,+ observation_degrees=inputs.observation_degrees,+ video_r=r.reshape(()),+ )++ return h3_anyflow_v15_loss(+ velocity,+ patchify_video(inputs.clean),+ patchify_video(inputs.video_noise),+ drawn["sigma_t"],+ drawn["sigma_r"],+ drawn["is_diffusion"],+ epsilon=float(self.train["finite_difference_epsilon"]),+ shift=12.0,+ process_group=get_dp_group(),+ )+++__all__ = ["H3OmniAnyFlowCore"]diff --git a/src/solarwm/backends/minimax_h3/omni_causal.py b/src/solarwm/backends/minimax_h3/omni_causal.pynew file mode 100644index 0000000..6eee7ed--- /dev/null+++ b/src/solarwm/backends/minimax_h3/omni_causal.py@@ -0,0 +1,682 @@+"""Block-causal omni rendering: the structure-conditioned renderer generated chunk by chunk.++A clip of ``5n+2`` latent frames is cut into chunks of at most ``chunk`` latents (5: 0-4, 5-9, ..., 45-46). The+packed document is the ordinary omni one - ``[text | first frame | structure | audio | target]``, the structure laid+over the target frames on their rotary times - and a training document appends a clean copy of the target (teacher+forcing). Who sees whom:++* text and the first frame (the static rows) see the static rows; the audio (silence) sees them and itself;+* structure chunk ``c`` sees the static rows and itself - its rows never depend on another chunk, so they are the+ same in training and in a stream that holds only the last ``window`` chunks;+* noisy chunk ``c`` sees the static rows, the audio, the structure of chunks ``c-window+1 .. c``, the clean frames of+ chunks ``c-window+1 .. c-1`` and itself;+* clean chunk ``c`` sees what noisy chunk ``c`` sees, its own clean frames in place of its noisy ones - exactly what a+ stream computes when it writes chunk ``c`` to its key/value cache.++Nothing sees a later chunk, of video or of structure. Diffusion forcing drops the clean copy: noisy chunk ``c`` sees+the noisy chunks ``c-window+1 .. c``, each at its own noise level.++Attention runs per query segment over exactly its visible keys (flash attention, no token-level mask): the plan is a+list of ``(query rows, key spans, cached chunks, chunk to commit)``. A streaming sampler keeps every layer's keys and+values of committed clean chunks in :class:`H3ChunkKVCache` and runs one chunk at a time.+"""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any++import numpy as np++from .layout import AUDIO_TAG, VIDEO_TAG, H3PackedLayout++STATIC_TIME = 1.0 # text rows: fixed, so a stream can keep them+CLEAN_LEVEL = 0.999 # given frames and observations, as the first frame always was+++# --- attention plan -------------------------------------------------------------------------------------------------+++@dataclass+class H3ChunkKVCache:+ """Per layer, the keys and values (head-local, after rotary) of committed rows, under ``(kind, chunk)``: a chunk's+ structure (``"structure"``) and its clean frames (``"clean"``). Each kind keeps its last ``window`` chunks."""++ layers: dict[int, dict[tuple[str, int], tuple[Any, Any]]] = field(default_factory=dict)+ window: int = 6++ def get(self, layer: int, name: tuple[str, int]) -> tuple[Any, Any]:+ try:+ return self.layers[layer][name]+ except KeyError as exc:+ raise RuntimeError(f"{name} was never committed at layer {layer}") from exc++ def put(self, layer: int, name: tuple[str, int], key: Any, value: Any) -> None:+ import torch++ if torch.is_grad_enabled():+ raise RuntimeError("a key/value commit runs without a graph")+ entries = self.layers.setdefault(layer, {})+ entries[name] = (key.detach().clone(), value.detach().clone())+ kind, chunk = name+ for old in [n for n in entries if n[0] == kind and n[1] <= chunk - self.window]:+ entries.pop(old)+++@dataclass(frozen=True)+class H3SegmentPlan:+ """Query segments in document order; each attends to the union of its key spans and cached chunks."""++ segments: tuple[tuple[int, int, tuple[tuple[int, int], ...], tuple[tuple[str, int], ...], tuple[str, int] | None], ...]+ sequence_length: int+ cache: H3ChunkKVCache | None = None++ def __post_init__(self) -> None:+ end = 0+ for start, stop, spans, cached, commit in self.segments:+ if start != end or stop <= start:+ raise ValueError("segments must tile the document in order")+ if not spans and not cached:+ raise ValueError(f"segment [{start}, {stop}) sees nothing")+ if (cached or commit is not None) and self.cache is None:+ raise ValueError("cached or committed chunks need a cache")+ end = stop+ if end != self.sequence_length:+ raise ValueError("segments must cover the whole document")+++def _merge(spans: list[tuple[int, int]]) -> tuple[tuple[int, int], ...]:+ out: list[list[int]] = []+ for start, stop in sorted(s for s in spans if s[1] > s[0]):+ if out and start <= out[-1][1]:+ out[-1][1] = max(out[-1][1], stop)+ else:+ out.append([start, stop])+ return tuple((a, b) for a, b in out)+++def segment_attention(query: Any, key: Any, value: Any, plan: H3SegmentPlan, layer: int) -> Any:+ """``[B, S, H, D]`` attention over full sequence / local heads (after any Ulysses exchange), segment by segment."""++ import torch++ if query.device.type == "cuda":+ from flash_attn import flash_attn_func+ else: # CPU (tests): the same attention through SDPA++ def flash_attn_func(q, k, v, dropout_p=0.0, causal=False):+ out = torch.nn.functional.scaled_dot_product_attention(+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2)+ )+ return out.transpose(1, 2)++ if int(query.shape[1]) != plan.sequence_length:+ raise ValueError(f"segment plan covers {plan.sequence_length} rows, attention got {query.shape[1]}")+ outputs = []+ for start, stop, spans, cached, commit in plan.segments:+ keys = [key[:, a:b] for a, b in spans]+ values = [value[:, a:b] for a, b in spans]+ for chunk in cached:+ k, v = plan.cache.get(layer, chunk)+ keys.append(k)+ values.append(v)+ outputs.append(+ flash_attn_func(+ query[:, start:stop].contiguous(),+ torch.cat(keys, dim=1).contiguous(),+ torch.cat(values, dim=1).contiguous(),+ dropout_p=0.0,+ causal=False,+ )+ )+ if commit is not None:+ plan.cache.put(layer, commit, key[:, start:stop], value[:, start:stop])+ return torch.cat(outputs, dim=1)+++# --- documents -----------------------------------------------------------------------------------------------------+++def chunk_frames(frames: int, chunk: int) -> list[tuple[int, int]]:+ """Latent frame ranges of the chunks of a ``frames``-latent clip, ``chunk`` at most each."""++ if not 0 < int(chunk) <= 5:+ raise ValueError(f"a chunk holds 1 to 5 latents, got {chunk}")+ return [(f, min(f + int(chunk), int(frames))) for f in range(0, int(frames), int(chunk))]+++@dataclass(frozen=True)+class H3CausalRows:+ """Row ranges of an omni document (``[text | condition video | audio | target]``) by role and chunk."""++ sequence_length: int+ text: tuple[int, int]+ static: tuple[tuple[int, int], ...]+ structure: tuple[tuple[int, int], ...] # one per chunk, () without a structure video+ audio: tuple[int, int]+ target: tuple[tuple[int, int], ...] # one per chunk+ frames: tuple[tuple[int, int], ...] # latent frames of each chunk+ rows_per_frame: int++ @property+ def chunks(self) -> int:+ return len(self.target)+++def causal_rows(layout: H3PackedLayout, structure_index: int | None, chunk: int) -> H3CausalRows:+ """Rows of ``layout`` (a Stage0.5 omni layout) by chunk; ``structure_index`` is the structure video's place among+ the layout's visual observations (None: no structure video)."""++ if layout.stage != "stage0p5" or layout.num_clean_video_rows:+ raise ValueError("causal rows read an ordinary omni (Stage0.5) layout")+ rows = int(layout.rows_per_video_frame)+ frames = int(layout.num_noisy_video_rows) // rows+ ranges = chunk_frames(frames, chunk)+ text = (0, int(layout.text_indices.size))+ condition = (text[1], text[1] + int(layout.num_condition_video_rows))+ audio = (condition[1], condition[1] + int(layout.audio_indices.size))+ target_start = audio[1]+ if int(layout.target_video_indices[0]) != target_start or int(layout.target_video_indices[-1]) != layout.sequence_length - 1:+ raise ValueError("the omni layout's target rows must close the document")+ observation_start = condition[1] - int(layout.num_observation_rows)+ static = [text, (condition[0], observation_start)]+ structure: list[tuple[int, int]] = []+ offset = observation_start+ for index, count in enumerate(layout.observation_rows):+ if index == structure_index:+ if count % frames:+ raise ValueError("the structure video must hold one equal slab of rows per target frame")+ per = count // frames+ structure = [(offset + a * per, offset + b * per) for a, b in ranges]+ else:+ static.append((offset, offset + count))+ offset += count+ target = tuple((target_start + a * rows, target_start + b * rows) for a, b in ranges)+ return H3CausalRows(+ sequence_length=int(layout.sequence_length),+ text=text,+ static=_merge(static),+ structure=tuple(structure),+ audio=audio,+ target=target,+ frames=tuple(ranges),+ rows_per_frame=rows,+ )+++def _window(c: int, window: int) -> range:+ return range(max(0, c - int(window) + 1), c + 1)+++def training_plan(rows: H3CausalRows, *, window: int, teacher: bool) -> tuple[H3SegmentPlan, tuple[int, int]]:+ """The plan of a training document: teacher forcing appends a clean copy of the target after it (returned as its+ row range), diffusion forcing has none."""++ n = rows.sequence_length+ target_rows = rows.target[-1][1] - rows.target[0][0]+ clean = tuple((a + target_rows, b + target_rows) for a, b in rows.target) if teacher else ()+ static = list(rows.static)+ segments = []++ def add(span: tuple[int, int], keys: list[tuple[int, int]]) -> None:+ if span[1] > span[0]:+ segments.append((span[0], span[1], _merge(keys), (), None))++ # The leading rows in document order: text, condition video (static and structure interleaved), audio.+ pieces = [(s, "static", None) for s in rows.static] + [(s, "structure", c) for c, s in enumerate(rows.structure)]+ for span, kind, c in sorted(pieces):+ add(span, static if kind == "static" else [*static, span])+ add(rows.audio, [*static, rows.audio])+ for c, span in enumerate(rows.target):+ seen = [*static, rows.audio, *(rows.structure[j] for j in _window(c, window) if rows.structure)]+ if teacher:+ add(span, [*seen, *(clean[j] for j in _window(c, window) if j < c), span])+ else:+ add(span, [*seen, *(rows.target[j] for j in _window(c, window))])+ for c, span in enumerate(clean):+ seen = [*static, rows.audio, *(rows.structure[j] for j in _window(c, window) if rows.structure)]+ add(span, [*seen, *(clean[j] for j in _window(c, window))])+ end = n + (target_rows if teacher else 0)+ return H3SegmentPlan(tuple(segments), end), ((n, end) if teacher else (0, 0))+++def with_clean_copy(layout: Any) -> dict[str, Any]:+ """The model's index arrays for ``layout`` followed by a copy of its target rows (same positions and tags): video+ rows ordered condition, target, clean copy."""++ import torch++ target = layout.source.target_video_indices+ n = int(layout.source.sequence_length)+ copy = torch.arange(n, n + int(target.size), device=layout.position_ids.device)+ target_t = torch.from_numpy(target).to(layout.position_ids.device)+ return {+ "position_ids": torch.cat((layout.position_ids, layout.position_ids[target_t])),+ "token_tags": torch.cat((layout.token_tags, layout.token_tags[target_t])),+ "video_indices": torch.cat((layout.video_indices, copy)),+ "audio_indices": layout.audio_indices,+ "text_indices": layout.text_indices,+ }+++def select_rows(layout: Any, keep: np.ndarray) -> dict[str, Any]:+ """The model's index arrays for the rows ``keep`` (ascending) of ``layout``'s document, renumbered; video rows keep+ their document order, which is the order the caller packs them in."""++ import torch++ device = layout.position_ids.device+ keep = np.asarray(keep, dtype=np.int64)+ if keep.size == 0 or np.any(np.diff(keep) <= 0):+ raise ValueError("rows to keep must be strictly ascending")+ where = np.full(int(layout.source.sequence_length), -1, dtype=np.int64)+ where[keep] = np.arange(keep.size)++ def renumber(indices: np.ndarray) -> Any:+ mapped = where[indices]+ return torch.from_numpy(mapped[mapped >= 0]).to(device)++ keep_t = torch.from_numpy(keep).to(device)+ out = {+ "position_ids": layout.position_ids[keep_t],+ "token_tags": layout.token_tags[keep_t],+ "video_indices": renumber(layout.source.video_indices),+ "audio_indices": renumber(layout.source.audio_indices),+ "text_indices": renumber(layout.source.text_indices),+ }+ # Every row is a text, video or audio row (text rows may carry the video tag: Qwen's picture tokens).+ if sum(int(out[k].numel()) for k in ("video_indices", "audio_indices", "text_indices")) != int(keep.size):+ raise RuntimeError("selected rows lost their text/video/audio bookkeeping")+ return out+++def row_times(length: int, rows: H3CausalRows, *, target_times: list[float], audio: float, clean: tuple[int, int] = (0, 0)) -> np.ndarray:+ """Per-row data-ward times of a document of ``length`` rows: static text fixed, observations and the clean copy+ at the condition level, audio at ``audio``, target chunk ``c`` at ``target_times[c]``."""++ times = np.full(length, CLEAN_LEVEL, dtype=np.float32)+ times[rows.text[0] : rows.text[1]] = STATIC_TIME+ times[rows.audio[0] : rows.audio[1]] = audio+ for (a, b), t in zip(rows.target, target_times, strict=True):+ times[a:b] = t+ times[clean[0] : clean[1]] = CLEAN_LEVEL+ return times+++def unique_times(times: np.ndarray, device: Any) -> tuple[Any, Any]:+ import torch++ values, inverse = np.unique(times, return_inverse=True)+ return (+ torch.from_numpy(values.astype(np.float32)).to(device),+ torch.from_numpy(inverse.reshape(-1).astype(np.int64)).to(device),+ )+++__all__ = [+ "H3CausalRows",+ "H3ChunkKVCache",+ "H3SegmentPlan",+ "causal_rows",+ "chunk_frames",+ "row_times",+ "segment_attention",+ "select_rows",+ "training_plan",+ "unique_times",+ "with_clean_copy",+]+++# --- training and streaming ----------------------------------------------------------------------------------------+++def _structure_index(batch: Any) -> int | None:+ """The structure video's place among the batch's observations: the one laid over every target frame."""++ frames = int(batch.target_latents.shape[1]) if batch.target_latents is not None else None+ found = [+ i+ for i, item in enumerate(batch.visual_observations or ())+ if item.frames is not None+ and frames is not None+ and int(item.latents.shape[1]) == len(item.frames)+ and set(range(frames)) <= {int(f) for f in item.frames}+ ]+ if len(found) > 1:+ raise ValueError("an omni sample carries one structure video")+ return found[0] if found else None+++def _remap(span: tuple[int, int], where: np.ndarray) -> tuple[int, int]:+ return (int(where[span[0]]), int(where[span[1] - 1]) + 1)+++class H3OmniCausalMixin:+ """Block-causal omni training and streaming on an :class:`~.stage0p5.H3Stage0p5Core` (``train.causal_omni``)."""++ causal: dict++ def _document(self, batch: Any, generator: Any) -> tuple[Any, Any, Any, H3CausalRows, Any]:+ import torch++ prompt, tags = self.prompt(batch, use_anchor=False)+ for item in batch.visual_observations or ():+ if abs(float(item.degree) - CLEAN_LEVEL) > 1e-6:+ raise ValueError("block-causal omni reads observations at the condition level")+ specs, _degrees, observation_rows = self.observation_rows(batch, generator)+ frames, grid = self.latent_frames(batch), self.latent_grid(batch)+ layout = self.layout(tags, anchors=(), observations=specs, grid=grid, frames=frames)+ rows = causal_rows(layout.source, _structure_index(batch), int(self.causal.get("chunk", 5)))+ if observation_rows is None:+ observation_rows = torch.zeros((1, 0, 96), device=self.device)+ return prompt, layout, observation_rows, rows, (frames, grid)++ def _audio(self, frames: int, generator: Any) -> Any:+ import torch++ clean = self.audio_rows(frames)+ noise = torch.randn(clean.shape, generator=generator, device=self.device, dtype=torch.float32)+ self._broadcast_logical_sample(noise)+ return CLEAN_LEVEL * clean + (1.0 - CLEAN_LEVEL) * noise++ def _run(self, *, video_rows, audio_rows, prompt, arrays, times, plan, model: Any = None) -> Any:+ from .distributed import is_sequence_parallel_enabled++ values, inverse = unique_times(times, self.device)+ video, _audio = (self.model if model is None else model)(+ hidden_states=video_rows,+ audio_hidden_states=audio_rows,+ encoder_hidden_states=prompt,+ timestep=values,+ timestep_indices=inverse,+ token_tags=arrays["token_tags"],+ position_ids=arrays["position_ids"],+ video_indices=arrays["video_indices"],+ audio_indices=arrays["audio_indices"],+ text_indices=arrays["text_indices"],+ attention_mask=plan,+ fused_prope=False,+ stage0p5_sequence_parallel=is_sequence_parallel_enabled(),+ return_dict=False,+ )+ return video++ def forward_loss(self, batch: Any, *, noise_seed: int | None) -> Any:+ """Flow matching on every chunk, each at its own noise level, under the block-causal plan: teacher forcing+ (``mode: teacher``, the clean target as history, optionally corrupted) or diffusion forcing (``diffusion``)."""++ import torch+ import torch.nn.functional as F++ from .layout import patchify_video++ generator = None if noise_seed is None else torch.Generator(device=self.device).manual_seed(int(noise_seed))+ teacher = str(self.causal.get("mode", "teacher")) == "teacher"+ document = self._document(batch, generator)+ _prompt, _layout, _observations, rows, (frames, _grid) = document+ clean = batch.target_latents.to(self.device, dtype=torch.float32).unsqueeze(0)+ noise = torch.randn(clean.shape, generator=generator, device=self.device, dtype=torch.float32)+ draws = [self.shifted_timestep(generator, shift=12.0, device=self.device) for _ in rows.frames]+ self._broadcast_logical_sample(clean, noise, *draws)+ level = torch.ones((1, 1, frames, 1, 1), device=self.device)+ for (a, b), t in zip(rows.frames, draws, strict=True):+ level[:, :, a:b] = t+ noisy = level * clean + (1.0 - level) * noise+ history = None+ if teacher:+ history = clean+ corrupt = float(self.causal.get("corrupt", 0.0))+ if corrupt > 0:+ # A clean history the model never meets in a stream: each chunk, with probability one half, mixed+ # with fresh noise by a uniform amount up to ``corrupt`` (AgentGarten corrupts its history likewise).+ mix = torch.rand(len(rows.frames), generator=generator, device=self.device) * corrupt+ pick = torch.rand(len(rows.frames), generator=generator, device=self.device) < 0.5+ extra = torch.randn(clean.shape, generator=generator, device=self.device, dtype=torch.float32)+ self._broadcast_logical_sample(mix, pick, extra)+ amount = torch.zeros((1, 1, frames, 1, 1), device=self.device)+ for (a, b), m, p in zip(rows.frames, mix, pick, strict=True):+ amount[:, :, a:b] = m * p.float()+ history = (1.0 - amount) * clean + amount * extra+ prediction = self.training_prediction(+ document, noisy, [float(t) for t in draws], history, generator=generator+ )+ target = patchify_video(clean - noise)+ return F.mse_loss(prediction.float(), target.float())++ def training_prediction(+ self, document: Any, noisy: Any, times: list[float], history: Any, *, generator: Any = None,+ jitter: Any = None, model: Any = None,+ ) -> Any:+ """The velocity rows of every target chunk of a training document: ``noisy`` the target with chunk ``c`` at+ time ``times[c]``, ``history`` the clean copy's content (teacher forcing) or None (diffusion forcing)."""++ import torch++ from .layout import patchify_video++ prompt, layout, observation_rows, rows, (frames, _grid) = document+ teacher = history is not None+ parts = [observation_rows, patchify_video(noisy)]+ plan, clean_span = training_plan(rows, window=int(self.causal.get("window", 6)), teacher=teacher)+ if teacher:+ if jitter is None:+ jitter = torch.randn(history.shape, generator=generator, device=self.device, dtype=torch.float32)+ self._broadcast_logical_sample(jitter)+ parts.append(patchify_video(CLEAN_LEVEL * history + (1.0 - CLEAN_LEVEL) * jitter))+ arrays = with_clean_copy(layout)+ else:+ arrays = {+ "position_ids": layout.position_ids,+ "token_tags": layout.token_tags,+ "video_indices": layout.video_indices,+ "audio_indices": layout.audio_indices,+ "text_indices": layout.text_indices,+ }+ row_t = row_times(plan.sequence_length, rows, target_times=times, audio=CLEAN_LEVEL, clean=clean_span)+ video = self._run(+ video_rows=torch.cat(parts, dim=1), audio_rows=self._audio(frames, generator), prompt=prompt,+ arrays=arrays, times=row_t, plan=plan, model=model,+ )+ start = int(observation_rows.shape[1])+ return video[:, start : start + int(patchify_video(noisy).shape[1])]++ def generate_causal(+ self, batch: Any, *, noise_seed: int, steps: int, window: int | None = None, record: list | None = None,+ cache_structure: bool = False,+ ) -> Any:+ """The clip chunk by chunk, as a stream renders it: each chunk denoised over ``steps`` schedule points (the+ shifted grid, ``steps - 1`` Euler forwards) against the cached keys and values of the chunks before it, then+ written to the cache at the condition level. Returns ``[1, 24, frames, h, w]`` latents."""++ import torch++ from .flow import make_shifted_schedule++ generator = torch.Generator(device=self.device).manual_seed(int(noise_seed))+ clip, _noisy, _times, _jitter = self.stream(+ batch, generator=generator, schedule=make_shifted_schedule(int(steps), shift=12.0),+ window=window, record=record, cache_structure=cache_structure,+ )+ return clip++ def stream(+ self,+ batch: Any,+ *,+ generator: Any,+ schedule: Any,+ window: int | None = None,+ exit_index: int | None = None,+ model: Any = None,+ record: list | None = None,+ cache_structure: bool = True,+ ) -> tuple[Any, Any, list[float], Any]:+ """Render ``batch`` chunk by chunk through a key/value cache. Without ``exit_index``: Euler over ``schedule``.+ With it (a few-step student, as DMD trains one): every chunk's clean estimate is re-noised to the next level+ between evaluations and taken at evaluation ``exit_index``. Returns the clean clip, the input each chunk had at+ its last evaluation, that evaluation's time per chunk, and the noise each committed chunk was written with -+ everything a replay of the same chunks needs (:meth:`training_prediction`)."""++ import torch++ from .layout import patchify_video, unpatchify_video++ window = int(window or self.causal.get("window", 6))+ prompt, layout, observation_rows, rows, (frames, grid) = self._document(batch, generator)+ audio = self._audio(frames, generator)+ cache = H3ChunkKVCache(window=window)+ source = layout.source+ n_cond = int(source.num_condition_video_rows)+ video_doc_rows = source.video_indices # document rows of the video rows, in packing order+ clip = torch.zeros((1, 24, frames, *grid), device=self.device)+ last_inputs, jitters = torch.zeros_like(clip), torch.zeros_like(clip)+ last_times: list[float] = []+ evaluations = len(schedule.timesteps) if exit_index is None else int(exit_index) + 1++ def draw(shape: tuple[int, ...]) -> Any:+ value = torch.randn(shape, generator=generator, device=self.device, dtype=torch.float32)+ self._broadcast_logical_sample(value)+ return value++ static_obs = [s for s in rows.static if s[0] >= rows.text[1]] # the first frame (and other static pictures)++ def sub_document(target: tuple[int, int], extra: list = ()) -> tuple[Any, np.ndarray, list, tuple[int, int], Any]:+ """The rows a stream recomputes for one forward: the static rows, the audio, ``extra`` (the structure window+ when it is not cached) and ``target`` (a structure or target chunk) - everything else comes from the cache."""++ spans = sorted([*rows.static, rows.audio, *extra, target])+ keep = np.concatenate([np.arange(a, b) for a, b in spans])+ where = np.full(rows.sequence_length, -1, dtype=np.int64)+ where[keep] = np.arange(keep.size)+ kept_video = np.asarray([r for r in video_doc_rows[:n_cond] if where[r] >= 0], dtype=np.int64)+ pick = torch.from_numpy(np.searchsorted(video_doc_rows[:n_cond], kept_video)).to(self.device)+ times = np.full(int(keep.size), CLEAN_LEVEL, dtype=np.float32)+ text = _remap(rows.text, where)+ times[text[0] : text[1]] = STATIC_TIME+ return select_rows(layout, keep), where, pick, _remap(target, where), times++ def plan_for(+ where: np.ndarray, mine: tuple[int, int], size: int, keys: list, cached: tuple, commit: Any, extra: list = ()+ ) -> H3SegmentPlan:+ static = [_remap(s, where) for s in rows.static]+ audio = _remap(rows.audio, where)+ pieces = [(span, static) for span in static] + [(audio, [*static, audio])]+ # structure chunks recomputed in this forward see the static rows and themselves, as always+ pieces += [(_remap(e, where), [*static, _remap(e, where)]) for e in extra]+ segments = [(a, b, _merge(k), (), None) for (a, b), k in sorted(pieces) if b > a]+ segments.append((mine[0], mine[1], _merge([*static, *keys, mine]), cached, commit))+ segments.sort()+ return H3SegmentPlan(tuple(segments), size, cache)++ for c, (f0, f1) in enumerate(rows.frames):+ history = tuple(("structure", j) for j in _window(c, window) if rows.structure and cache_structure) + tuple(+ ("clean", j) for j in _window(c, window) if j < c+ )+ # without a structure cache (one card: its keys and values would not fit beside the model) the window's+ # structure is recomputed in every forward - the same numbers, more work+ extra = [] if cache_structure or not rows.structure else [rows.structure[j] for j in _window(c, window)]+ if rows.structure and cache_structure:+ # The chunk's structure, once: it sees the static rows and itself, so its keys and values never change.+ arrays, where, pick, mine, times = sub_document(rows.structure[c])+ with torch.no_grad():+ self._run(+ video_rows=observation_rows.index_select(1, pick), audio_rows=audio, prompt=prompt,+ arrays=arrays, times=times, plan=plan_for(where, mine, len(times), [], (), ("structure", c)),+ model=model,+ )+ arrays, where, pick, mine, times = sub_document(rows.target[c], extra)+ condition = observation_rows.index_select(1, pick)+ audio_span = [_remap(rows.audio, where), *(_remap(e, where) for e in extra)]+ shape = (1, 24, f1 - f0, *grid)+ current = draw(shape)+ for index in range(evaluations):+ sigma, sigma_next = float(schedule.sigmas[index]), float(schedule.sigmas[index + 1])+ times[mine[0] : mine[1]] = 1.0 - sigma+ rows_out = self._run(+ video_rows=torch.cat((condition, patchify_video(current)), dim=1), audio_rows=audio,+ prompt=prompt, arrays=arrays, times=times.copy(),+ plan=plan_for(where, mine, len(times), audio_span, history, None, extra), model=model,+ )+ velocity = unpatchify_video(+ rows_out[:, int(condition.shape[1]) :], f1 - f0, *grid, channels=24+ ).float()+ if record is not None:+ record.append((c, index, current.clone(), velocity.clone(), 1.0 - sigma))+ last_inputs[:, :, f0:f1] = current+ if index == evaluations - 1:+ last_times.append(1.0 - sigma)+ current = current + (sigma if exit_index is not None else sigma - sigma_next) * velocity+ break+ if exit_index is None:+ current = current + (sigma - sigma_next) * velocity+ else:+ current = (1.0 - sigma_next) * (current + sigma * velocity) + sigma_next * draw(shape)+ clip[:, :, f0:f1] = current+ jitter = draw(shape)+ jitters[:, :, f0:f1] = jitter+ times[mine[0] : mine[1]] = CLEAN_LEVEL+ with torch.no_grad():+ self._run(+ video_rows=torch.cat((condition, patchify_video(CLEAN_LEVEL * current + (1 - CLEAN_LEVEL) * jitter)), dim=1),+ audio_rows=audio, prompt=prompt, arrays=arrays, times=times.copy(),+ plan=plan_for(where, mine, len(times), audio_span, history, ("clean", c), extra), model=model,+ )+ return clip, last_inputs, last_times, jitters+++def _core_class() -> type:+ from .stage0p5 import H3Stage0p5Core++ class H3OmniCausalCore(H3OmniCausalMixin, H3Stage0p5Core):+ """An omni core whose training loss and sampler are block-causal (``train.causal_omni``)."""++ def __init__(self, model: Any, silence_latents: Any, device: Any, causal: dict, **kwargs: Any) -> None:+ H3Stage0p5Core.__init__(self, model, silence_latents, device, **kwargs)+ validate_causal_omni(causal)+ self.causal = dict(causal)++ return H3OmniCausalCore+++def validate_causal_omni(block: Any) -> None:+ from collections.abc import Mapping++ from solarwm.errors import ConfigurationError++ if not isinstance(block, Mapping) or set(block) - {"mode", "chunk", "window", "corrupt"}:+ raise ConfigurationError("train.causal_omni holds mode, chunk, window and corrupt")+ if str(block.get("mode", "teacher")) not in {"teacher", "diffusion"}:+ raise ConfigurationError("train.causal_omni.mode must be teacher or diffusion")+ chunk, window = block.get("chunk", 5), block.get("window", 6)+ if isinstance(chunk, bool) or not isinstance(chunk, int) or not 1 <= chunk <= 5:+ raise ConfigurationError("train.causal_omni.chunk must be 1 to 5 latents")+ if isinstance(window, bool) or not isinstance(window, int) or window < 1:+ raise ConfigurationError("train.causal_omni.window must be a whole number of chunks >= 1")+ corrupt = block.get("corrupt", 0.0)+ if isinstance(corrupt, bool) or not isinstance(corrupt, (int, float)) or not 0.0 <= float(corrupt) < 1.0:+ raise ConfigurationError("train.causal_omni.corrupt must lie in [0, 1)")+ if float(corrupt) > 0 and str(block.get("mode", "teacher")) != "teacher":+ raise ConfigurationError("train.causal_omni.corrupt corrupts a teacher-forced history")+++def __getattr__(name: str) -> Any:+ if name == "H3OmniCausalCore":+ return _core_class()+ if name == "H3OmniCausalDMDCore":+ return _dmd_core_class()+ raise AttributeError(name)+++def _dmd_core_class() -> type:+ from .omni_dmd import H3OmniDMDCore++ class H3OmniCausalDMDCore(H3OmniCausalMixin, H3OmniDMDCore):+ """The omni DMD core with a block-causal student: the student streams and replays (:meth:`stream`,+ :meth:`training_prediction`), the teacher and critic score the whole clip as before."""++ def __init__(self, model: Any, silence_latents: Any, device: Any, train: Any, causal: dict, **kwargs: Any):+ H3OmniDMDCore.__init__(self, model, silence_latents, device, train, **kwargs)+ self.causal = {"mode": "teacher", **dict(causal)}++ return H3OmniCausalDMDCorediff --git a/src/solarwm/backends/minimax_h3/omni_data.py b/src/solarwm/backends/minimax_h3/omni_data.pynew file mode 100644index 0000000..9459554--- /dev/null+++ b/src/solarwm/backends/minimax_h3/omni_data.py@@ -0,0 +1,353 @@+"""Reader for omni samples: one safetensors member per sample, from ``tools/omni_preencode.py``.++A sample carries its target latents (any ``5n+2`` length, landscape or portrait), the released+``ref2va`` prompt embeddings, any number of untyped visual observations and, optionally, the known+degree of every target row. It has no first-frame anchor and no camera unless the member provides+them. The reader is deliberately small next to the release reader: an index of members, a+deterministic shuffle per epoch, a logical data-parallel partition, and a cursor that survives a+checkpoint. Members live wherever they were written, a directory or a remote object-storage prefix.+"""++from __future__ import annotations++import contextlib+import json+import os+import tempfile+from collections.abc import Mapping+from dataclasses import replace+from pathlib import Path+from typing import Any++import numpy as np++from solarwm import storage+from solarwm.errors import DataContractError++from .artifacts import (+ H3ArtifactBatch,+ H3VisualObservation,+ read_encoder_contract,+ validate_absolute_c2w,+ validate_normalized_intrinsics,+ views_from_c2w,+)+++def is_remote(path: str) -> bool:+ """A remote object-storage location (``<scheme>://...``), read through solarwm.storage."""++ return storage.is_remote(path)+++def _local(path: str):+ """Yield a local file for ``path``; a remote object goes through scratch and is removed."""++ if not is_remote(path):+ yield path+ return+ handle, scratch = tempfile.mkstemp(+ suffix=Path(path).suffix, dir=os.environ.get("OMNI_SCRATCH", "/tmp")+ )+ os.close(handle)+ try:+ storage.get(path, scratch)+ yield scratch+ finally:+ Path(scratch).unlink(missing_ok=True)+++def coarse(item: H3VisualObservation, spec: Mapping[str, Any]) -> H3VisualObservation:+ """An aligned video observation read coarser than it was encoded: its latent grid averaged over+ ``pool`` x ``pool`` cells and every ``stride``-th latent kept, each on its own target frame.++ The layout lays an aligned observation over the whole canvas whatever its grid, so a half-grid+ structure video still covers the picture; it costs a quarter of the tokens, and a third of them+ again in time (CWM gives its proxy at a quarter of the target, one frame in twelve). H3's front+ end resizes every video reference to its own size, so the coarse grid is made here, from the+ member already encoded. A single frame (the first frame) and a free observation are left alone.+ """++ import torch++ pool, stride = int(spec.get("pool") or 1), int(spec.get("stride") or 1)+ if item.frames is None or len(item.frames) < 2 or (pool <= 1 and stride <= 1):+ return item+ latents, frames, boxes = item.latents, item.frames, item.boxes+ if stride > 1:+ latents, frames = latents[:, ::stride], frames[::stride]+ # One box per frame follows its frame.+ if boxes is not None and len(boxes) == len(item.frames):+ boxes = boxes[::stride]+ if pool > 1:+ # Each side to the nearest even number of cells at 1/pool (the 2x2 patch needs even+ # sides):+ # 48x84 at a quarter is 12x21, read as 12x22, every cell the average of the latents it+ # overlaps. The grid still spans the whole canvas, as the layout lays it.+ height, width = int(latents.shape[-2]), int(latents.shape[-1])+ size = tuple(max(2, 2 * int(side / pool / 2 + 0.5)) for side in (height, width))+ if size == (height // pool, width // pool) and not height % pool and not width % pool:+ latents = torch.nn.functional.avg_pool2d(latents.float(), pool)+ else:+ latents = torch.nn.functional.adaptive_avg_pool2d(latents.float(), size)+ latents = latents.to(item.latents.dtype)+ return replace(item, latents=latents.contiguous(), frames=tuple(frames), boxes=boxes)+++class H3OmniStream:+ def __init__(+ self,+ *,+ root: str,+ index: str,+ topology: Any,+ seed: int,+ encoder_contract_path: str,+ shuffle: bool = True,+ aligned_control: bool = False,+ length_buckets: bool = False,+ ) -> None:+ # ``root`` and ``index`` are local paths or remote object-storage locations.+ self.root = root.rstrip("/")+ with _local(index) as path, open(path) as lines:+ rows = [json.loads(line) for line in lines if line.strip()]+ if not rows:+ raise DataContractError(f"omni index {index} is empty")+ self.rows = sorted(rows, key=lambda row: row["id"])+ self.rank, self.world = int(topology.dp_rank), int(topology.dp_world_size)+ if len(self.rows) < self.world:+ raise DataContractError("omni index holds fewer samples than data-parallel readers")+ self.seed, self.shuffle = int(seed), bool(shuffle)+ # A run whose control trunk reads the aligned observation cannot use a sample without one+ # laid over every target frame; such a sample is skipped here, with the unreadable ones,+ # rather than failing inside a training step where one rank's error stalls every rank.+ self.aligned_control = bool(aligned_control)+ # Every data-parallel reader draws a clip of the same length at the same step, so a step is+ # not held to the longest of forty random clips.+ self.length_buckets = bool(length_buckets)+ _, self.encoder_profile = read_encoder_contract(encoder_contract_path)+ self.epoch, self.cursor = 0, 0+ self.current_plan_fingerprint = f"omni:{Path(index).name}:{len(self.rows)}"++ def _order(self) -> np.ndarray:+ """This reader's rows for the epoch, computed once per epoch (an index runs to millions)."""++ cached = getattr(self, "_cached_order", None)+ if cached is not None and cached[0] == self.epoch:+ return cached[1]+ order = None+ if getattr(self, "length_buckets", False) and self.shuffle:+ order = self._bucketed_order()+ # An index too thin to fill one run per group would yield nothing and spin epochs.+ order = order if len(order) else None+ if order is None:+ order = np.arange(len(self.rows))+ if self.shuffle:+ np.random.default_rng([self.seed, self.epoch]).shuffle(order)+ usable = (+ len(order) - len(order) % self.world+ ) # every reader sees the same number of samples+ order = order[:usable]+ mine = order[self.rank :: self.world]+ self._cached_order = (self.epoch, mine)+ return mine++ def _bucketed_order(self) -> np.ndarray:+ """The epoch's global order in runs of ``world`` rows that share a clip length.++ Rows are grouped by target length and number of observations (what sets the packed+ sequence), each group shuffled and cut into runs of one row per reader, and the runs+ shuffled together. A group's last partial run waits for a later epoch. Readers take every+ ``world``-th row as before, so at each step they all hold the same length.+ """++ rng = np.random.default_rng([self.seed, self.epoch])+ groups: dict[tuple[int, int], list[int]] = {}+ for index, row in enumerate(self.rows):+ key = (int(row.get("latent_frames") or 47), int(row.get("observations") or 0))+ groups.setdefault(key, []).append(index)+ runs = []+ for key in sorted(groups):+ members = np.asarray(groups[key], dtype=np.int64)+ rng.shuffle(members)+ usable = len(members) - len(members) % self.world+ runs.extend(members[:usable].reshape(-1, self.world))+ if not runs:+ return np.zeros(0, dtype=np.int64)+ runs = np.stack(runs)+ rng.shuffle(runs)+ return runs.reshape(-1)++ def state_dict(self) -> dict[str, int]:+ return {"epoch": self.epoch, "cursor": self.cursor}++ def load_state_dict(self, state: dict[str, int]) -> None:+ self.epoch, self.cursor = int(state["epoch"]), int(state["cursor"])++ def close(self) -> None:+ return None++ def __iter__(self):+ while True:+ yield self.next()++ def next(self) -> H3ArtifactBatch:+ failures = 0+ while True:+ order = self._order()+ if self.cursor >= len(order):+ self.epoch, self.cursor = self.epoch + 1, 0+ continue+ row = self.rows[int(order[self.cursor])]+ self.cursor += 1+ try:+ return self.read(row)+ except Exception as exc: # one bad member must not take the run down+ failures += 1+ print(f"[omni-data] skipped {row.get('id')}: {exc!r}", flush=True)+ if failures >= 32:+ raise DataContractError("32 consecutive omni members failed to load") from exc++ def camera(self, tensors: Mapping[str, Any], latent_frames: int) -> tuple[Any, Any]:+ """The clip's camera in the model's frame, or nothing when the member carries none.++ A member stores absolute camera-to-world poses, one per latent, exactly as the released+ artifacts do; the first camera of the clip becomes the origin here, as it does there.+ """++ import torch++ c2w, intrinsics = tensors.get("camera_c2w"), tensors.get("camera_K")+ if c2w is None and intrinsics is None:+ return None, None+ if c2w is None or intrinsics is None:+ raise DataContractError("an omni camera needs both the poses and the intrinsics")+ if tuple(c2w.shape) != (latent_frames, 4, 4):+ raise DataContractError(+ f"omni camera holds {tuple(c2w.shape)}, not [{latent_frames},4,4]"+ )+ if tuple(intrinsics.shape) == (3, 3):+ intrinsics = intrinsics.unsqueeze(0).expand(latent_frames, -1, -1).clone()+ if tuple(intrinsics.shape) != (latent_frames, 3, 3):+ raise DataContractError(+ f"omni intrinsics hold {tuple(intrinsics.shape)}, not [{latent_frames},3,3]"+ )+ c2w, intrinsics = c2w.float(), intrinsics.float()+ if not bool(torch.isfinite(c2w).all()) or not bool(torch.isfinite(intrinsics).all()):+ raise DataContractError("omni camera holds non-finite values")+ validate_absolute_c2w(c2w.numpy())+ validate_normalized_intrinsics(intrinsics.numpy())+ views = views_from_c2w(c2w)+ if float(views.abs().max()) > 20.0:+ raise DataContractError("omni camera exceeds the magnitude guard")+ return views, intrinsics.contiguous()++ def first_frame(self, path: str, target: Any) -> H3VisualObservation:+ """The clip's first frame, encoded apart from its member, as an observation on latent 0.++ An index row may name a file holding ``first_frame_latents`` [24,1,h,w] (the frame held as+ a still and encoded on the target's grid, as ``tools/omni_preencode.py`` encodes an aligned+ image). It joins the member's observations as given; the member's prompt is kept, so the+ frame reaches the model through its latent alone and dropping it leaves exactly the sample+ without one - the same input a render without a first frame gives.+ """++ import torch+ from safetensors import safe_open++ if not (is_remote(path) or path.startswith("/")):+ path = f"{self.root}/{path}"+ with _local(path) as local, safe_open(local, framework="pt") as handle:+ latents = handle.get_tensor("first_frame_latents")+ if tuple(latents.shape) != (24, 1, *target.shape[2:]):+ raise DataContractError(+ f"first frame latents {tuple(latents.shape)} are not [24,1,"+ f"{','.join(str(v) for v in target.shape[2:])}]"+ )+ if not bool(torch.isfinite(latents.float()).all()):+ raise DataContractError("first frame latents hold non-finite values")+ return H3VisualObservation(latents=latents.contiguous(), frames=(0,), degree=0.999)++ def read(self, row: dict[str, Any]) -> H3ArtifactBatch:+ import torch+ from safetensors import safe_open++ member = row["member"]+ if not (is_remote(member) or member.startswith("/")):+ member = f"{self.root}/{member}" # an index may also name its members outright+ with _local(member) as path, safe_open(path, framework="pt") as handle:+ meta = json.loads((handle.metadata() or {}).get("omni", "{}"))+ tensors = {key: handle.get_tensor(key) for key in handle.keys()} # noqa: SIM118+ target, prompt, tags = (+ tensors["target_latents"],+ tensors["prompt_embeds"],+ tensors["text_token_tags"],+ )+ if target.ndim != 4 or target.shape[0] != 24 or (target.shape[1] - 2) % 5:+ raise DataContractError(+ f"omni target latents {tuple(target.shape)} are not [24,5n+2,h,w]"+ )+ if tuple(target.shape[2:]) not in {(48, 84), (84, 48)}:+ raise DataContractError(+ f"omni target grid {tuple(target.shape[2:])} is not 48x84 or 84x48"+ )+ if prompt.ndim != 2 or prompt.shape[-1] != 5120 or tags.shape != prompt.shape[:1]:+ raise DataContractError("omni prompt embeddings differ")+ if not bool(torch.isfinite(target.float()).all()) or not bool(+ torch.isfinite(prompt.float()).all()+ ):+ raise DataContractError("omni member holds non-finite values")+ observations = []+ for index, item in enumerate(meta.get("observations", ())):+ latents = tensors[f"observation_{index}_latents"]+ frames = item.get("frames")+ observations.append(+ H3VisualObservation(+ latents=latents.contiguous(),+ frames=None if frames is None else tuple(int(f) for f in frames),+ degree=float(item.get("degree", 0.999)),+ boxes=None+ if not item.get("boxes")+ else tuple(tuple(float(v) for v in box) for box in item["boxes"]),+ )+ )+ if row.get("coarse_observations"):+ observations = [coarse(item, row["coarse_observations"]) for item in observations]+ if row.get("free_observations"):+ # The unaligned-reference ablation: the structure video placed as the base's own reference,+ # its own stretch of the time axis; a single-frame observation (the first frame) stays aligned.+ observations = [+ replace(item, frames=None) if item.frames is not None and len(item.frames) > 1 else item+ for item in observations+ ]+ if row.get("first_frame"):+ observations.append(self.first_frame(row["first_frame"], target))+ views, intrinsics = self.camera(tensors, int(target.shape[1]))+ batch = H3ArtifactBatch(+ sample_id=str(row["id"]),+ start_frame=0,+ plan_fingerprint=self.current_plan_fingerprint,+ target_latents=target.contiguous(),+ anchor_latents=None,+ prompt_embeds=prompt.contiguous(),+ text_token_tags=tags.contiguous(),+ source_frame_indices=None,+ camera_viewmats=views,+ camera_K=intrinsics,+ source_fps=24.0,+ dataset_source=str(meta.get("source", row.get("source", ""))),+ # The ref2va prompt never shows the first frame, so it is also the anchor-free prompt.+ text_prompt_embeds=prompt.contiguous(),+ text_prompt_tags=tags.contiguous(),+ visual_observations=tuple(observations) or None,+ known_degrees=tensors.get("known_degrees"),+ )+ if self.aligned_control:+ from .stage0p5 import optional_control_latents++ optional_control_latents(batch) # a structure video the trunk cannot take fails here+ return batch+++__all__ = ["H3OmniStream"]diff --git a/src/solarwm/backends/minimax_h3/omni_dmd.py b/src/solarwm/backends/minimax_h3/omni_dmd.pynew file mode 100644index 0000000..9a2bd51--- /dev/null+++ b/src/solarwm/backends/minimax_h3/omni_dmd.py@@ -0,0 +1,341 @@+"""Structure-guided DMD2 few-step distillation on omni samples, from a released four-step LoRA.++The sample is prepared exactly as the ordinary omni recipe prepares it (structure packed beside the+target, first frame, prompt); every role - student, teacher (real score), critic (fake score) - sees+the same conditions and differs only in its weights, so each runs on its own model copy through+``_forward(model=...)``. The student samples as the public four-step LoRAs (LongLive-Plug, DMAD) were+distilled: a fresh-noise (re-noised) Euler rollout over ``forwards`` evaluations of the 12-shifted grid, with one+randomly chosen exit evaluation carrying the graph (DMD2 backward simulation, Self-Forcing / the+LongLive-Plug reference). The score models see that sample re-noised at a shifted, clamped level.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any++from .flow import make_shifted_schedule+from .layout import patchify_video, unpatchify_video+from .stage0p5 import H3Stage0p5Core, _Prepared+from .torch_flow import shift_sigma+++@dataclass+class H3DMDInputs:+ prepared: _Prepared+ audio_clean: Any+ frames: int+ grid: tuple[int, int]+++def _is_structure(batch: Any, item: Any) -> bool:+ """Whether an observation is the structure video: laid over every target frame. Kept here rather than imported, so+ the distillation code reads the same with the code directory its runs were launched from."""++ frames = int(batch.target_latents.shape[1])+ return (+ item.frames is not None+ and int(item.latents.shape[1]) == len(item.frames)+ and set(range(frames)) <= {int(f) for f in item.frames}+ )+++def structure_guided(real: Any, free: Any, guidance: float) -> Any:+ """The teacher's clean answer pushed along the structure direction: ``real + (w - 1) * (real - free)``,+ ``free`` the teacher's answer with the structure video taken out. w = 1 is the plain teacher. The 50-step teacher is+ softer and less faithful to the structure than its own few-step student; matching a teacher guided along what the+ structure adds asks the student for the structure-faithful distribution instead of the teacher's blur."""++ return real + (float(guidance) - 1.0) * (real - free)+++def anchor_condition_weight(batch: Any, weights: dict[str, float]) -> float:+ """How much the real-data anchor counts for a sample, by what it was given: structure and first frame+ (``full``), structure only, first frame only, or neither (``text``). A sample that names the target closely gives a+ single right answer to regress to; one with little given has many right answers, and regressing to its mean blurs."""++ aligned = [o for o in getattr(batch, "visual_observations", None) or () if o.frames is not None]+ structure = any(len(o.frames) > 1 for o in aligned)+ first = any(tuple(int(f) for f in o.frames) == (0,) for o in aligned)+ kind = "full" if structure and first else "structure" if structure else "first" if first else "text"+ return float(weights.get(kind, 0.0))+++class H3OmniDMDCore(H3Stage0p5Core):+ """Stage0.5 omni conditions with a few-step student rollout and score evaluations."""++ def __init__(+ self,+ model: Any,+ silence_latents: Any,+ device: Any,+ train: Any,+ *,+ geometry: Any = None,+ observations: Any = None,+ ) -> None:+ super().__init__(+ model, silence_latents, device, geometry=geometry, observations=observations+ )+ self.train = train++ def prepare_inputs(self, batch: Any, generator: Any = None) -> H3DMDInputs:+ prepared = self.prepare(+ batch,+ generator,+ lambda draw: self.shifted_timestep(draw, shift=12.0, device=self.device),+ )+ plan = prepared.plan+ # Every target row moves along one schedule; rows held at their own degree have no place.+ if plan is not None and plan.pattern != "shared":+ raise ValueError("DMD distillation needs the shared observation pattern alone")+ frames = self.latent_frames(batch)+ return H3DMDInputs(prepared, self.audio_rows(frames), frames, self.latent_grid(batch))++ def _random(self, shape: tuple[int, ...], generator: Any) -> Any:+ import torch++ value = torch.randn(shape, generator=generator, device=self.device, dtype=torch.float32)+ self._broadcast_logical_sample(value)+ return value++ def _video_forward(+ self,+ model: Any,+ inputs: H3DMDInputs,+ sample: Any,+ video_t: Any,+ audio_t: Any,+ audio_noise: Any,+ ) -> Any:+ import torch++ p = inputs.prepared+ rows = self._forward(+ video_rows=torch.cat((*p.condition_rows, patchify_video(sample)), dim=1),+ audio_rows=audio_t * inputs.audio_clean + (1.0 - audio_t) * audio_noise,+ prompt=p.prompt,+ layout=p.layout,+ video_t=video_t,+ audio_t=audio_t,+ viewmats=p.viewmats,+ K=p.K,+ frame_ids=p.frame_ids,+ geometry_control=p.geometry,+ observation_degrees=p.observation_degrees,+ model=model,+ )+ return unpatchify_video(rows, inputs.frames, *inputs.grid, channels=24).float()++ def rollout(+ self,+ inputs: H3DMDInputs,+ *,+ model: Any,+ forwards: int,+ exit_index: int,+ generator: Any = None,+ ) -> Any:+ """The student's clean estimate at evaluation ``exit_index`` (0-based), with its graph.++ Evaluations before the exit run without a graph; between evaluations the clean estimate is+ noised afresh to the next level, as the four-step LoRA samples (``generate(renoise=True)``).+ """++ import torch++ if not 0 <= int(exit_index) < int(forwards):+ raise ValueError(f"exit_index {exit_index} outside {forwards} evaluations")+ video = make_shifted_schedule(int(forwards) + 1, shift=12.0)+ audio = make_shifted_schedule(int(forwards) + 1, shift=3.0)+ if len(video.timesteps) != int(forwards) or len(audio.timesteps) != int(forwards):+ raise RuntimeError("the shifted grid collapsed points; too many evaluations")+ shape = (1, 24, inputs.frames, *inputs.grid)+ audio_noise = self._random(tuple(inputs.audio_clean.shape), generator)+ current = self._random(shape, generator)+ for index in range(int(exit_index) + 1):+ video_t = torch.tensor(float(video.timesteps[index]), device=self.device)+ audio_t = torch.tensor(float(audio.timesteps[index]), device=self.device)+ sigma = float(video.sigmas[index])+ with torch.set_grad_enabled(index == int(exit_index) and torch.is_grad_enabled()):+ velocity = self._video_forward(+ model, inputs, current, video_t, audio_t, audio_noise+ )+ denoised = current + sigma * velocity+ if index == int(exit_index):+ return denoised+ sigma_next = float(video.sigmas[index + 1])+ current = (1.0 - sigma_next) * denoised.detach() + sigma_next * self._random(+ shape, generator+ )+ raise AssertionError("unreachable")++ def score_inputs(self, clean: Any, generator: Any = None) -> tuple[Any, Any, Any, Any]:+ """``clean`` noised at one level: uniform, H3-shifted (video 12 / audio 3), clamped."""++ import torch++ raw = torch.rand((), generator=generator, device=self.device, dtype=torch.float32)+ self._broadcast_logical_sample(raw)+ shift = float(self.train.get("score_shift", 12.0))+ sigma = shift_sigma(raw, shift).clamp(0.02, 0.98)+ audio_t = 1.0 - shift_sigma(raw, 3.0).clamp(0.02, 0.98)+ noise = self._random(tuple(clean.shape), generator)+ noisy = (1.0 - sigma) * clean + sigma * noise+ return noisy, noise, 1.0 - sigma, audio_t++ def score_forward(+ self,+ model: Any,+ inputs: H3DMDInputs,+ noisy: Any,+ video_t: Any,+ audio_t: Any,+ audio_noise: Any = None,+ ) -> Any:+ """A score model's data-ward velocity at ``noisy`` (the H3 sign: clean - noise)."""++ if audio_noise is None:+ audio_noise = self._random(tuple(inputs.audio_clean.shape), None)+ return self._video_forward(+ model, inputs, noisy, video_t.reshape(()), audio_t.reshape(()), audio_noise+ )++ def anchor_loss(+ self,+ inputs: H3DMDInputs,+ *,+ model: Any,+ forwards: int,+ index: int,+ generator: Any = None,+ ) -> tuple[Any, float]:+ """The real-data anchor on the few-step path: the sample's real target noised to the student's+ level ``index`` of its ``forwards``-evaluation grid, the student's clean estimate from there, and its squared+ error to the real target, weighted by (1 - sigma) - nearly nothing at the noisiest level, where the only+ regression answer is a blurred mean. Returns (loss, sigma). The DMD student sees no real data otherwise."""++ import torch++ video = make_shifted_schedule(int(forwards) + 1, shift=12.0)+ audio = make_shifted_schedule(int(forwards) + 1, shift=3.0)+ clean = inputs.prepared.clean.float()+ sigma = float(video.sigmas[int(index)])+ video_t = torch.tensor(float(video.timesteps[int(index)]), device=self.device)+ audio_t = torch.tensor(float(audio.timesteps[int(index)]), device=self.device)+ noise = self._random(tuple(clean.shape), generator)+ audio_noise = self._random(tuple(inputs.audio_clean.shape), generator)+ noisy = (1.0 - sigma) * clean + sigma * noise+ velocity = self._video_forward(model, inputs, noisy, video_t, audio_t, audio_noise)+ estimate = noisy + sigma * velocity+ return (1.0 - sigma) * (estimate - clean).square().mean(), sigma++ def structure_free(self, inputs: H3DMDInputs, batch: Any) -> H3DMDInputs:+ """The sample's conditions with its structure video taken out - every other observation (the first frame) and+ the prompt kept, observations its plan dropped still dropped: training's own observation dropout, so the+ teacher has met these conditions. A run reading the structure through a control trunk turns the trunk off."""++ import dataclasses++ p = inputs.prepared+ if self.reads_observation:+ geometry = None if p.geometry is None else dataclasses.replace(p.geometry, gain=0.0)+ return dataclasses.replace(inputs, prepared=dataclasses.replace(p, geometry=geometry))+ items = tuple(batch.visual_observations or ())+ planned = None if p.plan is None else p.plan.keeps_observation++ def keep(index: int) -> bool:+ return (planned is None or planned(index)) and not _is_structure(batch, items[index])++ use_anchor = batch.anchor_latents is not None and (p.plan is None or p.plan.use_anchor)+ _prompt, tags = self.prompt(batch, use_anchor=use_anchor)+ specs, degrees, rows = self.observation_rows(batch, None, keep)+ layout = self.layout(tags, anchors=("first",) if use_anchor else (), observations=specs,+ grid=inputs.grid, frames=inputs.frames)+ parts = list(p.condition_rows[:1]) if use_anchor else []+ if rows is not None:+ parts.append(rows)+ return dataclasses.replace(+ inputs, prepared=dataclasses.replace(p, condition_rows=parts, layout=layout, observation_degrees=degrees)+ )++ def teacher_answer(+ self,+ teacher: Any,+ inputs: H3DMDInputs,+ batch: Any,+ noisy: Any,+ video_t: Any,+ audio_t: Any,+ audio_noise: Any,+ *,+ guidance: float = 1.0,+ sigma_min: float = 0.0,+ quality: float = 0.0,+ quality_sigma_max: float = 1.0,+ negative: Any = None,+ ) -> Any:+ """The teacher's clean answer at ``noisy`` - guided along the structure when ``guidance`` > 1 and the noise+ level is at least ``sigma_min`` (the structure is laid down at high noise). The structure-free+ pass runs whenever ``guidance`` > 1, below the cutoff too: the level is drawn per sequence-parallel group, and+ a group that skipped a forward of the sharded teacher would leave the others waiting in a collective.++ ``quality`` > 0 adds ``quality * (real - low)``, ``low`` the teacher's answer under the fixed+ low-quality prompt ``negative`` (embeds, tags), where the level is below ``quality_sigma_max`` - the details+ are laid down at low noise. That pass too runs at every level."""++ if float(quality) > 0.0 and negative is None:+ raise ValueError("quality guidance needs the low-quality prompt")+ sigma = 1.0 - video_t+ real = noisy + sigma * self.score_forward(teacher, inputs, noisy, video_t, audio_t, audio_noise)+ answer = real+ if float(guidance) != 1.0:+ free_inputs = self.structure_free(inputs, batch)+ free = noisy + sigma * self.score_forward(teacher, free_inputs, noisy, video_t, audio_t, audio_noise)+ answer = structure_guided(real, free, guidance if float(sigma) >= float(sigma_min) else 1.0)+ if float(quality) > 0.0:+ low_inputs = self.low_quality(inputs, batch, negative)+ low = noisy + sigma * self.score_forward(teacher, low_inputs, noisy, video_t, audio_t, audio_noise)+ if float(sigma) < float(quality_sigma_max):+ answer = answer + float(quality) * (real - low)+ return answer++ def low_quality(self, inputs: H3DMDInputs, batch: Any, negative: Any) -> H3DMDInputs:+ """The sample's own conditions under the fixed low-quality prompt ``negative`` (embeds [N,5120], tags [N]):+ the observations and their rows exactly as the sample has them, the layout rebuilt for the other prompt's+ tokens. The H3 ref2va prompt is text alone, so one encoding serves every sample."""++ import dataclasses++ import torch++ from .stage0p5 import trunk_keep++ embeds, tags = negative+ embeds = embeds.to(self.device, dtype=torch.bfloat16).unsqueeze(0)+ tags = tags.to(self.device, dtype=torch.long)+ self._broadcast_logical_sample(tags, embeds)+ p = inputs.prepared+ use_anchor = batch.anchor_latents is not None and (p.plan is None or p.plan.use_anchor)+ keep = None if p.plan is None else p.plan.keeps_observation+ if self.reads_observation:+ keep = trunk_keep(batch)+ specs, _degrees, _rows = self.observation_rows(batch, None, keep)+ layout = self.layout(tags, anchors=("first",) if use_anchor else (), observations=specs,+ grid=inputs.grid, frames=inputs.frames)+ viewmats, K, frame_ids = p.viewmats, p.K, p.frame_ids+ if frame_ids is not None:+ viewmats, K, frame_ids = self._camera(batch, layout)+ self._broadcast_logical_sample(viewmats, K)+ return dataclasses.replace(+ inputs, prepared=dataclasses.replace(p, prompt=embeds, layout=layout, viewmats=viewmats, K=K,+ frame_ids=frame_ids)+ )++ def flow_matching_loss(self, inputs: H3DMDInputs, model: Any) -> Any:+ """The ordinary omni loss on the sample's own data, on ``model`` (self-distillation)."""++ return self.loss_from_inputs(inputs.prepared, model=model)+++__all__ = ["H3DMDInputs", "H3OmniDMDCore", "anchor_condition_weight", "structure_guided"]diff --git a/src/solarwm/backends/minimax_h3/omni_dmd_runtime.py b/src/solarwm/backends/minimax_h3/omni_dmd_runtime.pynew file mode 100644index 0000000..8016e4b--- /dev/null+++ b/src/solarwm/backends/minimax_h3/omni_dmd_runtime.py@@ -0,0 +1,701 @@+"""Structure-guided DMD2 from a released four-step LoRA (DMAD in the release): student, teacher and critic on+three model copies. ``train.dmd.init_lora`` names the starting LoRA (the older name ``plug`` is still read).++``train.dmd.mode``:++``adapt_plug`` the released recipe. The control LoRA is frozen and merged into the student's base; the+ student's own adapter starts as the released four-step LoRA (seeded into the leading+ rank) and is distilled towards the teacher, base + control at many steps.+``self_distill`` an ablation: the released four-step LoRA is frozen and merged into the student's base; the+ student's own adapter is the control LoRA. The teacher is the starting control LoRA+ without the four-step LoRA, fixed, and the control LoRA also takes the ordinary+ flow-matching loss on the sample's data through the teacher copy (lent the student's+ control for that pass, then given back its own).+``joint`` an ablation: nothing is merged; the student's own adapter is the control LoRA and the+ four-step LoRA stacked into one rank-512 LoRA (control in slots 0-383, four-step LoRA in+ 384-511: their sum, unchanged), and both train: the DMD gradient reaches all of it, the+ flow-matching gradient (through the teacher copy, lent the control slots) only the+ control slots; the teacher's own answer stays the starting control. Without a control+ checkpoint the control slots start fresh (a zero update): joint training from the base.++The critic (fake score) starts from the teacher. Copies share the sample's conditions and every+random draw (the exit evaluation is drawn once for all ranks: FSDP needs the same forwards on each).+``train.dmd.teacher`` "follow" restores the first design, a teacher kept equal to the student's control+after every update (it fed the student's drift back as the answer; see _teacher_follows).+"""++from __future__ import annotations++import time+from dataclasses import replace+from pathlib import Path+from typing import Any++from solarwm.training import JsonlEventSink, StepPolicy+from solarwm.training.sgf import (+ compute_sgf_kl_gradient,+ project_out_residual,+ sgf_student_loss,+ should_update_student,+)++from .config import dmd_settings+from .distributed import get_sp_size, sync_lora_gradients+from .runtime import H3TrainingRuntime, _base_model_load_receipt+from .sgf import h3_sgf_critic_loss++MODES = ("adapt_plug", "self_distill", "joint", "causal")+TEACHED = ("self_distill", "joint") # modes whose teacher follows the student's control LoRA+++def decoupled_causal(dmd: Any) -> bool:+ """The block-causal recipe: the causal control LoRA (``student_init``) merged into the student's base and frozen,+ the student's own adapter a separate acceleration LoRA that starts as the released four-step LoRA of+ ``accel_init``; the teacher and critic keep the bidirectional control LoRA (``control``)."""++ return str(dmd.get("mode")) == "causal" and bool(dmd.get("decoupled", False))+++def seeds_released_plug(dmd: Any) -> bool:+ """Whether an adapt_plug student starts as the released four-step LoRA (the default) or from a zero update+ (``seed_init_lora: false``)."""++ return bool(dmd.get("seed_plug", True))+++class H3OmniDMDRuntime(H3TrainingRuntime):+ def __init__(self, config: Any) -> None:+ dmd = dmd_settings(config["train"]["dmd"])+ self.mode = str(dmd["mode"])+ if self.mode not in MODES:+ raise ValueError(f"train.dmd.mode must be one of {MODES}, got {self.mode!r}")+ self.dmd = dmd+ super().__init__(config, defer_resume=True)+ # No student EMA (LongLive-Plug's DMD runs without one): the adapter is replicated on every+ # rank, so its FP32 shadow would hold 8 GB a card that the three copies need.+ self.ema = None+ torch = self.torch+ torch.cuda.empty_cache()+ from .lora import released_lora_state, seed_lora_from_released++ if decoupled_causal(dmd):+ if dmd.get("accel_init"):+ # The acceleration LoRA starts as the released four-step LoRA, its update unchanged (the scale its+ # header names; alpha = rank when it names none).+ from safetensors import safe_open++ with safe_open(str(dmd["accel_init"]), framework="pt") as handle:+ header = handle.metadata() or {}+ scale = float(header["alpha"]) / float(header["rank"]) if "alpha" in header and "rank" in header else 1.0+ seeded = seed_lora_from_released(+ self.lora.parameter_by_key,+ released_lora_state(str(dmd["accel_init"])),+ live_scale=self.lora.alpha / self.lora.rank,+ released_scale=scale,+ )+ if self.is_main:+ print(f"[h3-dmd-init] causal acceleration LoRA from {dmd['accel_init']} in {seeded} modules over "+ "the frozen causal control", flush=True)+ elif self.is_main:+ print("[h3-dmd-init] causal acceleration LoRA from a zero update over the frozen causal control",+ flush=True)+ elif self.mode == "causal":+ raise ValueError("a causal DMD student is a separate acceleration LoRA: set train.dmd.decoupled")+ elif self.mode == "adapt_plug" and not seeds_released_plug(dmd):+ if self.is_main:+ print("[h3-dmd-init] student starts from a zero update (seed_init_lora false), not from the released LoRA", flush=True)+ elif self.mode == "adapt_plug":+ # The student's adapter starts as the released four-step LoRA: its update unchanged, spare rank fresh.+ seeded = seed_lora_from_released(+ self.lora.parameter_by_key,+ released_lora_state(str(dmd["plug"])),+ live_scale=self.lora.alpha / self.lora.rank,+ released_scale=float(dmd.get("plug_alpha", 128)) / float(dmd.get("plug_rank", 128)),+ )+ if self.is_main:+ print(f"[h3-dmd-init] student seeded from the released four-step LoRA in {seeded} modules", flush=True)+ elif self.mode == "joint":+ from .lora import adapter_state_as_released++ width = int(self.model_cfg["adapter"]["rank"])+ live = self.lora.alpha / self.lora.rank+ if dmd.get("control"):+ seed_lora_from_released(+ self.lora.parameter_by_key,+ adapter_state_as_released(self._control_state()),+ live_scale=live,+ released_scale=float(self.model_cfg["adapter"]["alpha"]) / width,+ )+ seeded = seed_lora_from_released(+ self.lora.parameter_by_key,+ released_lora_state(str(dmd["plug"])),+ live_scale=live,+ released_scale=float(dmd.get("plug_alpha", 128)) / float(dmd.get("plug_rank", 128)),+ offset=width,+ )+ if self.is_main:+ print(+ f"[h3-dmd-init] student stacked from {'control' if dmd.get('control') else 'fresh control'}"+ f" + the four-step LoRA in {seeded} modules",+ flush=True,+ )+ cpu_rng, cuda_rng = torch.get_rng_state(), torch.cuda.get_rng_state(self.device)+ try:+ self.teacher, self.teacher_lora = self._build_score_role(trainable=False)+ critic, critic_lora = self._build_score_role(trainable=True)+ finally:+ torch.set_rng_state(cpu_rng)+ torch.cuda.set_rng_state(cuda_rng, self.device)+ from .optimizer import build_adapter_optimizer++ optimizer = build_adapter_optimizer(critic_lora, self.train_cfg["critic_optimizer"], None)+ scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda _: 1.0)+ self.extra_roles["critic"] = {+ "model": critic,+ "lora": critic_lora,+ "optimizer": optimizer,+ "scheduler": scheduler,+ }+ self.contract = replace(+ self.contract,+ extras={+ **self.contract.extras,+ "dmd": {+ "mode": self.mode,+ "plug": str(dmd.get("plug") or ""),+ "seed_plug": seeds_released_plug(dmd),+ "control": str(dmd.get("control") or ""),+ "forwards": int(dmd.get("forwards", 4)),+ },+ },+ )+ # The teacher as built holds the starting control: the fixed DMD answer (dmd.teacher "fixed",+ # the default) - kept on the CPU so flow matching can borrow the teacher copy and give it back.+ self._remember_fixed_teacher()+ resume = str(self.checkpoint_cfg.get("resume_from") or "")+ if resume:+ self.load_checkpoint(resume)+ if self.mode in TEACHED and self._teacher_follows():+ self._sync_teacher()++ # --- construction -------------------------------------------------------------------------++ def _control_state(self) -> dict[str, Any]:+ if not self.dmd.get("control"):+ # A joint run from the base: the control LoRA is the student's own fresh control+ # slots (a zero update), which the teacher then follows and the critic starts from.+ return {+ key: self._control_slots(key).detach().cpu().clone()+ for key in self.lora.parameter_by_key+ }+ root = Path(str(self.dmd["control"]))+ adapter = self.torch.load(root / "adapter.pt", map_location="cpu", weights_only=True)+ return adapter["state"]++ def _student_adapter(self) -> tuple[Any, tuple[int, ...]]:+ if self.mode != "joint":+ return super()._student_adapter()+ # Control (rank 384, alpha 384) and the four-step LoRA (rank 128, alpha 128) both update at scale 1,+ # so one rank-512 LoRA at alpha 512 holds their sum exactly.+ width = int(self.model_cfg["adapter"]["rank"]) + int(self.dmd.get("plug_rank", 128))+ return {"rank": width, "alpha": width, "dropout": 0.0}, (width,)++ def _before_adapter(self, transformer: Any) -> None:+ from .lora import adapter_state_as_released, merge_lora_delta, released_lora_state++ if self.mode == "joint":+ return # control and four-step LoRA both live in the student's own (trained) adapter+ if self.mode == "adapt_plug" or decoupled_causal(self.dmd):+ # The trained control LoRA becomes part of the student's base, frozen: for a causal student its own causal+ # control LoRA (student_init), not the teacher's.+ control = (+ self.torch.load(Path(str(self.dmd["student_init"])) / "adapter.pt", map_location="cpu",+ weights_only=True)["state"]+ if decoupled_causal(self.dmd) else self._control_state()+ )+ merged = merge_lora_delta(+ transformer,+ adapter_state_as_released(control),+ alpha=float(self.model_cfg["adapter"]["alpha"]),+ rank=int(self.model_cfg["adapter"]["rank"]),+ )+ else:+ # The four-step LoRA becomes part of the student's base, frozen, as inference merges it.+ merged = merge_lora_delta(+ transformer,+ released_lora_state(str(self.dmd["plug"])),+ alpha=float(self.dmd.get("plug_alpha", 128)),+ rank=int(self.dmd.get("plug_rank", 128)),+ )+ if self.rank == 0:+ print(+ f"[h3-dmd-init] {self.mode}: merged a LoRA into {merged} student weights",+ flush=True,+ )++ def _make_core(self) -> Any:+ from .observations import observation_profile_from_train_config+ from .omni_dmd import H3OmniDMDCore++ if self.mode == "causal":+ from .omni_causal import H3OmniCausalDMDCore++ return H3OmniCausalDMDCore(+ self.model,+ self.silence,+ self.device,+ self.dmd,+ dict(self.dmd["causal_stream"]),+ geometry=self.geometry,+ observations=observation_profile_from_train_config(self.train_cfg),+ )++ return H3OmniDMDCore(+ self.model,+ self.silence,+ self.device,+ self.dmd,+ geometry=self.geometry,+ observations=observation_profile_from_train_config(self.train_cfg),+ )++ def _build_score_role(self, *, trainable: bool) -> tuple[Any, Any]:+ """Base + control LoRA, no four-step LoRA.++ The teacher carries the control LoRA itself (a self-distillation keeps it equal to the+ student's and passes the flow-matching gradient through it, so it keeps a graph). The+ critic starts as the teacher at a fraction of its memory: the control LoRA merged into+ its base and a fresh rank-``critic_rank`` adapter (zero update) on top, as LongLive-Plug+ trains its fake score at rank 128.+ """++ from .fsdp import wrap_h3_fsdp+ from .lora import adapter_state_as_released, inject_h3_lora, merge_lora_delta+ from .optional import load_transformer++ seed = 0x48330000 + int(self.config["data"]["seed"])+ self.torch.manual_seed(seed)+ self.torch.cuda.manual_seed_all(seed)+ modules = load_transformer(self.model_cfg, device=self.device)+ modules.transformer.requires_grad_(False)+ graph = trainable or self.mode in TEACHED+ control = (+ {key: value.detach().cpu() for key, value in self.lora.state_dict().items()}+ if self.mode == "self_distill"+ else self._control_state()+ )+ adapter_cfg = self.model_cfg["adapter"]+ ranks: tuple[int, ...] = (384,)+ if trainable:+ merge_lora_delta(+ modules.transformer,+ adapter_state_as_released(control),+ alpha=float(adapter_cfg["alpha"]),+ rank=int(adapter_cfg["rank"]),+ )+ rank = int(self.dmd.get("critic_rank", 128))+ adapter_cfg = {"rank": rank, "alpha": rank, "dropout": 0.0}+ ranks = (rank,)+ model, lora = inject_h3_lora(+ modules.transformer,+ adapter_cfg,+ base_identity=_base_model_load_receipt(self.model_cfg, modules.transformer),+ ranks=ranks,+ )+ model = wrap_h3_fsdp(+ model,+ local_rank=self.topology.local_rank,+ transformer_block_cls=modules.transformer_block_cls,+ fp32_units=modules.fp32_fsdp_units,+ ignored_parameters=lora.parameters,+ activation_checkpointing=graph,+ frozen_base_shard_size=self.train_cfg["fsdp"].get("frozen_base_shard_size"),+ saved_attention_blocks=int(self.train_cfg["fsdp"].get("saved_attention_blocks", 0)),+ )+ if not trainable:+ lora.load_state_dict(control, broadcast=False)+ if not graph:+ model.requires_grad_(False)+ return model.eval(), lora++ def _control_slots(self, key: str) -> Any:+ """The student's control LoRA under the teacher's key (joint: its first rank slots)."""++ live = self.lora.parameter_by_key[key]+ if self.mode != "joint":+ return live+ width = int(self.model_cfg["adapter"]["rank"])+ return live[:width] if key.endswith("lora_A.weight") else live[:, :width]++ def _sync_teacher(self) -> None:+ """The teacher is the student's control LoRA, without the four-step LoRA."""++ with self.torch.no_grad():+ for key, parameter in self.teacher_lora.parameter_by_key.items():+ parameter.copy_(self._control_slots(key))++ def _teacher_follows(self) -> bool:+ """dmd.teacher "follow": the teacher is the student's control after every update (the first+ design). "fixed": the teacher stays the starting+ control - a teacher that follows the student feeds the student's drift back as its own+ answer; self-distillation lost structure (0.644 engine-depth delta1 against 0.876) and+ joint swung between 0.61 and 0.87, while adapt_plug, whose teacher never moved, kept it."""++ return str(self.dmd.get("teacher", "fixed")) == "follow"++ def _remember_fixed_teacher(self) -> None:+ self.teacher_fixed = {+ key: parameter.detach().to("cpu", copy=True)+ for key, parameter in self.teacher_lora.parameter_by_key.items()+ }++ def _restore_fixed_teacher(self) -> None:+ """The teacher back to the starting control, after flow matching borrowed it."""++ with self.torch.no_grad():+ for key, parameter in self.teacher_lora.parameter_by_key.items():+ parameter.copy_(self.teacher_fixed[key].to(parameter.device))++ def _teacher_after_update(self) -> None:+ if self.mode in TEACHED and self._teacher_follows():+ self._sync_teacher()++ def _add_teacher_gradients(self) -> None:+ """Move the teacher's flow-matching gradient onto the student's control slots."""++ with self.torch.no_grad():+ for key, parameter in self.teacher_lora.parameter_by_key.items():+ extra = parameter.grad+ if extra is None:+ continue+ live = self.lora.parameter_by_key[key]+ if live.grad is None:+ live.grad = self.torch.zeros_like(live)+ if key.endswith("lora_A.weight"):+ live.grad[: extra.shape[0]].add_(extra)+ else:+ live.grad[:, : extra.shape[1]].add_(extra)+ parameter.grad = None++ # --- one outer step -----------------------------------------------------------------------++ def _shared_exit(self) -> int:+ """One exit evaluation for every rank: FSDP gathers per forward, so all ranks run alike."""++ torch = self.torch+ value = torch.randint(0, int(self.dmd.get("forwards", 4)), (), device=self.device)+ if self.dist.is_initialized():+ self.dist.broadcast(value, src=0)+ return int(value.item())++ def _shared_choice(self, count: int) -> int:+ value = self.torch.randint(0, int(count), (), device=self.device)+ if self.dist.is_initialized():+ self.dist.broadcast(value, src=0)+ return int(value.item())++ def _stream_student(self, batch: Any, *, graph: bool) -> Any:+ """The block-causal student's clip: streamed chunk by chunk through its key/value cache without a graph, every+ chunk exiting at one shared evaluation; with ``graph`` replayed as one teacher-forced document from the+ recorded inputs - the same numbers, now differentiable, the gradient reaching how each chunk was written as+ context for the chunks after it too (self-gradient forcing)."""++ from .distributed import get_dp_rank+ from .flow import make_shifted_schedule+ from .layout import unpatchify_video++ torch, core = self.torch, self.core+ forwards = int(self.dmd.get("forwards", 4))+ seed = self._shared_choice(2**30) + 7919 * int(get_dp_rank())+ schedule = make_shifted_schedule(forwards + 1, shift=12.0)+ exit_index = self._shared_exit()+ with torch.no_grad():+ clip, inputs, times, jitters = core.stream(+ batch, generator=torch.Generator(device=self.device).manual_seed(seed), schedule=schedule,+ exit_index=exit_index, model=self.model,+ )+ if not graph:+ return clip+ replay = torch.Generator(device=self.device).manual_seed(seed)+ document = core._document(batch, replay)+ rows = core.training_prediction(+ document, inputs, times, clip, generator=replay, jitter=jitters, model=self.model+ )+ frames, grid = document[4]+ velocity = unpatchify_video(rows, frames, *grid, channels=24).float()+ level = torch.ones((1, 1, frames, 1, 1), device=self.device)+ for (a, b), t in zip(document[3].frames, times, strict=True):+ level[:, :, a:b] = 1.0 - float(t)+ return inputs + level * velocity++ def _finish_update(self, model: Any, lora: Any, optimizer: Any, scheduler: Any) -> float:+ sync_lora_gradients(lora.trainable_parameters)+ finite, norm = self._finite_clip_norm(+ (parameter for parameter in model.parameters() if parameter.requires_grad),+ float(self.train_cfg["optimizer"].get("gradient_clip", 10.0)),+ )+ if not finite:+ raise FloatingPointError(f"non-finite H3 DMD gradient norm={norm}")+ optimizer.step()+ scheduler.step()+ optimizer.zero_grad(set_to_none=True)+ return norm++ def _phase(self, name: str) -> None:+ if self.is_main:+ print(+ f"[h3-dmd-phase] outer_step={self.global_step + 1} {name} "+ f"elapsed_s={time.monotonic() - self._step_started:.1f}",+ flush=True,+ )++ @property+ def structure_guidance(self) -> float:+ """dmd.structure_guidance: the teacher's answer pushed along the structure direction; 1 = plain.+ dmd.structure_guidance_warmup: over that many student steps w climbs linearly from 1 - a+ self-distilling student (the control LoRA itself) lost its structure within 50 steps at w = 2 from the first."""++ guidance = float(self.dmd.get("structure_guidance", 1.0))+ warmup = int(self.dmd.get("structure_guidance_warmup", 0))+ if warmup <= 0:+ return guidance+ return 1.0 + (guidance - 1.0) * min(1.0, self.student_step / warmup)++ @property+ def structure_guidance_sigma_min(self) -> float:+ """dmd.structure_guidance_sigma_min: the guidance acts only at re-noise levels at least this."""++ return float(self.dmd.get("structure_guidance_sigma_min", 0.0))++ @property+ def quality_guidance(self) -> float:+ """dmd.quality_guidance: the teacher's answer pushed away from its answer under a fixed+ low-quality prompt; 0 = off."""++ return float(self.dmd.get("quality_guidance", 0.0))++ @property+ def quality_guidance_sigma_max(self) -> float:+ """dmd.quality_guidance_sigma_max: the quality guidance acts only below this re-noise level."""++ return float(self.dmd.get("quality_guidance_sigma_max", 1.0))++ @property+ def quality_negative(self) -> Any:+ """dmd.quality_negative: the low-quality prompt's (embeds, tags), read once from its safetensors."""++ path = self.dmd.get("quality_negative")+ if not path:+ return None+ cached = getattr(self, "_quality_negative", None)+ if cached is None:+ from safetensors import safe_open++ from .omni_data import _local++ with _local(str(path)) as local, safe_open(local, framework="pt") as handle:+ cached = (handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags"))+ self._quality_negative = cached+ return cached++ def negative_for(self, batch: Any) -> Any:+ """The low-quality prompt for ``batch``: the one file for every sample, or - when dmd.quality_negative names a+ directory - the sample's own ``<sample_id>.safetensors``. The ref2va prompt carries the sample's references,+ so its low-quality twin is encoded with them, per sample."""++ path = str(self.dmd.get("quality_negative") or "")+ if path.endswith(".safetensors"):+ return self.quality_negative+ from safetensors import safe_open++ from .omni_data import _local++ with _local(f"{path.rstrip('/')}/{batch.sample_id}.safetensors") as local, safe_open(local, framework="pt") as handle:+ return handle.get_tensor("prompt_embeds"), handle.get_tensor("text_token_tags")++ def _dmd_gradient(self, fake: Any, real: Any, output: Any) -> Any:+ """The DMD update on the student's endpoint. With ``dmd.pdmd`` the teacher-critic difference first loses its+ component along the critic-student endpoint residual (PDMD, 2609.35768: that residual estimates the critic's+ error), then is normalized as usual."""++ if bool(self.dmd.get("pdmd", False)):+ fake = real.float() + project_out_residual(fake - real, fake - output)+ return compute_sgf_kl_gradient(fake_x0=fake, real_x0=real, student_output=output)++ def _anchor(self, stats: dict[str, float], batch: Any, inputs: Any, forwards: int) -> None:+ """The real-data anchor on the few-step path (control + four-step LoRA): the sample's real target noised to one of the+ student's levels and regressed to, weighted by what the sample was given. Every rank runs the+ forward whatever its sample's weight: the model is sharded across ranks, and a rank that skipped it while+ another ran it left the two in different collectives (the 1-node smoke test hung)."""++ from .omni_dmd import anchor_condition_weight++ given = anchor_condition_weight(batch, dict(self.dmd.get("anchor_condition_weights", ANCHOR_WEIGHTS)))+ self._phase("anchor")+ term, sigma = self.core.anchor_loss(+ inputs, model=self.model, forwards=forwards, index=self._shared_exit()+ )+ if not bool(self.torch.isfinite(term)):+ raise FloatingPointError("non-finite H3 DMD anchor loss")+ (float(self.dmd["anchor_weight"]) * given * term / get_sp_size()).backward()+ stats["loss_anchor"] = float(term.detach())+ stats["anchor_sigma"] = float(sigma)+ stats["anchor_given"] = float(given)++ def _student_update(self, stats: dict[str, float]) -> None:+ torch, core = self.torch, self.core+ critic = self.extra_roles["critic"]+ forwards = int(self.dmd.get("forwards", 4))+ batch = self._next_batch()+ inputs = core.prepare_inputs(batch)+ self._phase("student_rollout")+ if self.mode == "causal":+ output = self._stream_student(batch, graph=True)+ else:+ output = core.rollout(+ inputs, model=self.model, forwards=forwards, exit_index=self._shared_exit()+ )+ with torch.no_grad():+ self._phase("student_scores")+ noisy, _noise, video_t, audio_t = core.score_inputs(output.detach())+ audio_noise = core._random(tuple(inputs.audio_clean.shape), None)+ sigma = 1.0 - video_t+ fake = noisy + sigma * core.score_forward(+ critic["model"], inputs, noisy, video_t, audio_t, audio_noise+ )+ real = core.teacher_answer(+ self.teacher, inputs, batch, noisy, video_t, audio_t, audio_noise,+ guidance=self.structure_guidance, sigma_min=self.structure_guidance_sigma_min,+ quality=self.quality_guidance, quality_sigma_max=self.quality_guidance_sigma_max,+ negative=self.negative_for(batch) if self.quality_guidance > 0 else None,+ )+ if not bool(torch.isfinite(fake).all() and torch.isfinite(real).all()):+ raise FloatingPointError("non-finite H3 DMD score output")+ gradient = self._dmd_gradient(fake, real, output.detach())+ loss = sgf_student_loss(output, gradient)+ self._phase("student_backward")+ (loss / get_sp_size()).backward()+ stats["loss_student"] = float(loss.detach())+ del output, noisy, fake, real, gradient, loss+ if float(self.dmd.get("anchor_weight", 0.0)) > 0:+ self._anchor(stats, batch, inputs, forwards)+ weight = float(self.dmd.get("flow_matching_weight", 1.0))+ if self.mode in TEACHED and weight > 0:+ # The ordinary loss on the sample's own data, through the teacher copy (control+ # LoRA, no four-step LoRA); its gradient joins the student's control LoRA gradient.+ self._phase("flow_matching")+ if not self._teacher_follows():+ self._sync_teacher() # borrow the teacher copy: the student's control, no four-step LoRA+ fm = core.flow_matching_loss(inputs, self.teacher)+ if not bool(torch.isfinite(fm)):+ raise FloatingPointError("non-finite H3 DMD flow-matching loss")+ (weight * fm / get_sp_size()).backward()+ self._add_teacher_gradients()+ if not self._teacher_follows():+ self._restore_fixed_teacher()+ stats["loss_flow_matching"] = float(fm.detach())+ stats["student_grad_norm"] = self._finish_update(+ self.model, self.lora, self.optimizer, self.scheduler+ )+ self._teacher_after_update()+ self.student_step += 1+ if self.ema is not None:+ self.ema_update(self.student_step)++ def _critic_update(self, stats: dict[str, float]) -> None:+ torch, core = self.torch, self.core+ critic = self.extra_roles["critic"]+ batch = self._next_batch()+ inputs = core.prepare_inputs(batch)+ self._phase("critic_rollout")+ with torch.no_grad():+ if self.mode == "causal":+ clean = self._stream_student(batch, graph=False)+ else:+ clean = core.rollout(+ inputs,+ model=self.model,+ forwards=int(self.dmd.get("forwards", 4)),+ exit_index=self._shared_exit(),+ )+ noisy, noise, video_t, audio_t = core.score_inputs(clean)+ self._phase("critic_forward")+ velocity = core.score_forward(critic["model"], inputs, noisy, video_t, audio_t)+ loss = h3_sgf_critic_loss(velocity, noise=noise, clean=clean)+ if not bool(torch.isfinite(loss)):+ raise FloatingPointError("non-finite H3 DMD critic loss")+ self._phase("critic_backward")+ (loss / get_sp_size()).backward()+ stats["loss_critic"] = float(loss.detach())+ stats["critic_grad_norm"] = self._finish_update(+ critic["model"], critic["lora"], critic["optimizer"], critic["scheduler"]+ )++ def train_outer_step(self) -> dict[str, float]:+ self._step_started = time.monotonic()+ ratio = int(self.dmd.get("critic_updates_per_student", 5))+ update_student = should_update_student(self.global_step, ratio)+ stats: dict[str, float] = {"student_updated": float(update_student), "loss_student": 0.0}+ if update_student:+ self._student_update(stats)+ self._critic_update(stats)+ self._global_step += 1+ stats.update(+ student_step=float(self.student_step),+ loss=stats["loss_student"] if update_student else stats["loss_critic"],+ lr=float(self.optimizer.param_groups[0]["lr"]),+ critic_lr=float(self.extra_roles["critic"]["optimizer"].param_groups[0]["lr"]),+ compute_time_s=time.monotonic() - self._step_started,+ )+ if self.is_main:+ extra = (+ f" fm_loss={stats['loss_flow_matching']:.6f}"+ if "loss_flow_matching" in stats+ else ""+ ) + (+ f" anchor_loss={stats['loss_anchor']:.6f} anchor_sigma={stats['anchor_sigma']:.3f}"+ if "loss_anchor" in stats+ else ""+ )+ print(+ f"[h3-dmd] step={self.global_step} student_step={self.student_step} "+ f"student_loss={stats['loss_student']:.6f} critic_loss={stats['loss_critic']:.6f}"+ f"{extra} compute_time_s={stats['compute_time_s']:.2f}",+ flush=True,+ )+ return stats+++# How much the real-data anchor counts by what a sample was given: structure + first frame fully, structure alone a+# little (its appearance is open), first frame or text alone not at all (many right answers; regressing blurs).+ANCHOR_WEIGHTS = {"full": 1.0, "structure": 0.3, "first": 0.0, "text": 0.0}+++def run_omni_dmd_training(config: Any) -> int:+ runtime = H3OmniDMDRuntime(config)+ policy = StepPolicy(+ max_steps=int(config["train"]["max_steps"]),+ save_every=int(config["checkpoint"]["save_every_steps"]),+ save_steps=tuple(config["checkpoint"].get("save_steps", ())),+ validate_every=int(config["validation"]["validate_every_steps"]),+ validation_steps=(),+ )+ sink = (+ JsonlEventSink(Path(str(config["runtime"]["output_dir"])) / "training-events.jsonl")+ if runtime.is_main+ else None+ )+ try:+ while runtime.global_step < policy.max_steps:+ stats = runtime.train_outer_step()+ step = runtime.global_step+ if sink is not None:+ sink({"event": "train_step", "step": step, **stats})+ if policy.should_save(step):+ runtime.save_checkpoint(step)+ finally:+ runtime.reader.close()+ return 0+++__all__ = ["H3OmniDMDRuntime", "run_omni_dmd_training"]diff --git a/src/solarwm/backends/minimax_h3/optimizer.py b/src/solarwm/backends/minimax_h3/optimizer.pynew file mode 100644index 0000000..87005bb--- /dev/null+++ b/src/solarwm/backends/minimax_h3/optimizer.py@@ -0,0 +1,228 @@+"""FP32-master AdamW for BF16 H3 LoRA parameters."""++from __future__ import annotations++import copy+import math+from collections import defaultdict+from collections.abc import Iterable, Mapping+from typing import Any++import torch++FP32_MASTER_ADAMW_SCHEMA = "solarwm.fp32-master-adamw.v1"+_SLOTS = frozenset({"step", "master_param", "exp_avg", "exp_avg_sq"})+++class FP32MasterAdamW(torch.optim.Optimizer):+ """AdamW keyed by live model params with FP32 master weights and moments."""++ def __init__(+ self,+ params: Iterable[torch.nn.Parameter] | Iterable[dict[str, Any]],+ lr: float = 1e-3,+ betas: tuple[float, float] = (0.9, 0.999),+ eps: float = 1e-8,+ weight_decay: float = 1e-2,+ ) -> None:+ self._validate_hparams(lr, betas, eps, weight_decay)+ super().__init__(+ params,+ {+ "lr": float(lr),+ "betas": tuple(float(value) for value in betas),+ "eps": float(eps),+ "weight_decay": float(weight_decay),+ "optimizer_state_schema": FP32_MASTER_ADAMW_SCHEMA,+ },+ )+ for group in self.param_groups:+ self._validate_group(group)++ @staticmethod+ def _validate_hparams(+ lr: float, betas: tuple[float, float], eps: float, weight_decay: float+ ) -> None:+ if not math.isfinite(float(lr)) or float(lr) < 0:+ raise ValueError("learning rate must be finite and non-negative")+ if len(betas) != 2 or any(not 0 <= float(value) < 1 for value in betas):+ raise ValueError("AdamW betas must be in [0,1)")+ if not math.isfinite(float(eps)) or float(eps) < 0:+ raise ValueError("epsilon must be finite and non-negative")+ if not math.isfinite(float(weight_decay)) or float(weight_decay) < 0:+ raise ValueError("weight decay must be finite and non-negative")++ @classmethod+ def _validate_group(cls, group: Mapping[str, Any]) -> None:+ if group.get("optimizer_state_schema") != FP32_MASTER_ADAMW_SCHEMA:+ raise ValueError("unsupported FP32-master optimizer state schema")+ cls._validate_hparams(+ group["lr"], tuple(group["betas"]), group["eps"], group["weight_decay"]+ )++ @staticmethod+ def _new_state(parameter: torch.nn.Parameter) -> dict[str, torch.Tensor]:+ if not parameter.is_floating_point():+ raise TypeError("FP32MasterAdamW supports floating parameters only")+ return {+ "step": torch.zeros((), dtype=torch.float32, device="cpu"),+ "master_param": parameter.detach().to(dtype=torch.float32, copy=True),+ "exp_avg": torch.zeros_like(parameter, dtype=torch.float32),+ "exp_avg_sq": torch.zeros_like(parameter, dtype=torch.float32),+ }++ @staticmethod+ def _validate_state(parameter: torch.nn.Parameter, values: Mapping[str, Any]) -> None:+ if set(values) != _SLOTS:+ raise RuntimeError("FP32MasterAdamW state slots differ")+ step = values["step"]+ if (+ not isinstance(step, torch.Tensor)+ or step.numel() != 1+ or step.device.type != "cpu"+ or step.dtype != torch.float32+ ):+ raise RuntimeError("optimizer step must be a CPU FP32 scalar")+ for name in ("master_param", "exp_avg", "exp_avg_sq"):+ value = values[name]+ if (+ not isinstance(value, torch.Tensor)+ or value.dtype != torch.float32+ or value.device != parameter.device+ or tuple(value.shape) != tuple(parameter.shape)+ ):+ raise RuntimeError(f"invalid FP32MasterAdamW slot {name!r}")++ @torch.no_grad()+ def step(self, closure: Any = None) -> Any:+ loss = None+ if closure is not None:+ with torch.enable_grad():+ loss = closure()+ for group in self.param_groups:+ self._validate_group(group)+ lr = float(group["lr"])+ beta1, beta2 = (float(value) for value in group["betas"])+ epsilon = float(group["eps"])+ decay = float(group["weight_decay"])+ for parameter in group["params"]:+ gradient = parameter.grad+ if gradient is None:+ continue+ if gradient.is_sparse:+ raise RuntimeError("FP32MasterAdamW does not support sparse gradients")+ values = self.state[parameter]+ if not values:+ values.update(self._new_state(parameter))+ self._validate_state(parameter, values)+ values["step"].add_(1)+ step = int(values["step"].item())+ master = values["master_param"]+ first = values["exp_avg"]+ second = values["exp_avg_sq"]+ gradient_fp32 = gradient.detach().float()+ if decay:+ master.mul_(1.0 - lr * decay)+ first.mul_(beta1).add_(gradient_fp32, alpha=1.0 - beta1)+ second.mul_(beta2).addcmul_(gradient_fp32, gradient_fp32, value=1.0 - beta2)+ correction1 = 1.0 - beta1**step+ correction2 = 1.0 - beta2**step+ denominator = second.sqrt().div_(math.sqrt(correction2)).add_(epsilon)+ master.addcdiv_(first, denominator, value=-(lr / correction1))+ parameter.copy_(master)+ return loss++ def load_state_dict(self, state_dict: Mapping[str, Any]) -> None:+ """Restore slots without PyTorch casting them to the live BF16 dtype."""++ raw_state = state_dict.get("state")+ raw_groups = state_dict.get("param_groups")+ if not isinstance(raw_state, Mapping) or not isinstance(raw_groups, (list, tuple)):+ raise ValueError("invalid optimizer state dictionary")+ if len(raw_groups) != len(self.param_groups):+ raise ValueError("optimizer group count differs")+ id_to_parameter: dict[Any, torch.nn.Parameter] = {}+ restored_groups: list[dict[str, Any]] = []+ for saved, current in zip(raw_groups, self.param_groups, strict=True):+ saved_ids = saved.get("params")+ if not isinstance(saved_ids, (list, tuple)):+ raise ValueError("optimizer group has no saved parameter IDs")+ parameters = list(current["params"])+ if len(saved_ids) != len(parameters):+ raise ValueError("optimizer parameter count differs")+ id_to_parameter.update(zip(saved_ids, parameters, strict=True))+ restored = copy.deepcopy(dict(saved))+ restored["params"] = parameters+ self._validate_group(restored)+ restored_groups.append(restored)+ if any(identifier not in id_to_parameter for identifier in raw_state):+ raise ValueError("optimizer state references an unknown parameter")+ restored_state: defaultdict[torch.nn.Parameter, dict[str, torch.Tensor]] = defaultdict(dict)+ for identifier, raw_slots in raw_state.items():+ if not raw_slots:+ continue+ if not isinstance(raw_slots, Mapping) or set(raw_slots) != _SLOTS:+ raise ValueError("optimizer checkpoint slots differ")+ parameter = id_to_parameter[identifier]+ raw_step = raw_slots["step"]+ step = float(+ raw_step.detach().cpu().item() if isinstance(raw_step, torch.Tensor) else raw_step+ )+ if not math.isfinite(step) or step < 0 or not step.is_integer():+ raise ValueError("optimizer checkpoint step is invalid")+ slots = {"step": torch.tensor(step, dtype=torch.float32, device="cpu")}+ for name in ("master_param", "exp_avg", "exp_avg_sq"):+ value = raw_slots[name]+ if not isinstance(value, torch.Tensor) or tuple(value.shape) != tuple(+ parameter.shape+ ):+ raise ValueError(f"optimizer checkpoint slot {name!r} differs")+ slots[name] = value.detach().to(+ device=parameter.device, dtype=torch.float32, copy=True+ )+ self._validate_state(parameter, slots)+ restored_state[parameter] = slots+ self.param_groups = restored_groups+ self.state = restored_state+++def build_adapter_optimizer(+ lora: Any,+ optimizer_cfg: Mapping[str, Any],+ geometry_cfg: Mapping[str, Any] | None = None,+ *,+ default_betas: tuple[float, float] = (0.9, 0.95),+ default_weight_decay: float = 0.01,+) -> FP32MasterAdamW:+ """One group for LoRA, and a second one for a geometry ControlNet.++ A run without the ControlNet builds exactly the historical single-group+ optimizer. The ControlNet trains from scratch, so it owns its learning rate+ and weight decay; frozen LoRA (``lora_trainable: false``) gets no group.+ """++ hyperparameters = {+ "lr": float(optimizer_cfg["learning_rate"]),+ "betas": tuple(float(value) for value in optimizer_cfg.get("betas", default_betas)),+ "eps": float(optimizer_cfg.get("epsilon", 1.0e-8)),+ "weight_decay": float(optimizer_cfg.get("weight_decay", default_weight_decay)),+ }+ if getattr(lora, "geometry_profile", None) is None:+ return FP32MasterAdamW(lora.parameters, **hyperparameters)+ if geometry_cfg is None:+ raise ValueError("a geometry ControlNet run requires its own optimizer settings")+ groups: list[dict[str, Any]] = []+ lora_parameters = [parameter for parameter in lora.lora_parameters if parameter.requires_grad]+ if lora_parameters:+ groups.append({"params": lora_parameters})+ groups.append(+ {+ "params": list(lora.geometry_parameters),+ "lr": float(geometry_cfg["learning_rate"]),+ "weight_decay": float(geometry_cfg["weight_decay"]),+ }+ )+ return FP32MasterAdamW(groups, **hyperparameters)+++__all__ = ["FP32_MASTER_ADAMW_SCHEMA", "FP32MasterAdamW", "build_adapter_optimizer"]diff --git a/src/solarwm/backends/minimax_h3/optional.py b/src/solarwm/backends/minimax_h3/optional.pynew file mode 100644index 0000000..337be14--- /dev/null+++ b/src/solarwm/backends/minimax_h3/optional.py@@ -0,0 +1,232 @@+"""Lazy loaders for the MiniMax-H3 optional runtime."""++from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass+from pathlib import Path+from typing import Any++from solarwm.errors import BackendContractError++H3_FP32_PARAMETER_NAMES = frozenset(+ f"{owner}.{slot}"+ for owner in (+ "proj_in",+ "audio_proj_in",+ "time_embedder.linear_1",+ "time_embedder.linear_2",+ "proj_out",+ "audio_proj_out",+ )+ for slot in ("weight", "bias")+)+++@dataclass+class H3RuntimeModules:+ transformer: Any = None+ transformer_block_cls: Any = None+ fp32_fsdp_units: tuple[Any, ...] = ()+ text_encoder: Any = None+ processor: Any = None+ tokenizer: Any = None+ video_vae: Any = None+ audio_vae: Any = None+ video_scheduler: Any = None+ audio_scheduler: Any = None+++def require_h3_runtime() -> tuple[Any, Any, Any]:+ """Resolve heavy packages with an image-specific actionable error."""++ try:+ import diffusers+ import torch+ import transformers+ except ImportError as exc: # pragma: no cover - optional local dependency+ raise BackendContractError(+ "MiniMax-H3 execution requires torch, an H3-compatible Diffusers "+ "build, Transformers, PEFT, "+ "FlashAttention, and safetensors"+ ) from exc+ if not torch.cuda.is_available():+ raise BackendContractError("MiniMax-H3 heavy execution requires CUDA")+ return torch, diffusers, transformers+++def torch_dtype(name: str) -> Any:+ torch, _diffusers, _transformers = require_h3_runtime()+ aliases = {+ "bfloat16": torch.bfloat16,+ "bf16": torch.bfloat16,+ "float16": torch.float16,+ "fp16": torch.float16,+ "float32": torch.float32,+ "fp32": torch.float32,+ }+ try:+ return aliases[str(name).lower()]+ except KeyError as exc:+ raise BackendContractError(f"unsupported H3 torch dtype {name!r}") from exc+++def validate_parameter_dtypes(model: Any) -> None:+ """Require the official BF16 block / six-FP32-owner split."""++ torch, _diffusers, _transformers = require_h3_runtime()+ parameters = dict(model.named_parameters())+ missing = sorted(H3_FP32_PARAMETER_NAMES - set(parameters))+ if missing:+ raise BackendContractError(f"H3 checkpoint lacks FP32 owners: {missing}")+ mismatches = []+ for name, parameter in parameters.items():+ expected = torch.float32 if name in H3_FP32_PARAMETER_NAMES else torch.bfloat16+ if parameter.dtype != expected:+ mismatches.append(f"{name}:{parameter.dtype}->{expected}")+ if len(mismatches) == 8:+ break+ if mismatches:+ raise BackendContractError(+ f"H3 checkpoint parameter dtype contract differs; first mismatches={mismatches}"+ )+++def _resolve_owner(model: Any, dotted: str) -> Any:+ current = model+ for part in dotted.split("."):+ current = getattr(current, part)+ return current+++def load_transformer(model_cfg: Mapping[str, Any], *, device: Any) -> H3RuntimeModules:+ """Strict-load the official 33B transformer through the camera adapter."""++ _torch, _diffusers, _transformers = require_h3_runtime()+ from .model import MiniMaxH3TransformerBlock, SolarMiniMaxH3Transformer3DModel++ path = str(model_cfg["checkpoint_path"])+ kwargs = {+ "subfolder": str(model_cfg.get("transformer_subfolder", "transformer")),+ "torch_dtype": torch_dtype(str(model_cfg.get("torch_dtype", "bfloat16"))),+ "device_map": model_cfg.get("transformer_device_map"),+ "low_cpu_mem_usage": bool(model_cfg.get("low_cpu_mem_usage", True)),+ "local_files_only": bool(model_cfg.get("local_files_only", False)),+ }+ if model_cfg.get("revision") is not None:+ kwargs["revision"] = model_cfg["revision"]+ transformer = SolarMiniMaxH3Transformer3DModel.strict_from_pretrained(path, **kwargs)+ validate_parameter_dtypes(transformer)+ # W6 uses our explicit compiled BlockMask path. Native unmasked attention+ # (including Qwen token refinement and SGF score models) remains Flash.+ requested_backend = str(model_cfg.get("attention_backend", "flash"))+ transformer.set_attention_backend("flash" if requested_backend == "flex" else requested_backend)+ if model_cfg.get("transformer_device_map") is None:+ transformer = transformer.to(device)+ transformer.camera_intrinsics_mode = str(model_cfg.get("camera_intrinsics_mode", "wan_fixed"))+ from .controlnet_contract import geometry_profile_from_model_config++ geometry = geometry_profile_from_model_config(model_cfg)+ if geometry is not None:+ # Built after the strict load and dtype audit, on the target device:+ # FSDP leaves the trunk's replicated (ignored) parameters where they are.+ transformer.enable_geometry_controlnet(geometry, device=device)+ fp32_units = tuple(+ _resolve_owner(transformer, owner)+ for owner in (+ "proj_in",+ "audio_proj_in",+ "time_embedder.linear_1",+ "time_embedder.linear_2",+ "proj_out",+ "audio_proj_out",+ )+ )+ return H3RuntimeModules(+ transformer=transformer,+ transformer_block_cls=MiniMaxH3TransformerBlock,+ fp32_fsdp_units=fp32_units,+ )+++def load_conditioners(+ model_cfg: Mapping[str, Any],+ *,+ device: Any,+ qwen: bool = True,+ video_vae: bool = True,+ audio_vae: bool = True,+ schedulers: bool = True,+) -> H3RuntimeModules:+ """Load official Qwen, VisualVAE, AudioVAE, and scheduler components."""++ torch, _diffusers, _transformers = require_h3_runtime()+ from diffusers import (+ AutoencoderKLMiniMaxH3,+ AutoencoderKLMiniMaxH3Audio,+ MiniMaxH3Scheduler,+ )+ from transformers import (+ Qwen2TokenizerFast,+ Qwen3VLForConditionalGeneration,+ Qwen3VLProcessor,+ )++ root = Path(str(model_cfg["checkpoint_path"]))+ dtype = torch_dtype(str(model_cfg.get("torch_dtype", "bfloat16")))+ result = H3RuntimeModules()+ if qwen:+ result.text_encoder = (+ Qwen3VLForConditionalGeneration.from_pretrained(+ str(root), subfolder="text_encoder", torch_dtype=dtype+ )+ .eval()+ .requires_grad_(False)+ .to(device)+ )+ result.tokenizer = Qwen2TokenizerFast.from_pretrained(str(root / "tokenizer"))+ result.processor = Qwen3VLProcessor.from_pretrained(str(root / "processor"))+ if video_vae:+ # The offline encoder uses the configured BF16 component,+ # while the official validation decoder is loaded in FP32 and runs+ # its convolutional kernels under FP16 autocast.+ video_dtype = torch.float32 if not qwen else dtype+ result.video_vae = (+ AutoencoderKLMiniMaxH3.from_pretrained(+ str(root), subfolder="vae", torch_dtype=video_dtype+ )+ .eval()+ .requires_grad_(False)+ .to(device)+ )+ if audio_vae:+ result.audio_vae = (+ AutoencoderKLMiniMaxH3Audio.from_pretrained(+ str(root), subfolder="audio_vae", torch_dtype=dtype+ )+ .eval()+ .requires_grad_(False)+ .to(device)+ )+ if schedulers:+ result.video_scheduler = MiniMaxH3Scheduler.from_pretrained(+ str(root), subfolder=str(model_cfg.get("scheduler_subfolder", "scheduler"))+ )+ # The model package may provide a distinct audio scheduler; fall+ # back to a separately configured copy with the same class/config.+ audio_subfolder = str(model_cfg.get("audio_scheduler_subfolder", "scheduler"))+ result.audio_scheduler = MiniMaxH3Scheduler.from_pretrained(+ str(root), subfolder=audio_subfolder+ )+ return result+++__all__ = [+ "H3_FP32_PARAMETER_NAMES",+ "H3RuntimeModules",+ "load_conditioners",+ "load_transformer",+ "require_h3_runtime",+ "torch_dtype",+ "validate_parameter_dtypes",+]diff --git a/src/solarwm/backends/minimax_h3/preencode_runner.py b/src/solarwm/backends/minimax_h3/preencode_runner.pynew file mode 100644index 0000000..e4c7cd5--- /dev/null+++ b/src/solarwm/backends/minimax_h3/preencode_runner.py@@ -0,0 +1,427 @@+"""Concrete raw-WDS to H3 preencoded-WDS production runner."""++from __future__ import annotations++import hashlib+import json+import os+import uuid+from collections.abc import Mapping, Sequence+from itertools import pairwise+from pathlib import Path+from typing import Any++from solarwm.config.loader import canonical_json+from solarwm.data import (+ CanonicalSampler,+ IndexRow,+ RawSampleReader,+ ReaderIdentity,+ SamplingConfig,+ TarShardReader,+ read_index,+ resolve_index_path,+ resolver_from_config,+)+from solarwm.errors import DataContractError+from solarwm.preencode import EncodedPayload, write_index, write_shard+from solarwm.runtime.create_only import publish_directory_no_replace+from solarwm.runtime.distributed import collective_call, collective_rank_zero_call++from .controlnet_contract import geometry_member_field, geometry_renders_from_preencode_config+from .official_codec import H3_VIDEO_VAE_SEED, OfficialH3Codec+from .optional import load_conditioners, require_h3_runtime+from .raw_data import (+ decode_camera,+ decode_geometry_render,+ decode_resize_video,+ normalize_raw_source_windows,+)++H3_ENCODER_CONTRACT_PATH = "encoder-contract.json"+H3_SILENCE_PATH = "silence/h3-silence-158f.safetensors"+H3_INDEX_PATH = "index.jsonl"+H3_COMPLETE_PATH = "COMPLETE.json"+++def _publish_staging(output_root: Path, requested_root: Path) -> None:+ publish_directory_no_replace(+ output_root,+ requested_root,+ error_type=DataContractError,+ label="H3 preencode output",+ )+++def _published_rows(receipts: Sequence[Any]) -> list[dict[str, Any]]:+ """Promote sampler controls required to replay every encoded window."""++ result: list[dict[str, Any]] = []+ for receipt in receipts:+ for source in receipt.rows:+ row = dict(source)+ provenance_metadata = dict(row.get("metadata", {}))+ member_digest = provenance_metadata.get("member_digest")+ if not isinstance(member_digest, Mapping):+ raise DataContractError("published H3 row lacks member content digest provenance")+ members = row.get("members")+ if not isinstance(members, Mapping):+ raise DataContractError("published H3 row lacks canonical member names")+ try:+ tensor_member = str(members["tensors.safetensors"])+ manifest_member = str(members["manifest.json"])+ tensor_digest = str(member_digest["tensors.safetensors"])+ manifest_digest = str(member_digest["manifest.json"])+ provenance_member = str(row["provenance_member"])+ except KeyError as exc:+ raise DataContractError("published H3 row has incomplete member identity") from exc+ provenance = {+ "schema": "solarwm.preencoded-sample.v1",+ "sample_id": row["sample_id"],+ "key": row["key"],+ "source_sample_id": row["source_sample_id"],+ "start_frame": row["start_frame"],+ "source_frame_indices": row["source_frame_indices"],+ "encoder_contract_digest": row["encoder_contract_digest"],+ "members": dict(members),+ "metadata": provenance_metadata,+ }+ source_indices = tuple(int(value) for value in row["source_frame_indices"])+ if len(source_indices) != 158 or any(+ right != left + 1 for left, right in pairwise(source_indices)+ ):+ raise DataContractError("published H3 source indices must be 158 contiguous rows")+ metadata = dict(provenance_metadata)+ minimum_frames = max(source_indices) + 1+ raw_source_frames = metadata.pop("source_num_frames", None)+ source_frames = minimum_frames if raw_source_frames is None else int(raw_source_frames)+ if source_frames < minimum_frames:+ raise DataContractError("published H3 source length is shorter than its window")+ row["num_frames"] = source_frames+ source_repeats = int(metadata.pop("source_epoch_repeats", 1))+ if source_repeats < 1:+ raise DataContractError("published H3 epoch_repeats must be positive")+ row["epoch_repeats"] = source_repeats+ if metadata.get("source_fps") is not None:+ row["fps"] = float(metadata["source_fps"])+ row["metadata"] = metadata+ row.update(+ {+ "h3_preencoded_member": tensor_member,+ "h3_provenance_member": provenance_member,+ "manifest_member": manifest_member,+ "tensor_digest": tensor_digest,+ "manifest_digest": manifest_digest,+ "provenance_digest": hashlib.blake2s(canonical_json(provenance)).hexdigest(),+ }+ )+ if (+ row.get("shard_size")+ and row.get("shard_md5_b64")+ and not row.get("shard_generation")+ ):+ # The freshly committed tree is local. Preserve its complete+ # immutable identity without pretending it already has a storage+ # provider generation; upload tooling must replace this value.+ row["shard_generation"] = f"local-digest:{row['shard_digest']}"+ result.append(row)+ return result+++def _payload(+ *,+ sample: Any,+ plan: Any,+ values: Mapping[str, Any],+ contract_digest: str,+) -> EncodedPayload:+ from safetensors.torch import save++ tensor_bytes = save(dict(values), metadata={"format": "h3.158f.v1"})+ manifest = {+ "schema": "solarwm.minimax-h3-preencoded.v1",+ "sample_id": plan.sample_id,+ "caption": sample.caption,+ "start_frame": plan.start_frame,+ "source_frame_indices": list(plan.source_frame_indices),+ "encoder_contract_digest": contract_digest,+ "source_manifest": dict(sample.manifest),+ }+ video_manifest = sample.manifest.get("video", {})+ source_num_frames = sample.index_values.get("num_frames") or (+ video_manifest.get("num_frames") if isinstance(video_manifest, Mapping) else None+ )+ manifest_bytes = canonical_json(manifest)+ return EncodedPayload(+ sample_id=plan.sample_id,+ key=plan.key,+ source_sample_id=str(sample.manifest.get("source_sample_id") or plan.sample_id),+ start_frame=plan.start_frame,+ source_frame_indices=tuple(plan.source_frame_indices),+ encoder_contract_digest=contract_digest,+ members={+ "tensors.safetensors": tensor_bytes,+ "manifest.json": manifest_bytes,+ },+ metadata={+ "format_version": "h3.158f.v1",+ "source_fps": video_manifest.get("fps")+ if isinstance(video_manifest, Mapping)+ else None,+ "source_num_frames": source_num_frames,+ "source_epoch_repeats": int(sample.index_values.get("epoch_repeats", 1)),+ "member_digest": {+ "tensors.safetensors": hashlib.blake2s(tensor_bytes).hexdigest(),+ "manifest.json": hashlib.blake2s(manifest_bytes).hexdigest(),+ },+ },+ )+++def run_preencode(config: Mapping[str, Any]) -> int:+ """Encode one finite raw-index epoch and publish deterministic local WDS."""++ torch, _diffusers, _transformers = require_h3_runtime()+ import torch.distributed as dist++ rank = int(os.environ.get("RANK", "0"))+ world = int(os.environ.get("WORLD_SIZE", "1"))+ local_rank = int(os.environ.get("LOCAL_RANK", "0"))+ torch.cuda.set_device(local_rank)+ if world > 1 and not dist.is_initialized():+ dist.init_process_group(backend="nccl", init_method="env://")+ device = torch.device("cuda", local_rank)+ model_cfg = config["model"]+ data_cfg = config["data"]+ output_cfg = config["preencode"]+ renders = geometry_renders_from_preencode_config(output_cfg)++ def phase(call: Any, label: str) -> Any:+ return collective_call(+ call,+ dist=dist,+ rank=rank,+ world_size=world,+ label=f"H3 preencode {label}",+ error_type=DataContractError,+ )++ def rank_zero_phase(call: Any, label: str) -> Any:+ return collective_rank_zero_call(+ call,+ dist=dist,+ rank=rank,+ world_size=world,+ label=f"H3 preencode {label}",+ error_type=DataContractError,+ )++ def load_codec() -> OfficialH3Codec:+ modules = load_conditioners(model_cfg, device=device, schedulers=False)+ return OfficialH3Codec(+ text_encoder=modules.text_encoder,+ tokenizer=modules.tokenizer,+ processor=modules.processor,+ video_vae=modules.video_vae,+ audio_vae=modules.audio_vae,+ device=device,+ encoder_identity=str(model_cfg["codec_identity"]),+ geometry=renders,+ )++ codec = phase(load_codec, "codec setup")+ requested_root = Path(str(output_cfg["output_root"])).resolve()++ def setup_staging() -> str:+ if requested_root.exists() or requested_root.is_symlink():+ raise DataContractError(f"H3 preencode output already exists: {requested_root}")+ requested_root.parent.mkdir(parents=True, exist_ok=True)+ staging = requested_root.with_name(f".{requested_root.name}.{uuid.uuid4().hex}.partial")+ staging.mkdir()+ return str(staging)++ staging_value = str(rank_zero_phase(setup_staging, "staging setup"))+ output_root = Path(staging_value)++ def validate_staging_visibility() -> None:+ if not output_root.is_dir() or output_root.is_symlink():+ raise DataContractError("H3 preencode staging directory was not created")++ phase(validate_staging_visibility, "staging visibility")++ def write_silence() -> str:+ silence_path = output_root / H3_SILENCE_PATH+ silence_path.parent.mkdir(parents=True, exist_ok=True)+ silence_bytes = codec.silence_artifact_bytes()+ silence_path.write_bytes(silence_bytes)+ return hashlib.blake2s(silence_bytes).hexdigest()++ silence_digest = str(rank_zero_phase(write_silence, "silence artifact"))+ phase(codec.bind_silence_profile, "silence binding")++ def write_encoder_contract() -> None:+ contract_path = output_root / H3_ENCODER_CONTRACT_PATH+ contract_path.write_text(+ json.dumps(codec.contract.as_dict(), indent=2, sort_keys=True) + "\n",+ encoding="utf-8",+ )++ rank_zero_phase(write_encoder_contract, "encoder-contract publication")++ def source_setup() -> tuple[Any, ...]:+ rows = normalize_raw_source_windows(read_index(resolve_index_path(data_cfg, "index")))+ source_order = {row.sample_id: row.ordinal for row in rows}+ if len(source_order) != len(rows):+ raise DataContractError("H3 preencode source index contains duplicate sample IDs")+ # Preencoding materializes each physical row once. The source weighting is+ # copied into the published row rather than redundantly encoding repeats.+ sampling_rows = tuple(+ IndexRow.from_mapping(+ row.ordinal,+ {**row.values, "epoch_repeats": 1},+ )+ for row in rows+ )+ sampler = CanonicalSampler(+ sampling_rows,+ SamplingConfig(+ seed=int(data_cfg.get("seed", 42)),+ pixel_frames=158,+ random_start=bool(data_cfg.get("random_start", False)),+ fixed_start_from_index=bool(data_cfg.get("fixed_start_from_index", False)),+ shuffle_buffer=int(data_cfg.get("shuffle_buffer", 4096)),+ partition_mode="global_occurrence",+ ),+ ReaderIdentity(rank=rank, world_size=world),+ )+ transport = data_cfg["transport"]+ resolver = resolver_from_config(+ str(transport["root"]),+ cache_dir=transport.get("cache_dir"),+ max_gib=float(transport.get("cache_max_gib", 256)),+ )+ return rows, source_order, sampler, resolver++ rows, source_order, sampler, resolver = phase(source_setup, "source preflight")++ def encode_rank() -> list[dict[str, Any]]:+ receipts = []+ pending: list[EncodedPayload] = []+ per_shard = int(output_cfg.get("samples_per_shard", 8))+ with TarShardReader(resolver, max_open=2) as shards:+ reader = RawSampleReader(rows, shards)+ shard_index = 0+ for plan in sampler.iter_epoch(0):+ raw = reader.materialize(plan)+ frames, transform = decode_resize_video(+ raw.members["video_member"],+ plan.source_frame_indices,+ decord_num_threads=int(data_cfg.get("decord_num_threads", 2)),+ )+ intrinsics_bytes = None+ intrinsics_member = str(raw.index_values.get("intrinsics_member") or "")+ if intrinsics_member:+ intrinsics_bytes = shards.read(rows[plan.row_ordinal], intrinsics_member)+ c2w, K = decode_camera(+ raw.members["camera_member"],+ plan.source_frame_indices,+ transform,+ intrinsics_bytes=intrinsics_bytes,+ )+ geometry = None+ if renders is not None:+ # Render members sit next to the video, like intrinsics, and share+ # its exact frame window and cover-crop transform.+ geometry = {}+ for key in renders.tensor_keys():+ field = geometry_member_field(key)+ member = str(raw.index_values.get(field) or "")+ if not member:+ raise DataContractError(+ f"H3 raw row {plan.sample_id!r} lacks the {field!r} render"+ )+ geometry[key] = decode_geometry_render(+ shards.read(rows[plan.row_ordinal], member),+ plan.source_frame_indices,+ transform,+ )+ values = codec.encode(+ sample_id=plan.sample_id,+ pixels=frames,+ caption=raw.caption,+ camera={+ "camera_c2w": c2w,+ "camera_K": K,+ "source_frame_indices": plan.source_frame_indices,+ },+ seed=H3_VIDEO_VAE_SEED,+ geometry=geometry,+ )+ pending.append(+ _payload(+ sample=raw,+ plan=plan,+ values=values,+ contract_digest=codec.contract.digest,+ )+ )+ if len(pending) == per_shard:+ receipts.append(+ write_shard(+ output_root,+ f"shards/rank-{rank:05d}-part-{shard_index:06d}.tar",+ pending,+ )+ )+ shard_index += 1+ pending = []+ if pending:+ receipts.append(+ write_shard(+ output_root,+ f"shards/rank-{rank:05d}-part-{shard_index:06d}.tar",+ pending,+ )+ )+ return _published_rows(receipts)++ local_rows = phase(encode_rank, "rank encoding")+ if world > 1:+ gathered: list[Any] = [None] * world+ dist.all_gather_object(gathered, local_rows)+ all_rows = [row for rank_rows in gathered for row in rank_rows]+ else:+ all_rows = local_rows++ def finalize() -> bool:+ try:+ all_rows.sort(key=lambda row: source_order[str(row["sample_id"])])+ except KeyError as exc:+ raise DataContractError("published H3 row is absent from the source index") from exc+ index_digest = write_index(output_root / H3_INDEX_PATH, all_rows)+ complete = {+ "schema": "solarwm.minimax-h3-preencode-complete.v1",+ "samples": len(all_rows),+ "encoder_contract_digest": codec.contract.digest,+ "silence_digest": silence_digest,+ "index_digest": index_digest,+ "shards": sorted({str(row["shard"]) for row in all_rows}),+ }+ (output_root / H3_COMPLETE_PATH).write_text(+ json.dumps(complete, indent=2, sort_keys=True) + "\n",+ encoding="utf-8",+ )+ _publish_staging(output_root, requested_root)+ return True++ rank_zero_phase(finalize, "corpus finalization")+ return 0+++__all__ = [+ "H3_COMPLETE_PATH",+ "H3_ENCODER_CONTRACT_PATH",+ "H3_INDEX_PATH",+ "H3_SILENCE_PATH",+ "run_preencode",+]diff --git a/src/solarwm/backends/minimax_h3/raw_data.py b/src/solarwm/backends/minimax_h3/raw_data.pynew file mode 100644index 0000000..70bb15d--- /dev/null+++ b/src/solarwm/backends/minimax_h3/raw_data.py@@ -0,0 +1,353 @@+"""Raw H3 decoding helpers for offline preencoding."""++from __future__ import annotations++import io+from collections.abc import Mapping, Sequence+from itertools import pairwise+from typing import Any++import numpy as np++from solarwm.data.index import IndexRow+from solarwm.errors import DataContractError+++def normalize_raw_source_windows(rows: Sequence[IndexRow]) -> tuple[IndexRow, ...]:+ """Use the first contiguous 158 frames when a generic raw row has no window."""++ normalized = []+ for row in rows:+ values = dict(row.values)+ has_start = "start_frame" in values+ has_indices = "source_frame_indices" in values+ if has_start and not has_indices:+ try:+ start = int(values["start_frame"])+ except (TypeError, ValueError) as exc:+ raise DataContractError(+ f"H3 raw row {row.sample_id!r} has an invalid start frame"+ ) from exc+ values["source_frame_indices"] = list(range(start, start + 158))+ elif has_indices and not has_start:+ try:+ indices = tuple(int(value) for value in values["source_frame_indices"])+ except (TypeError, ValueError) as exc:+ raise DataContractError(+ f"H3 raw row {row.sample_id!r} has invalid source indices"+ ) from exc+ if len(indices) != 158 or indices != tuple(range(indices[0], indices[0] + 158)):+ raise DataContractError(+ f"H3 raw row {row.sample_id!r} source indices are not contiguous"+ )+ values["start_frame"] = indices[0]+ elif not has_start:+ values["start_frame"] = 0+ values["source_frame_indices"] = list(range(158))+ normalized.append(IndexRow.from_mapping(row.ordinal, values))+ return tuple(normalized)+++def decode_resize_video(+ video_bytes: bytes,+ source_indices: Sequence[int],+ *,+ decord_num_threads: int = 2,+) -> tuple[np.ndarray, dict[str, int]]:+ """Decode the exact 158-frame window and apply H3's Lanczos cover crop."""++ try:+ import decord+ from PIL import Image+ except ImportError as exc: # pragma: no cover - optional runtime dependency+ raise DataContractError("H3 raw decoding requires decord and Pillow") from exc+ indices = tuple(int(value) for value in source_indices)+ if len(indices) != 158 or any(right != left + 1 for left, right in pairwise(indices)):+ raise DataContractError("H3 raw decoding requires 158 contiguous source indices")+ decord.bridge.set_bridge("native")+ try:+ reader = decord.VideoReader(+ io.BytesIO(video_bytes), num_threads=max(1, int(decord_num_threads))+ )+ frames = reader.get_batch(list(indices)).asnumpy()+ except Exception as exc:+ raise DataContractError(f"cannot decode the selected H3 video window: {exc}") from exc+ if tuple(frames.shape[:1]) != (158,) or frames.ndim != 4 or frames.shape[-1] != 3:+ raise DataContractError(f"H3 decoded frames have shape {frames.shape}")+ source_h, source_w = int(frames.shape[1]), int(frames.shape[2])+ target_h, target_w = 768, 1344+ scale = max(target_w / source_w, target_h / source_h)+ resized_w = max(target_w, round(source_w * scale))+ resized_h = max(target_h, round(source_h * scale))+ left = max(0, (resized_w - target_w) // 2)+ top = max(0, (resized_h - target_h) // 2)+ output = np.empty((158, target_h, target_w, 3), dtype=np.uint8)+ for index, frame in enumerate(frames):+ image = Image.fromarray(np.asarray(frame, dtype=np.uint8), mode="RGB")+ if image.size != (resized_w, resized_h):+ image = image.resize((resized_w, resized_h), Image.Resampling.LANCZOS)+ if image.size != (target_w, target_h):+ image = image.crop((left, top, left + target_w, top + target_h))+ output[index] = np.asarray(image, dtype=np.uint8)+ return output, {+ "source_h": source_h,+ "source_w": source_w,+ "resized_h": resized_h,+ "resized_w": resized_w,+ "crop_top": top,+ "crop_left": left,+ "target_h": target_h,+ "target_w": target_w,+ }+++def _invert_se3(matrices: np.ndarray) -> np.ndarray:+ matrices = np.asarray(matrices, dtype=np.float32)+ rotation = np.swapaxes(matrices[..., :3, :3], -1, -2)+ output = np.zeros_like(matrices)+ output[..., :3, :3] = rotation+ output[..., :3, 3] = -np.einsum("...ij,...j->...i", rotation, matrices[..., :3, 3])+ output[..., 3, 3] = 1+ return output+++def _load_intrinsics_candidate(+ archive: Any,+ intrinsics_bytes: bytes | None,+) -> np.ndarray | None:+ if intrinsics_bytes:+ loaded = np.load(io.BytesIO(intrinsics_bytes), allow_pickle=False)+ if isinstance(loaded, np.lib.npyio.NpzFile):+ try:+ for key in ("camera_K", "K", "intrinsics"):+ if key in loaded.files:+ return np.asarray(loaded[key])+ raise DataContractError("H3 intrinsics NPZ requires camera_K, K, or intrinsics")+ finally:+ loaded.close()+ return np.asarray(loaded)+ files = set(archive.files)+ for key in ("camera_K", "K", "intrinsics"):+ if key in files:+ return np.asarray(archive[key])+ if {"fx", "fy"} <= files:+ fx, fy = np.asarray(archive["fx"]), np.asarray(archive["fy"])+ cx = np.asarray(archive["cx"]) if "cx" in files else 0.5+ cy = np.asarray(archive["cy"]) if "cy" in files else 0.5+ return np.stack(np.broadcast_arrays(fx, fy, cx, cy), axis=-1)+ return None+++def _normalise_K(+ candidate: np.ndarray | None,+ source_indices: Sequence[int],+ transform: Mapping[str, int],+ *,+ enforce_focal_guard: bool = True,+) -> np.ndarray:+ count = len(source_indices)+ if candidate is None:+ output = np.zeros((count, 3, 3), dtype=np.float32)+ output[:, 0, 0] = 969.6969696969696 / (960.0 * 2.0)+ output[:, 1, 1] = 969.6969696969696 / (540.0 * 2.0)+ output[:, 0, 2] = output[:, 1, 2] = 0.5+ output[:, 2, 2] = 1+ return output+ value = np.asarray(candidate, dtype=np.float64)+ maximum = max(source_indices)+ if value.shape == (3, 3):+ matrices = np.broadcast_to(value, (count, 3, 3)).copy()+ elif value.ndim == 3 and value.shape[1:] == (3, 3):+ if value.shape[0] > maximum:+ matrices = value[np.asarray(source_indices)].copy()+ elif value.shape[0] == 1:+ matrices = np.broadcast_to(value, (count, 3, 3)).copy()+ else:+ raise DataContractError("H3 camera K cannot align to source indices")+ elif value.shape == (4,) or (value.ndim == 2 and value.shape[1] == 4):+ vectors = value.reshape(1, 4) if value.shape == (4,) else value+ if vectors.shape[0] > maximum:+ vectors = vectors[np.asarray(source_indices)]+ elif vectors.shape[0] == 1:+ vectors = np.broadcast_to(vectors, (count, 4))+ else:+ raise DataContractError("H3 camera intrinsics vectors cannot align")+ matrices = np.zeros((count, 3, 3), dtype=np.float64)+ matrices[:, 0, 0] = vectors[:, 0]+ matrices[:, 1, 1] = vectors[:, 1]+ matrices[:, 0, 2] = vectors[:, 2]+ matrices[:, 1, 2] = vectors[:, 3]+ matrices[:, 2, 2] = 1+ else:+ raise DataContractError(f"unsupported H3 camera K shape {value.shape}")+ normalized = bool(+ np.nanmax(np.abs(matrices[:, (0, 1), (0, 1)])) <= 4+ and np.nanmax(np.abs(matrices[:, (0, 1), (2, 2)])) <= 4+ )+ if normalized:+ matrices[:, 0, :] *= transform["source_w"]+ matrices[:, 1, :] *= transform["source_h"]+ matrices[:, 0, 0] *= transform["resized_w"] / transform["source_w"]+ matrices[:, 1, 1] *= transform["resized_h"] / transform["source_h"]+ matrices[:, 0, 2] = (+ matrices[:, 0, 2] * transform["resized_w"] / transform["source_w"] - transform["crop_left"]+ )+ matrices[:, 1, 2] = (+ matrices[:, 1, 2] * transform["resized_h"] / transform["source_h"] - transform["crop_top"]+ )+ matrices[:, 0, :] /= transform["target_w"]+ matrices[:, 1, :] /= transform["target_h"]+ matrices[:, 2, :] = (0, 0, 1)+ focal = matrices[:, (0, 1), (0, 1)]+ if not np.isfinite(matrices).all() or (+ enforce_focal_guard and (np.any(focal <= 0) or np.any(focal > 4))+ ):+ raise DataContractError("H3 camera K exceeds the normalized PRoPE guards")+ return matrices.astype(np.float32)+++def decode_camera(+ camera_bytes: bytes,+ source_indices: Sequence[int],+ transform: Mapping[str, int],+ *,+ intrinsics_bytes: bytes | None = None,+) -> tuple[np.ndarray, np.ndarray]:+ """Select absolute C2W and normalized K rows for one exact H3 window."""++ with np.load(io.BytesIO(camera_bytes), allow_pickle=False) as archive:+ files = set(archive.files)+ if "vipe_status" in files:+ status = str(np.asarray(archive["vipe_status"]).reshape(-1)[0]).strip().lower()+ if status != "ok":+ raise DataContractError(f"VIPE camera status is {status!r}")+ if "vipe_c2w" in files:+ c2w_full = np.asarray(archive["vipe_c2w"], dtype=np.float32)+ elif "vipe_w2c" in files:+ c2w_full = _invert_se3(archive["vipe_w2c"])+ elif "camera_c2w" in files or "c2w" in files:+ key = "camera_c2w" if "camera_c2w" in files else "c2w"+ c2w_full = np.asarray(archive[key], dtype=np.float32)+ elif "camera_w2c" in files or "w2c" in files:+ key = "camera_w2c" if "camera_w2c" in files else "w2c"+ c2w_full = _invert_se3(archive[key])+ else:+ raise DataContractError("H3 camera NPZ has no explicit c2w/w2c array")+ K = _load_intrinsics_candidate(archive, intrinsics_bytes)+ if c2w_full.ndim != 3 or c2w_full.shape[1:] != (4, 4):+ raise DataContractError("H3 camera pose array must be [N,4,4]")+ if c2w_full.shape[0] <= max(source_indices):+ raise DataContractError("H3 camera track is shorter than the selected window")+ c2w = c2w_full[np.asarray(source_indices)].copy()+ if not np.isfinite(c2w).all():+ raise DataContractError("H3 camera c2w contains non-finite values")+ return c2w.astype(np.float32), _normalise_K(K, source_indices, transform)+++# A render member is an ``.npz`` holding ``coords``: uint16 ``[F,H,W,3]`` with+# RGB = XYZ, the FastVideo renderer's 16-bit PNG encoding of the canonical+# coordinate ``x in [-1,1]`` as ``round((x + 1) * 32767.5)``. An all-zero pixel+# is a ray that hit no geometry.+H3_GEOMETRY_RENDER_KEY = "coords"+++def _read_render_frames(member: bytes | str | Any, start: int, stop: int) -> np.ndarray:+ """Read frames ``[start, stop)`` of the render without inflating the whole clip.++ Source clips can be far longer than one window; a stored (uncompressed)+ ``.npz`` member is sliced by byte offset, a compressed one is streamed.+ ``member`` is the member's bytes or a path to it.+ """++ import zipfile++ name = f"{H3_GEOMETRY_RENDER_KEY}.npy"+ source = io.BytesIO(member) if isinstance(member, (bytes, bytearray)) else member+ try:+ with zipfile.ZipFile(source) as archive:+ if name not in archive.namelist():+ raise DataContractError(f"H3 geometry render lacks the {name!r} array")+ with archive.open(name) as handle:+ version = np.lib.format.read_magic(handle)+ if version == (1, 0):+ shape, fortran, dtype = np.lib.format.read_array_header_1_0(handle)+ elif version == (2, 0):+ shape, fortran, dtype = np.lib.format.read_array_header_2_0(handle)+ else:+ raise DataContractError(f"unsupported .npy format version {version}")+ if dtype != np.dtype("<u2") or fortran or len(shape) != 4 or shape[-1] != 3:+ raise DataContractError(+ f"H3 geometry render must be C-ordered little-endian uint16 [F,H,W,3], "+ f"got {dtype} {shape}"+ )+ if stop > shape[0]:+ raise DataContractError(+ "H3 geometry render is shorter than the selected window"+ )+ frame_bytes = int(np.prod(shape[1:])) * 2+ handle.seek(handle.tell() + start * frame_bytes)+ payload = handle.read((stop - start) * frame_bytes)+ except (OSError, ValueError, zipfile.BadZipFile) as exc:+ raise DataContractError(f"cannot read the H3 geometry render: {exc}") from exc+ if len(payload) != (stop - start) * frame_bytes:+ raise DataContractError("H3 geometry render ended inside the selected window")+ return np.frombuffer(payload, dtype="<u2").reshape((stop - start, *shape[1:]))+++def _nearest_axis(geometry: int, resized: int, target: int, offset: int) -> np.ndarray:+ """Centre-aligned nearest sampling of one axis of the video's cover crop.++ Output pixel ``i`` sits at ``i + offset + 0.5`` on the resized canvas, i.e. at+ ``(i + offset + 0.5) * geometry / resized`` on a render that shares the+ video's field of view at its own resolution.+ """++ positions = (np.arange(target, dtype=np.float64) + offset + 0.5) * (geometry / resized)+ return np.clip(np.floor(positions).astype(np.int64), 0, geometry - 1)+++def decode_geometry_render(+ member: bytes | Any,+ source_indices: Sequence[int],+ transform: Mapping[str, int],+) -> np.ndarray:+ """Decode one render window into ``[F,768,1344,3]`` float32 pixels in ``[0, 1]``.++ The geometric transform is the RGB video's own cover resize and centre crop,+ so every pixel keeps its correspondence with the target frame. Sampling is+ nearest rather than Lanczos: coordinates must never be blended across a+ geometry edge. Pixels with no geometry move to the unit midpoint 0.5 (the+ coordinate origin), keeping the VAE input inside its trained range.+ """++ indices = np.asarray(tuple(int(value) for value in source_indices), dtype=np.int64)+ if indices.size == 0 or int(indices.min()) < 0:+ raise DataContractError("H3 geometry render window is empty or negative")+ coords = _read_render_frames(member, int(indices.min()), int(indices.max()) + 1)+ coords = coords[indices - int(indices.min())]+ geometry_h, geometry_w = int(coords.shape[1]), int(coords.shape[2])+ source_h, source_w = int(transform["source_h"]), int(transform["source_w"])+ if abs(geometry_w * source_h - source_w * geometry_h) > 0.01 * source_w * geometry_h:+ raise DataContractError(+ f"H3 geometry render {geometry_w}x{geometry_h} does not share the video's "+ f"{source_w}x{source_h} field of view"+ )+ rows = _nearest_axis(+ geometry_h, int(transform["resized_h"]), int(transform["target_h"]), transform["crop_top"]+ )+ columns = _nearest_axis(+ geometry_w, int(transform["resized_w"]), int(transform["target_w"]), transform["crop_left"]+ )+ window = coords[:, rows][:, :, columns]+ unit = window.astype(np.float32) / np.float32(65535.0)+ unit[~window.any(axis=-1)] = 0.5+ return unit+++__all__ = [+ "H3_GEOMETRY_RENDER_KEY",+ "decode_camera",+ "decode_geometry_render",+ "decode_resize_video",+ "normalize_raw_source_windows",+]diff --git a/src/solarwm/backends/minimax_h3/runtime.py b/src/solarwm/backends/minimax_h3/runtime.pynew file mode 100644index 0000000..83f975f--- /dev/null+++ b/src/solarwm/backends/minimax_h3/runtime.py@@ -0,0 +1,1799 @@+"""MiniMax-H3 training, transactional checkpoints and shared validation lifecycle."""++from __future__ import annotations++import hashlib+import json+import math+import os+import random+import time+from collections.abc import Mapping, Sequence+from contextlib import nullcontext+from dataclasses import replace+from pathlib import Path+from typing import Any++import numpy as np++from solarwm.checkpoint import (+ CheckpointContract,+ CheckpointTransaction,+ assert_resume_compatible,+ verify_checkpoint,+)+from solarwm.data import resolve_index_path+from solarwm.errors import BackendContractError+from solarwm.inference import (+ InferenceCase,+ InferenceEngine,+ publish_comparison_complete,+ publish_comparison_partition,+ run_validation,+)+from solarwm.inference.validation_plan import (+ load_validation_plan,+ publish_validation_plan,+ validation_plan_key,+ validation_plan_payload,+)+from solarwm.runtime import Topology, rng_identity+from solarwm.runtime.distributed import gather_and_assert_sp_identity+from solarwm.runtime.output_layout import (+ checkpoint_model_dir,+ cleanup_validation_staging,+ public_validation_dir,+ validation_staging_root,+)+from solarwm.runtime.randomness import seed_process+from solarwm.runtime.safe_state import (+ decode_numpy_rng_state,+ decode_python_rng_state,+ encode_numpy_rng_state,+ encode_python_rng_state,+)+from solarwm.training import (+ BatchIdentity,+ GradientStatus,+ JsonlEventSink,+ MicrobatchResult,+ StepPolicy,+ TrainingEngine,+)++from .artifacts import H3PreencodedStream, h3_silence_profile, load_silence_latents+from .controlnet_contract import (+ H3_GEOMETRY_MODULE,+ geometry_checkpoint_extras,+ geometry_profile_from_model_config,+)+from .distributed import get_sp_group, get_sp_rank, get_sp_size, sync_lora_gradients+from .inference import camera_fingerprint, package_generated+from .optional import load_conditioners, load_transformer, require_h3_runtime+from .stage0p5 import H3Stage0p5Core+++def _base_model_profile(model_cfg: Mapping[str, Any]) -> dict[str, Any]:+ """Portable, readable identity for a compatible MiniMax-H3 base model."""++ profile = {+ "schema": "solarwm.minimax-h3-base-profile.v1",+ "family": "minimax_h3",+ "architecture": str(model_cfg.get("architecture", "minimax-h3-33b")),+ "component": str(model_cfg.get("transformer_subfolder", "transformer")),+ "parameter_dtype_profile": "bfloat16-blocks+float32-io-timestep-v1",+ }+ revision = str(model_cfg.get("revision") or "").strip()+ if revision:+ profile["revision"] = revision+ return profile+++def _base_model_load_receipt(model_cfg: Mapping[str, Any], model: Any) -> dict[str, Any]:+ """Describe the strictly loaded model with portable semantic fields.++ A geometry ControlNet is not part of the base model: its identity travels in+ the checkpoint contract, so it is excluded from this inventory.+ """++ parameters = tuple(+ (name, parameter)+ for name, parameter in model.named_parameters()+ if not name.startswith(f"{H3_GEOMETRY_MODULE}.")+ )+ if not parameters or len({name for name, _ in parameters}) != len(parameters):+ raise BackendContractError("H3 base model parameter inventory is empty or duplicated")+ dtype_inventory: dict[str, dict[str, int]] = {}+ for _name, parameter in parameters:+ dtype = str(parameter.dtype).removeprefix("torch.")+ entry = dtype_inventory.setdefault(dtype, {"tensors": 0, "parameters": 0})+ entry["tensors"] += 1+ entry["parameters"] += int(parameter.numel())+ return {+ "schema": "solarwm.minimax-h3-base-load.v1",+ "profile": _base_model_profile(model_cfg),+ "strict_state_load": {+ "missing_keys": 0,+ "unexpected_keys": 0,+ "mismatched_shapes": 0,+ "load_errors": 0,+ },+ "parameter_inventory": {+ "tensors": len(parameters),+ "parameters": sum(int(parameter.numel()) for _, parameter in parameters),+ "dtypes": dict(sorted(dtype_inventory.items())),+ },+ }+++def _base_weights_label(model_cfg: Mapping[str, Any]) -> str:+ profile = _base_model_profile(model_cfg)+ revision = f"@{profile['revision']}" if profile.get("revision") else ""+ return f"{profile['architecture']}/{profile['component']}{revision}"+++def _topology(*, sp_size: int, require_torchrun: bool) -> Topology:+ required = {"WORLD_SIZE", "RANK", "LOCAL_WORLD_SIZE", "LOCAL_RANK"}+ if required <= set(os.environ):+ return Topology.from_environ(sp_size)+ if require_torchrun:+ raise BackendContractError("H3 SP2 training must be launched with torchrun rank variables")+ return Topology(1, 0, 1, 0, sp_size=1)+++def _reader(+ config: Mapping[str, Any],+ topology: Topology,+ *,+ index_field: str = "train_index",+ fixed_validation: bool = False,+ fixed_validation_sample_count: int | None = None,+ fixed_validation_sample_ids: Sequence[str] | None = None,+) -> Any:+ if fixed_validation and config["validation"].get("prepared_plan"):+ from .validation_inputs import H3PreparedValidationStream++ return H3PreparedValidationStream(config, topology)+ if fixed_validation and config["train"]["stage"] in {"stage1", "stage2"}:+ from .validation_stream import H3IndexedValidationStream++ return H3IndexedValidationStream(config, topology, frozen_ids=fixed_validation_sample_ids)+ data = config["data"]+ if data.get("omni_index") and not fixed_validation:+ from .omni_data import H3OmniStream++ geometry = geometry_profile_from_model_config(config["model"])+ return H3OmniStream(+ root=str(data["omni_root"]),+ index=str(data["omni_index"]),+ topology=topology,+ seed=int(data.get("seed", 42)),+ encoder_contract_path=str(data["encoder_contract_path"]),+ aligned_control=geometry is not None and geometry.reads_observation,+ length_buckets=bool(data.get("omni_length_buckets", False)),+ )+ transport = data["transport"]+ common = {+ "root": str(transport["root"]),+ "index": str(resolve_index_path(data, index_field)),+ "topology": topology,+ "seed": int(data.get("seed", 42)),+ "cache_dir": transport.get("cache_dir"),+ "cache_max_gib": float(transport.get("cache_max_gib", 256)),+ "shuffle_buffer": int(data.get("shuffle_buffer", 4096)),+ "num_workers": 1 if fixed_validation else int(data.get("num_workers", 1)),+ }+ if fixed_validation:+ validation = config["validation"]+ common.update(+ fixed_validation_selection_seed=int(validation["selection_seed"]),+ fixed_validation_noise_seed=int(validation["noise_seed"]),+ )+ return H3PreencodedStream(+ **common,+ encoder_contract_path=str(data["encoder_contract_path"]),+ fixed_validation=fixed_validation,+ fixed_validation_sample_count=fixed_validation_sample_count,+ fixed_validation_sample_ids=fixed_validation_sample_ids,+ camera_audit_latents=47 if fixed_validation else int(data["train_target_latents"]),+ geometry=geometry_profile_from_model_config(config["model"]),+ )+++def _validation_schedule(+ validation: Mapping[str, Any],+ topology: Topology,+) -> tuple[int, int]:+ raw_count = validation.get("sample_count")+ if isinstance(raw_count, bool) or not isinstance(raw_count, int) or raw_count <= 0:+ raise BackendContractError("H3 validation.sample_count must be a positive integer")+ dp_world_size = int(topology.dp_world_size)+ if raw_count % dp_world_size:+ raise BackendContractError(+ "H3 validation.sample_count must form complete logical-DP waves: "+ f"sample_count={raw_count} dp_world_size={dp_world_size}"+ )+ return raw_count, raw_count // dp_world_size+++def _publish_h3_validation_complete(+ destination: Path,+ *,+ step: int,+ pass_name: str,+ expected_local_slots: Sequence[int],+ global_slots: int,+) -> None:+ compare_paths = tuple(sorted((destination / "compare").glob("rank*.mp4")))+ manifest_paths = tuple(sorted((destination / "manifests").glob("rank_*.json")))+ local_slots = tuple(sorted(int(slot) for slot in expected_local_slots))+ if len(compare_paths) != len(local_slots) or len(manifest_paths) != len(local_slots):+ raise BackendContractError(+ "H3 validation output count differs before completion: "+ f"compare={len(compare_paths)} manifests={len(manifest_paths)} "+ f"expected_local={len(local_slots)} global={global_slots}"+ )+ slots: list[int] = []+ for path in manifest_paths:+ try:+ payload = json.loads(path.read_text(encoding="utf-8"))+ slots.append(int(payload["logical_validation_rank"]))+ except (+ OSError,+ UnicodeError,+ json.JSONDecodeError,+ KeyError,+ TypeError,+ ValueError,+ ) as exc:+ raise BackendContractError(+ f"H3 validation manifest has no usable logical slot: {path}"+ ) from exc+ if sorted(slots) != list(local_slots):+ raise BackendContractError(+ "H3 validation completion requires each node-local logical slot exactly once"+ )+ publish_comparison_complete(+ destination,+ step=int(step),+ pass_name=pass_name,+ local_slots=len(local_slots),+ global_slots=int(global_slots),+ )+++def _flow_map(train: Mapping[str, Any]) -> bool:+ """Whether the run trains (or renders) a flow map: Stage1, or a distilled omni run."""++ return str(train.get("objective", "")) == "anyflow_forward_map"+++def _checkpoint_contract(+ *,+ encoder_profile: Mapping[str, Any],+ silence_profile: Mapping[str, Any],+ base_model: Mapping[str, Any],+ config: Mapping[str, Any] | None = None,+ geometry: Any = None,+) -> CheckpointContract:+ from .observations import observation_profile_from_train_config++ train = config["train"] if config is not None else {}+ stage = str(train.get("stage", "stage0p5"))+ observations = observation_profile_from_train_config(train)+ stage_extras = {}+ if stage != "stage0p5":+ stage_extras = {+ "chunk_latents": 5,+ "window_chunks": 6,+ "target_latents": 45 if stage == "stage1" else 47,+ "rollout_latents": 50,+ "student_rope_mode": "sliding_local" if stage == "stage2" else "native_absolute",+ "training_profile": {+ key: value+ for key, value in train.items()+ if key not in {"max_steps", "global_batch_size"}+ },+ }+ return CheckpointContract(+ family="minimax_h3",+ stage=stage,+ causal_mode=str(train.get("causal_mode", "bidirectional")),+ objective=str(train.get("objective", "flow_matching")),+ objective_variant="v1_5" if _flow_map(train) else "data_ward_velocity",+ camera_translation_transform="logd4",+ parameterization="peft-lora-r384-alpha384",+ # Runs with visual observations pack longer documents and pick their own SP size.+ sp_size=(+ int(config["distributed"]["sequence_parallel_size"])+ if observations is not None+ else 4+ if stage == "stage2"+ else 2+ ),+ data_generation="h3.158f.v1",+ extras={+ "encoder_profile": json.loads(json.dumps(dict(encoder_profile), sort_keys=True)),+ "silence_profile": json.loads(json.dumps(dict(silence_profile), sort_keys=True)),+ "base_model": json.loads(json.dumps(dict(base_model), sort_keys=True)),+ "lora_target_count": 312,+ "lora_trainable_parameters": 2_075_394_048,+ "optimizer": "fp32_master_adamw",+ "ema": "rank_local_fp32_student_start39"+ if stage == "stage2"+ else "rank_local_fp32_start0",+ **stage_extras,+ **geometry_checkpoint_extras(geometry),+ **({} if observations is None else {"observations": observations.fingerprint()}),+ },+ )+++def _assert_encoder_silence_identity(+ encoder_values: Mapping[str, Any], silence_profile: Mapping[str, Any]+) -> None:+ """Reject readable silence semantics outside the selected encoder profile."""++ encoder_extras = encoder_values.get("extras", {})+ expected_silence = (+ encoder_extras.get("silence_artifact_profile")+ if isinstance(encoder_extras, Mapping)+ else None+ )+ compatible_duration = expected_silence == h3_silence_profile() and any(+ dict(silence_profile) == h3_silence_profile(pixel_frames=frames)+ for frames in (153, 158, 170)+ )+ if (+ expected_silence is not None+ and expected_silence != dict(silence_profile)+ and not compatible_duration+ ):+ raise BackendContractError(+ "H3 encoder contract was produced with a different silence artifact"+ )+++def _collective_failures(+ dist: Any,+ topology: Topology,+ local_error: str,+) -> list[str]:+ errors = [local_error]+ if dist.is_initialized():+ errors = [""] * topology.raw_world_size+ dist.all_gather_object(errors, local_error)+ return [error for error in errors if error]+++def _collective_call(+ call: Any,+ *,+ dist: Any,+ topology: Topology,+ label: str,+) -> Any:+ """Run rank-local work and report failures before the next collective phase."""++ result = None+ local_error = ""+ try:+ result = call()+ except Exception as exc:+ local_error = f"rank {topology.raw_rank}: {type(exc).__name__}: {exc}"+ failures = _collective_failures(dist, topology, local_error)+ if failures:+ raise BackendContractError(f"H3 distributed {label} failed: " + " | ".join(failures))+ if result is None:+ raise BackendContractError(f"H3 distributed {label} produced no result")+ return result+++def _load_h3_validation_plan(+ config: Mapping[str, Any],+ *,+ dist: Any,+ topology: Topology,+ sample_count: int,+) -> tuple[tuple[InferenceCase, ...] | None, Path, str, str]:+ path = (+ Path(str(config["runtime"]["output_dir"])).expanduser().resolve()+ / "validation"+ / "frozen-plan.json"+ )+ key = validation_plan_key("minimax_h3", config)++ def read_local() -> tuple[tuple[InferenceCase, ...] | None, str | None]:+ cases = load_validation_plan(+ path,+ backend="minimax_h3",+ plan_key=key,+ expected_count=sample_count,+ )+ digest = hashlib.sha256(path.read_bytes()).hexdigest() if cases is not None else None+ return cases, digest++ cases, digest = _collective_call(+ read_local,+ dist=dist,+ topology=topology,+ label="validation plan read",+ )+ states = [digest]+ if dist.is_initialized():+ states = [None] * topology.raw_world_size+ dist.all_gather_object(states, digest)+ present = [value for value in states if value is not None]+ if present and len(present) != len(states):+ raise BackendContractError(+ "frozen H3 validation plan exists on only part of the distributed world"+ )+ if len(set(present)) > 1:+ raise BackendContractError("frozen H3 validation plan differs between nodes")+ return cases, path, key, str(digest or "")+++def _gather_h3_validation_cases(+ cases: Sequence[InferenceCase],+ *,+ dist: Any,+ topology: Topology,+) -> tuple[InferenceCase, ...]:+ partitions: list[Any] = [tuple(cases)]+ if dist.is_initialized():+ partitions = [None] * topology.raw_world_size+ dist.all_gather_object(partitions, tuple(cases))+ by_slot: dict[int, InferenceCase] = {}+ for partition in partitions:+ if not isinstance(partition, tuple):+ raise BackendContractError("H3 gathered an invalid validation case partition")+ for case in partition:+ if not isinstance(case, InferenceCase):+ raise BackendContractError("H3 gathered a non-InferenceCase validation value")+ previous = by_slot.setdefault(case.slot, case)+ if previous != case:+ raise BackendContractError(f"H3 SP peers disagree on validation slot {case.slot}")+ ordered = tuple(by_slot[slot] for slot in sorted(by_slot))+ if [case.slot for case in ordered] != list(range(len(ordered))):+ raise BackendContractError("H3 gathered validation slots are not complete")+ return ordered+++def _publish_h3_validation_plan(+ path: Path,+ *,+ key: str,+ cases: Sequence[InferenceCase],+ dist: Any,+ topology: Topology,+) -> str:+ payload = validation_plan_payload(backend="minimax_h3", plan_key=key, cases=cases)+ digest = hashlib.sha256(payload).hexdigest()+ digests = [digest]+ if dist.is_initialized():+ digests = [None] * topology.raw_world_size+ dist.all_gather_object(digests, digest)+ if len(set(digests)) != 1:+ raise BackendContractError("H3 ranks disagree before freezing validation plan")+ local_error = ""+ if int(topology.local_rank) == 0:+ try:+ publish_validation_plan(+ path,+ backend="minimax_h3",+ plan_key=key,+ cases=cases,+ )+ except Exception as exc:+ local_error = f"rank {topology.raw_rank}: {type(exc).__name__}: {exc}"+ failures = _collective_failures(dist, topology, local_error)+ if failures:+ raise BackendContractError(+ "H3 validation plan publication failed collectively: " + " | ".join(failures)+ )+ if dist.is_initialized():+ dist.barrier()+ return digest+++class H3TrainingRuntime:+ """Heavy implementation of the shared :class:`TrainingRuntime` protocol."""++ def __init__(self, config: Mapping[str, Any], *, defer_resume: bool = False) -> None:+ torch, _diffusers, _transformers = require_h3_runtime()+ import torch.distributed as dist++ from .ema import H3ShardedEMA+ from .fsdp import finite_clip_norm, initialize_distributed, wrap_h3_fsdp+ from .lora import inject_h3_lora+ from .optimizer import build_adapter_optimizer++ self.torch = torch+ self.dist = dist+ self.config = config+ self.model_cfg = config["model"]+ self.train_cfg = config["train"]+ self.stage = str(self.train_cfg["stage"])+ self.flow_map = _flow_map(self.train_cfg)+ self.geometry = geometry_profile_from_model_config(self.model_cfg)+ self.extra_roles: dict[str, Any] = {}+ self.student_step = 0+ self.checkpoint_cfg = config["checkpoint"]+ self.runtime_cfg = config["runtime"]+ sp_size = int(config["distributed"]["sequence_parallel_size"])+ provisional = _topology(sp_size=sp_size, require_torchrun=True)+ configured_world = int(config["distributed"]["world_size"])+ if provisional.raw_world_size != configured_world:+ raise BackendContractError(+ f"torchrun WORLD_SIZE={provisional.raw_world_size} differs from "+ f"configured H3 world_size={configured_world}"+ )+ initialize_distributed(sp_size=sp_size, local_rank=provisional.local_rank)+ self.topology = Topology.from_environ(sp_size)+ self.rank = self.topology.raw_rank+ self.device = torch.device("cuda", self.topology.local_rank)+ self.is_main = self.rank == 0+ identity = rng_identity("minimax_h3", int(config["data"].get("seed", 42)), self.topology)+ seed_process(identity.model_init_seed)+ modules = load_transformer(self.model_cfg, device=self.device)+ if self.flow_map:+ modules.transformer.enable_anyflow(float(self.train_cfg["anyflow_gate"]))+ modules.fp32_fsdp_units = (+ *modules.fp32_fsdp_units,+ modules.transformer.delta_embedding.linear_1,+ modules.transformer.delta_embedding.linear_2,+ )+ modules.transformer.train().requires_grad_(False)+ self._before_adapter(modules.transformer)+ activation_checkpointing = bool(self.train_cfg["fsdp"]["activation_checkpointing"])+ if self.geometry is not None:+ modules.transformer.geometry_controlnet.activation_checkpointing = (+ activation_checkpointing+ )+ base_model = _base_model_load_receipt(self.model_cfg, modules.transformer)+ adapter_cfg, adapter_ranks = self._student_adapter()+ wrapped, self.lora = inject_h3_lora(+ modules.transformer,+ adapter_cfg,+ base_identity=base_model,+ ranks=adapter_ranks,+ )+ self.model = wrap_h3_fsdp(+ wrapped,+ local_rank=self.topology.local_rank,+ transformer_block_cls=modules.transformer_block_cls,+ fp32_units=modules.fp32_fsdp_units,+ ignored_parameters=self.lora.parameters,+ activation_checkpointing=activation_checkpointing,+ frozen_base_shard_size=self.train_cfg["fsdp"].get("frozen_base_shard_size"),+ saved_attention_blocks=int(self.train_cfg["fsdp"].get("saved_attention_blocks", 0)),+ )+ self._weights_id = _base_weights_label(self.model_cfg)+ initialization = self.checkpoint_cfg.get("initialization") or {}+ # A Stage0.5 run starts from the base unless it names a student: a geometry run may, and a+ # flow-map run always distils one.+ if self.stage != "stage0p5" or initialization.get("student") is not None:+ from .weights import load_initial_weights++ self._weights_id = load_initial_weights(initialization["student"], self.lora)+ if self.geometry is not None and initialization.get("geometry_controlnet_weights"):+ from .weights import load_geometry_controlnet_weights++ overlay = load_geometry_controlnet_weights(+ str(initialization["geometry_controlnet_weights"]), self.lora+ )+ self._weights_id = f"{self._weights_id}+{overlay}"+ self.objective_seed = int(identity.objective_seed)+ optimizer_cfg = self.train_cfg["optimizer"]+ self.optimizer = build_adapter_optimizer(+ self.lora,+ optimizer_cfg,+ self.train_cfg.get("geometry_controlnet_optimizer"),+ )+ warmup = int(optimizer_cfg["warmup_steps"])+ total = int(self.train_cfg["max_steps"])++ def lr_scale(step: int) -> float:+ if step < warmup:+ return float(step) / max(1, warmup)+ progress = min(1.0, (step - warmup) / max(1, total - warmup))+ minimum = float(optimizer_cfg.get("min_lr_ratio", 0.1))+ return minimum + (1.0 - minimum) * 0.5 * (1.0 + math.cos(math.pi * progress))++ self.scheduler = torch.optim.lr_scheduler.LambdaLR(self.optimizer, lr_scale)+ self.ema = (+ None+ if self.stage == "stage2"+ else H3ShardedEMA(+ self.model,+ decay=float(self.checkpoint_cfg["ema"]["decay"]),+ device=self.device,+ trainable_only=True,+ )+ )+ self.reader = _reader(config, self.topology)+ self.silence, silence_profile = load_silence_latents(+ str(config["data"]["silence_latents_path"]),+ pixel_frames=153 if self.stage == "stage1" else 158,+ )+ _assert_encoder_silence_identity(self.reader.encoder_profile, silence_profile)+ # Objective RNG advances continuously across microbatches and is+ # restored exactly by checkpoints.+ torch.manual_seed(self.objective_seed)+ torch.cuda.manual_seed_all(self.objective_seed)+ self.core = self._make_core()+ self.contract = _checkpoint_contract(+ encoder_profile=self.reader.encoder_profile,+ silence_profile=silence_profile,+ base_model=base_model,+ config=config,+ geometry=self.geometry,+ )+ self._global_step = 0+ self._finite_clip_norm = finite_clip_norm+ resume = str(self.checkpoint_cfg.get("resume_from") or "").strip()+ if resume and not defer_resume:+ self.load_checkpoint(resume)++ def _student_adapter(self) -> tuple[Mapping[str, Any], tuple[int, ...]]:+ """The run's adapter config and the ranks it may take (a subclass may widen it)."""++ return self.model_cfg["adapter"], (384,)++ def _before_adapter(self, transformer: Any) -> None:+ """A subclass may change the base before this run's adapter is injected (a merged LoRA)."""++ del transformer++ def _make_core(self) -> Any:+ if self.stage == "stage1":+ from .stage1 import H3Stage1Core++ validation_silence, _ = load_silence_latents(+ str(self.config["data"]["silence_latents_path"]),+ pixel_frames=170,+ )+ return H3Stage1Core(+ self.model,+ self.silence,+ self.device,+ self.train_cfg,+ validation_silence=validation_silence,+ geometry=self.geometry,+ )+ from .observations import observation_profile_from_train_config++ observations = observation_profile_from_train_config(self.train_cfg)+ if self.train_cfg.get("causal_omni"):+ # Block-causal omni rendering: train.causal_omni {mode, chunk, window, corrupt}.+ from .omni_causal import H3OmniCausalCore++ return H3OmniCausalCore(+ self.model,+ self.silence,+ self.device,+ dict(self.train_cfg["causal_omni"]),+ geometry=self.geometry,+ observations=observations,+ )+ if self.flow_map:+ from .omni_anyflow import H3OmniAnyFlowCore++ return H3OmniAnyFlowCore(+ self.model,+ self.silence,+ self.device,+ self.train_cfg,+ geometry=self.geometry,+ observations=observations,+ )+ return H3Stage0p5Core(+ self.model,+ self.silence,+ self.device,+ geometry=self.geometry,+ observations=observations,+ )++ @property+ def global_step(self) -> int:+ return self._global_step++ def zero_grad(self) -> None:+ self.optimizer.zero_grad(set_to_none=True)++ def _next_batch(self, stream: Any | None = None) -> Any:+ source = self.reader if stream is None else stream+ return source.next()++ def _next_batch_collective(self, stream: Any | None = None) -> Any:+ """Fail all FSDP/SP ranks before forward if any reader/codec fails."""++ return _collective_call(+ lambda: self._next_batch(stream),+ dist=self.dist,+ topology=self.topology,+ label="data preflight",+ )++ def _noise_identity(self) -> tuple[int, str]:+ state = self.torch.cuda.get_rng_state(self.device).cpu().numpy().tobytes()+ digest = hashlib.blake2s(state).digest()+ return int.from_bytes(digest[:8], "little") & 0x7FFFFFFF, digest.hex()++ def train_microbatch(self, micro_index: int, grad_accum: int) -> MicrobatchResult:+ if micro_index == 0:+ self._step_started = time.monotonic()+ self.torch.cuda.reset_peak_memory_stats(self.device)+ batch = self._next_batch()+ noise_seed, _ = self._noise_identity()+ identity = BatchIdentity(+ sample_ids=(batch.sample_id,),+ start_frames=(batch.start_frame,),+ noise_seeds=(noise_seed,),+ checkpoint_id=self._weights_id,+ plan_fingerprint=batch.plan_fingerprint,+ )+ should_sync = micro_index == grad_accum - 1+ context = (+ nullcontext()+ if should_sync or not hasattr(self.model, "no_sync")+ else self.model.no_sync()+ )+ with context:+ loss = self.core.forward_loss(batch, noise_seed=None)+ if not bool(self.torch.isfinite(loss).item()):+ raise FloatingPointError(f"H3 {self.stage} loss is non-finite")+ (loss / (int(grad_accum) * get_sp_size())).backward()+ loss_value = float(loss.item())+ return MicrobatchResult(+ identity=identity, losses={str(self.train_cfg["objective"]): loss_value}+ )++ def assert_sp_peer_identity(self, identity: BatchIdentity) -> None:+ del identity++ def prepare_optimizer_step(self) -> GradientStatus:+ sync_lora_gradients(self.lora.trainable_parameters)+ finite, norm = self._finite_clip_norm(+ (parameter for parameter in self.model.parameters() if parameter.requires_grad),+ float(self.train_cfg["optimizer"].get("gradient_clip", 1.0)),+ )+ return GradientStatus(finite=finite, norm=norm)++ def optimizer_step(self) -> None:+ self.optimizer.step()++ def scheduler_step(self) -> None:+ self.scheduler.step()++ def ema_update(self, step: int) -> None:+ ema_cfg = self.checkpoint_cfg["ema"]+ if step >= int(ema_cfg["start_step"]) and step % int(ema_cfg["update_every_steps"]) == 0:+ self.ema.update(self.model)++ def set_global_step(self, step: int) -> None:+ self._global_step = int(step)++ def _rank_state(self) -> dict[str, Any]:+ return {+ "reader": self.reader.state_dict(),+ "python_rng": encode_python_rng_state(random.getstate()),+ "numpy_rng": encode_numpy_rng_state(np.random.get_state()),+ "torch_rng": self.torch.get_rng_state(),+ "cuda_rng": self.torch.cuda.get_rng_state(self.device).cpu(),+ }++ def _release_cached_memory(self) -> None:+ """Give the caching allocator's free blocks back to the device before a checkpoint collective:+ DMD runs near a card's memory limit hit CUDA out of memory inside NCCL at the post-save barrier."""++ if self.torch.cuda.is_available():+ self.torch.cuda.empty_cache()++ def save_checkpoint(self, step: int) -> str:+ torch, dist = self.torch, self.dist+ local_state = self._rank_state()+ self._release_cached_memory()+ if dist.is_initialized():+ rank_states: list[Any] = [None] * self.topology.raw_world_size+ dist.all_gather_object(rank_states, local_state)+ else:+ rank_states = [local_state]+ checkpoint_id = ""+ save_error = ""+ if self.is_main:+ try:+ target = checkpoint_model_dir(+ str(self.runtime_cfg["output_dir"]),+ step=int(step),+ width=6,+ )+ with CheckpointTransaction(target) as transaction:+ torch.save(+ {+ "metadata": self.lora.metadata(),+ "state": {+ key: value.detach().cpu()+ for key, value in self.lora.state_dict().items()+ },+ },+ transaction.path / "adapter.pt",+ )+ torch.save(self.optimizer.state_dict(), transaction.path / "optimizer.pt")+ torch.save(self.scheduler.state_dict(), transaction.path / "scheduler.pt")+ torch.save(+ self.ema.state_dict() if self.ema is not None else None,+ transaction.path / "ema.pt",+ )+ for role, state in self.extra_roles.items():+ torch.save(+ {+ "adapter": {+ key: value.detach().cpu()+ for key, value in state["lora"].state_dict().items()+ },+ "optimizer": state["optimizer"].state_dict(),+ "scheduler": state["scheduler"].state_dict(),+ "parameter_keys": list(state["lora"].parameter_by_key),+ "global_step": int(step),+ "student_step": self.student_step,+ },+ transaction.path / f"{role}.pt",+ )+ torch.save(rank_states, transaction.path / "rank-state.pt")+ torch.save(+ {+ "global_step": int(step),+ "weights_id": self._weights_id,+ "student_step": self.student_step,+ },+ transaction.path / "runtime.pt",+ )+ verified = transaction.commit(+ step=int(step),+ contract=self.contract,+ required_components=(+ "adapter.pt",+ "optimizer.pt",+ "scheduler.pt",+ "ema.pt",+ "rank-state.pt",+ "runtime.pt",+ *(f"{role}.pt" for role in self.extra_roles),+ ),+ metadata={+ "roles": {+ "adapter.pt": "trainable_model",+ "optimizer.pt": "fp32_master_optimizer",+ "scheduler.pt": "lr_scheduler",+ "ema.pt": "fp32_rank_local_ema",+ "rank-state.pt": "per_raw_rank_rng_and_reader",+ "runtime.pt": "optimizer_step_and_weight_parent",+ },+ "base_model_saved": False,+ "base_model_reason": "immutable official checkpoint identity",+ },+ )+ checkpoint_id = verified.manifest_digest+ except Exception as exc:+ save_error = f"rank 0: {type(exc).__name__}: {exc}"+ self._release_cached_memory()+ if dist.is_initialized():+ values = [save_error, checkpoint_id]+ dist.broadcast_object_list(values, src=0)+ save_error, checkpoint_id = str(values[0]), str(values[1])+ if save_error:+ raise BackendContractError(f"H3 distributed checkpoint commit failed: {save_error}")+ dist.barrier()+ elif save_error:+ raise BackendContractError(f"H3 checkpoint commit failed: {save_error}")+ self._weights_id = checkpoint_id+ return checkpoint_id++ def load_checkpoint(self, path: str) -> None:+ torch = self.torch+ verified = _collective_call(+ lambda: self._verify_resume_checkpoint(path),+ dist=self.dist,+ topology=self.topology,+ label="checkpoint verification",+ )++ def restore_local_state() -> Mapping[str, Any]:+ root = verified.path+ runtime = torch.load(root / "runtime.pt", map_location="cpu", weights_only=True)+ if int(runtime["global_step"]) != verified.step:+ raise BackendContractError("H3 checkpoint runtime step differs from manifest")+ if self.stage == "stage2" and int(runtime.get("student_step", -1)) != max(+ 0, (verified.step - 1) // 5+ ):+ raise BackendContractError("H3 SGF checkpoint student update count differs")+ adapter = torch.load(+ root / "adapter.pt",+ map_location="cpu",+ weights_only=True,+ )+ if adapter.get("metadata") != self.lora.metadata():+ raise BackendContractError("H3 LoRA metadata differs at resume")+ # The manifest byte-verifies one shared adapter file on every rank,+ # so no parameter broadcast is needed inside this failure-collecting phase.+ self.lora.load_state_dict(adapter["state"], broadcast=False)+ self.optimizer.load_state_dict(+ torch.load(root / "optimizer.pt", map_location="cpu", weights_only=True)+ )+ self.scheduler.load_state_dict(+ torch.load(root / "scheduler.pt", map_location="cpu", weights_only=True)+ )+ ema_state = torch.load(root / "ema.pt", map_location="cpu", weights_only=True)+ if ema_state is None:+ if self.stage != "stage2" and not self.train_cfg.get("dmd"):+ raise BackendContractError("H3 checkpoint is missing EMA state")+ self.ema = None+ else:+ if self.ema is None:+ from .ema import H3ShardedEMA++ self.ema = H3ShardedEMA(+ self.model, decay=float(ema_state["decay"]), device=self.device+ )+ self.ema.load_state_dict(ema_state, self.model)+ for role, state in self.extra_roles.items():+ saved = torch.load(root / f"{role}.pt", map_location="cpu", weights_only=True)+ if (+ saved["parameter_keys"] != list(state["lora"].parameter_by_key)+ or saved["global_step"] != verified.step+ or saved["student_step"] != runtime["student_step"]+ ):+ raise BackendContractError(f"H3 {role} checkpoint identity differs")+ state["lora"].load_state_dict(saved["adapter"], broadcast=False)+ state["optimizer"].load_state_dict(saved["optimizer"])+ state["scheduler"].load_state_dict(saved["scheduler"])+ rank_states = torch.load(+ root / "rank-state.pt",+ map_location="cpu",+ weights_only=True,+ )+ if len(rank_states) != self.topology.raw_world_size:+ raise BackendContractError("H3 checkpoint raw-rank state count differs")+ rank_state = rank_states[self.rank]+ self.reader.load_state_dict(rank_state["reader"])+ random.setstate(decode_python_rng_state(rank_state["python_rng"]))+ np.random.set_state(decode_numpy_rng_state(rank_state["numpy_rng"]))+ torch.set_rng_state(rank_state["torch_rng"])+ torch.cuda.set_rng_state(rank_state["cuda_rng"], self.device)+ return runtime++ runtime = _collective_call(+ restore_local_state,+ dist=self.dist,+ topology=self.topology,+ label="checkpoint restore",+ )+ if int(runtime["global_step"]) != verified.step:+ raise BackendContractError("H3 checkpoint runtime step differs from manifest")+ self._global_step = verified.step+ self.student_step = int(runtime.get("student_step", 0))+ self._weights_id = verified.manifest_digest++ def _verify_resume_checkpoint(self, path: str) -> Any:+ verified = verify_checkpoint(path)+ assert_resume_compatible(self.contract, verified.contract)+ return verified++ def validate(self, step: int) -> Mapping[str, Any]:+ python_rng, numpy_rng = random.getstate(), np.random.get_state()+ cpu_rng = self.torch.get_rng_state()+ cuda_rng = self.torch.cuda.get_rng_state(self.device)+ try:+ return self._validate(step)+ finally:+ random.setstate(python_rng)+ np.random.set_state(numpy_rng)+ self.torch.set_rng_state(cpu_rng)+ self.torch.cuda.set_rng_state(cuda_rng, self.device)++ def _validate(self, step: int) -> Mapping[str, Any]:+ generation = {+ "generation_mode": "bidirectional" if self.stage == "stage0p5" else "autoregressive",+ "sample_solver": "flowmap"+ if self.flow_map+ else {+ "stage0p5": "shifted-euler-data-ward",+ "stage1": "flowmap",+ "stage2": "self_forcing",+ }[self.stage],+ "train_latent_frames": 45 if self.stage == "stage1" else 47,+ "rollout_latent_frames": 47 if self.stage == "stage0p5" else 50,+ "decode_latent_frames": 47,+ }+ sample_count, num_waves = _validation_schedule(+ self.config["validation"],+ self.topology,+ )+ frozen_plan, plan_path, plan_key, plan_digest = _load_h3_validation_plan(+ self.config,+ dist=self.dist,+ topology=self.topology,+ sample_count=sample_count,+ )+ plan_source = "loaded" if frozen_plan is not None else "created"+ stream = _collective_call(+ lambda: _reader(+ self.config,+ self.topology,+ index_field="test_index",+ fixed_validation=True,+ fixed_validation_sample_count=sample_count,+ fixed_validation_sample_ids=(+ tuple(case.sample_id for case in frozen_plan)+ if frozen_plan is not None+ else None+ ),+ ),+ dist=self.dist,+ topology=self.topology,+ label="validation reader setup",+ )+ fixed_cases: list[tuple[int, Any, InferenceCase]] = []+ try:+ if hasattr(stream, "prepare"):+ stream.prepare(self.dist)+ for wave_index in range(num_waves):+ batch = self._next_batch_collective(stream)+ expected_slot = wave_index * int(self.topology.dp_world_size) + int(+ self.topology.dp_rank+ )+ identity_error = ""+ if batch.validation_noise_seed is None or batch.validation_slot is None:+ identity_error = (+ f"rank {self.rank}: H3 validation batch lacks its selected slot/noise"+ )+ elif batch.validation_slot != expected_slot:+ identity_error = (+ f"rank {self.rank}: H3 validation wave {wave_index} expected slot "+ f"{expected_slot}, got {batch.validation_slot}"+ )+ identity_failures = _collective_failures(+ self.dist,+ self.topology,+ identity_error,+ )+ if identity_failures:+ raise BackendContractError(+ "H3 validation fixed-plan identity failed: " + " | ".join(identity_failures)+ )+ seed = int(batch.validation_noise_seed)+ case = InferenceCase(+ slot=int(batch.validation_slot),+ sample_id=batch.sample_id,+ prompt="",+ start_frame=batch.start_frame,+ noise_seed=seed,+ camera_fingerprint=camera_fingerprint(batch),+ metadata={+ "key": batch.sample_id,+ "plan_fingerprint": batch.plan_fingerprint,+ "source_pixel_frames": 158,+ "output_pixel_frames": 158,+ **generation,+ "dataset_source": batch.dataset_source,+ "dataset": batch.dataset_source or batch.sample_id.split("/")[0],+ "student_step": self.student_step,+ "camera_translation_transform": "logd4",+ "artifact_valid": True,+ },+ )+ if frozen_plan is not None:+ expected_case = frozen_plan[expected_slot]+ if (+ case.slot != expected_case.slot+ or case.sample_id != expected_case.sample_id+ or case.start_frame != expected_case.start_frame+ or case.noise_seed != expected_case.noise_seed+ or case.camera_fingerprint != expected_case.camera_fingerprint+ ):+ raise BackendContractError(+ f"H3 frozen validation slot {expected_slot} materialization drifted"+ )+ case = expected_case+ self.assert_sp_peer_identity(+ BatchIdentity(+ (batch.sample_id,),+ (batch.start_frame,),+ (seed,),+ self._weights_id,+ batch.plan_fingerprint,+ )+ )+ fixed_cases.append((wave_index, batch, case))+ finally:+ stream.close()+ if frozen_plan is None:+ gathered_cases = _gather_h3_validation_cases(+ tuple(case for _, _, case in fixed_cases),+ dist=self.dist,+ topology=self.topology,+ )+ if len(gathered_cases) != sample_count:+ raise BackendContractError(+ "H3 frozen validation plan has the wrong global case count"+ )+ plan_digest = _publish_h3_validation_plan(+ plan_path,+ key=plan_key,+ cases=gathered_cases,+ dist=self.dist,+ topology=self.topology,+ )+ was_training = self.model.training+ self.model.eval()+ reports: dict[str, Any] = {}+ try:+ for pass_name in self.config["validation"]["passes"]:+ name = str(pass_name)+ if name == "ema" and self.ema is None:+ continue+ comparison_destination = public_validation_dir(+ str(self.runtime_cfg["output_dir"]),+ step=int(step),+ pass_name=name,+ )+ context = self.ema.swapped_into(self.model) if name == "ema" else nullcontext()+ entered = False+ enter_error = ""+ try:+ context.__enter__()+ entered = True+ except Exception as exc:+ enter_error = f"rank {self.rank}: {type(exc).__name__}: {exc}"+ enter_failures = _collective_failures(+ self.dist,+ self.topology,+ enter_error,+ )+ if enter_failures:+ if entered:+ context.__exit__(None, None, None)+ raise BackendContractError(+ "H3 distributed validation weight swap failed: "+ + " | ".join(enter_failures)+ )+ output_directories: list[str] = []+ ordered_manifest_digests: list[str] = []+ conditioners = None+ try:+ for wave_index, batch, case in fixed_cases:+ generated_latents = _collective_call(+ lambda batch=batch, case=case: self.core.generate(+ batch,+ noise_seed=case.noise_seed,+ num_inference_steps=int(+ self.config["validation"]["num_inference_steps"]+ ),+ ),+ dist=self.dist,+ topology=self.topology,+ label=f"validation {name} wave {wave_index} generation",+ )++ output_error = ""+ if get_sp_rank() == 0:+ try:+ active_case = replace(+ case,+ metadata={+ **dict(case.metadata),+ "student_step": self.student_step,+ "num_inference_steps": int(+ self.config["validation"]["num_inference_steps"]+ ),+ "generation_pass": {+ "name": name,+ "weights": name,+ "mode": generation["generation_mode"],+ "solver": generation["sample_solver"],+ "num_inference_steps": int(+ self.config["validation"]["num_inference_steps"]+ ),+ "rollout_latent_frames": int(+ case.metadata["rollout_latent_frames"]+ ),+ "output_rollout_latent_frames": int(+ case.metadata["rollout_latent_frames"]+ ),+ },+ },+ )+ if conditioners is None:+ import gc++ gc.collect()+ self.torch.cuda.empty_cache()+ conditioners = load_conditioners(+ self.model_cfg,+ device=self.device,+ qwen=False,+ video_vae=True,+ audio_vae=False,+ schedulers=False,+ )+ packaged = package_generated(+ generated_latents,+ video_vae=conditioners.video_vae,+ device=self.device,+ weights_id=f"{self._weights_id}:{name}:{step}",+ num_inference_steps=int(+ self.config["validation"]["num_inference_steps"]+ ),+ reference_latents=batch.target_latents,+ sample_solver=generation["sample_solver"],+ )++ class CachedAdapter:+ family = "minimax_h3"++ def __init__(self, sample: Any) -> None:+ self.sample = sample++ def generate(self, _case: Any, *, weights_id: str) -> Any:+ del _case, weights_id+ return self.sample++ destination = (+ validation_staging_root(str(self.runtime_cfg["output_dir"]))+ / f"step-{int(step):08d}"+ / name+ / f"wave-{wave_index:03d}"+ / f"dp-rank-{self.topology.dp_rank:05d}"+ )+ summary = run_validation(+ CachedAdapter(packaged),+ [active_case],+ weights_id=f"{self._weights_id}:{name}:{step}",+ output_dir=destination,+ )+ publish_comparison_partition(+ summary.output_dir,+ comparison_destination,+ step=int(step),+ pass_name=name,+ cases=sample_count,+ dp_world_size=int(self.topology.dp_world_size),+ sp_size=int(self.topology.sp_size),+ run_root=Path(str(self.runtime_cfg["output_dir"])),+ logical_world_size_per_round=int(self.topology.dp_world_size),+ )+ ordered_manifest_digests.append(summary.ordered_manifest_digest)+ except Exception as exc:+ output_error = (+ f"rank {self.rank}/dp {self.topology.dp_rank} wave "+ f"{wave_index}: {type(exc).__name__}: {exc}"+ )+ output_failures = _collective_failures(+ self.dist,+ self.topology,+ output_error,+ )+ if output_failures:+ raise BackendContractError(+ "H3 distributed validation output failed: "+ + " | ".join(output_failures)+ )+ finally:+ conditioners = None+ self.torch.cuda.empty_cache()+ restore_error = ""+ if entered:+ try:+ context.__exit__(None, None, None)+ except Exception as exc:+ restore_error = f"rank {self.rank}: {type(exc).__name__}: {exc}"+ restore_failures = _collective_failures(+ self.dist,+ self.topology,+ restore_error,+ )+ if restore_failures:+ raise BackendContractError(+ "H3 distributed validation weight restore failed: "+ + " | ".join(restore_failures)+ )+ complete_error = ""+ if int(self.topology.local_rank) == 0:+ try:+ expected_local_slots = tuple(+ wave_index * int(self.topology.dp_world_size)+ + int(self.topology.node_id) * int(self.topology.local_dp_world_size)+ + local_dp_rank+ for wave_index in range(num_waves)+ for local_dp_rank in range(int(self.topology.local_dp_world_size))+ )+ _publish_h3_validation_complete(+ comparison_destination,+ step=int(step),+ pass_name=name,+ expected_local_slots=expected_local_slots,+ global_slots=sample_count,+ )+ except Exception as exc:+ complete_error = f"rank {self.rank}: {type(exc).__name__}: {exc}"+ complete_failures = _collective_failures(+ self.dist,+ self.topology,+ complete_error,+ )+ if complete_failures:+ raise BackendContractError(+ "H3 validation commit failed: " + " | ".join(complete_failures)+ )+ cleanup_error = ""+ if int(self.topology.local_rank) == 0:+ try:+ cleanup_validation_staging(+ validation_staging_root(str(self.runtime_cfg["output_dir"]))+ / f"step-{int(step):08d}"+ / name,+ output_dir=str(self.runtime_cfg["output_dir"]),+ )+ except Exception as exc:+ cleanup_error = f"rank {self.rank}: {type(exc).__name__}: {exc}"+ cleanup_failures = _collective_failures(+ self.dist,+ self.topology,+ cleanup_error,+ )+ if cleanup_failures:+ raise BackendContractError(+ "H3 validation staging cleanup failed: " + " | ".join(cleanup_failures)+ )+ output_directories.append(str(comparison_destination))+ reports[name] = {+ "writer": get_sp_rank() == 0,+ "sample_count": sample_count,+ "local_cases": len(fixed_cases),+ "slots": [case.slot for _, _, case in fixed_cases],+ "sample_ids": [case.sample_id for _, _, case in fixed_cases],+ "ordered_manifest_digests": ordered_manifest_digests,+ "output_dirs": output_directories,+ }+ finally:+ self.model.train(was_training)+ reports["validation_plan"] = {+ "path": str(plan_path),+ "source": plan_source,+ "digest": plan_digest,+ }+ return reports+++def run_training(config: Mapping[str, Any]) -> int:+ if config["train"]["stage"] == "stage2":+ from .stage2_runtime import run_sgf_training++ return run_sgf_training(config)+ if config["train"].get("objective") == "dmd":+ from .omni_dmd_runtime import run_omni_dmd_training++ return run_omni_dmd_training(config)+ runtime = H3TrainingRuntime(config)+ train = config["train"]+ checkpoint = config["checkpoint"]+ validation = config["validation"]+ policy = StepPolicy(+ max_steps=int(train["max_steps"]),+ grad_accum=int(train["gradient_accumulation_steps"]),+ save_every=int(checkpoint.get("save_every_steps", 0)),+ save_steps=tuple(int(step) for step in checkpoint.get("save_steps", ())),+ validate_every=int(validation.get("validate_every_steps", 0)),+ validation_steps=(+ (int(validation["smoke_step"]),) if int(validation["smoke_step"]) > 0 else ()+ ),+ )+ writer = (+ JsonlEventSink(Path(str(config["runtime"]["output_dir"])) / "training-events.jsonl")+ if runtime.is_main+ else None+ )++ def sink(event: Mapping[str, Any]) -> None:+ if writer is None:+ return+ if event["event"] == "optimizer_step":+ runtime.torch.cuda.synchronize(runtime.device)+ event = {+ **event,+ "compute_time_s": time.monotonic() - runtime._step_started,+ "lr": float(runtime.optimizer.param_groups[0]["lr"]),+ "peak_allocated_gib": runtime.torch.cuda.max_memory_allocated(runtime.device)+ / (1024**3),+ }+ loss = event["losses"][str(train["objective"])]+ print(+ f"[h3-{runtime.stage}] step={event['step']} loss={loss:.8f} "+ f"lr={event['lr']:.3e} compute_time_s={event['compute_time_s']:.2f}",+ flush=True,+ )+ writer(event)++ try:+ TrainingEngine(runtime, policy, event_sink=sink).run()+ finally:+ runtime.reader.close()+ return 0+++def _load_adapter_checkpoint(+ path: str,+ lora: Any,+ *,+ torch: Any,+ expected_contract: CheckpointContract,+ broadcast: bool = True,+) -> str:+ verified = verify_checkpoint(path)+ assert_resume_compatible(expected_contract, verified.contract)+ adapter = torch.load(verified.path / "adapter.pt", map_location="cpu", weights_only=True)+ if adapter.get("metadata") != lora.metadata():+ raise BackendContractError("inference LoRA metadata differs")+ lora.load_state_dict(adapter["state"], broadcast=broadcast)+ return verified.manifest_digest+++def _load_lora_checkpoint(+ path: str,+ lora: Any,+ *,+ weight_source: str,+ broadcast: bool = False,+) -> str:+ """Load a Stage0.5 LoRA package without inheriting training state."""++ source = str(weight_source).strip().lower()+ if source not in {"live", "ema"}:+ raise BackendContractError("H3 inference weight_source must be live or ema")+ checkpoint = Path(path)+ root = checkpoint if checkpoint.is_dir() else checkpoint.parent+ if (root / "checkpoint-manifest.json").is_file():+ from .weights import load_initial_weights++ identity = load_initial_weights(+ {"path": str(root), "stage": "stage0p5", "weight_source": source}, lora+ )+ if broadcast:+ import torch.distributed as dist++ if dist.is_available() and dist.is_initialized():+ for parameter in lora.parameter_by_key.values():+ dist.broadcast(parameter.detach(), src=0)+ return identity+ sidecar = root / (+ "adapter_model.safetensors" if source == "live" else "adapter_model_ema.safetensors"+ )+ if not sidecar.is_file():+ raise BackendContractError(f"H3 {source} sidecar is missing: {sidecar}")+ size = sidecar.stat().st_size+ if size <= 0:+ raise BackendContractError(f"H3 {source} sidecar is empty: {sidecar}")+ try:+ from safetensors.torch import load_file+ except ImportError as exc: # pragma: no cover - optional H3 dependency+ raise BackendContractError("H3 inference requires safetensors") from exc+ values = load_file(str(sidecar), device="cpu")+ lora.load_state_dict(values, broadcast=broadcast)+ if sidecar.stat().st_size != size:+ raise BackendContractError(f"H3 {source} sidecar changed while it was being loaded")+ return f"inventory:file={sidecar.name}:bytes={size}:role={source}"+++def run_inference(config: Mapping[str, Any]) -> int:+ if config.get("inference", {}).get("length_policy") == "source":+ from .full_inference import run_source_length_inference++ return run_source_length_inference(config)+ torch, _diffusers, _transformers = require_h3_runtime()+ import torch.distributed as dist++ from .fsdp import initialize_distributed, wrap_h3_fsdp+ from .lora import inject_h3_lora++ stage = str(config["train"]["stage"])+ geometry = geometry_profile_from_model_config(config["model"])+ distributed = config.get("distributed", {})+ configured_sp = int(distributed.get("sequence_parallel_size", 1))+ topology = _topology(sp_size=configured_sp, require_torchrun=True)+ configured_world = int(distributed["world_size"])+ if topology.raw_world_size != configured_world:+ raise BackendContractError(+ f"torchrun WORLD_SIZE={topology.raw_world_size} differs from configured "+ f"H3 inference world_size={configured_world}"+ )+ initialize_distributed(sp_size=topology.sp_size, local_rank=topology.local_rank)+ topology = (+ Topology.from_environ(topology.sp_size)+ if {"WORLD_SIZE", "RANK", "LOCAL_WORLD_SIZE", "LOCAL_RANK"} <= set(os.environ)+ else topology+ )+ device = torch.device("cuda", topology.local_rank)+ identity = rng_identity("minimax_h3", int(config["data"].get("seed", 42)), topology)+ seed_process(identity.model_init_seed)+ modules = load_transformer(config["model"], device=device)+ if _flow_map(config["train"]):+ modules.transformer.enable_anyflow(float(config["train"]["anyflow_gate"]))+ modules.fp32_fsdp_units = (+ *modules.fp32_fsdp_units,+ modules.transformer.delta_embedding.linear_1,+ modules.transformer.delta_embedding.linear_2,+ )+ modules.transformer.eval().requires_grad_(False)+ base_model = _base_model_load_receipt(config["model"], modules.transformer)+ model, lora = inject_h3_lora(+ modules.transformer,+ config["model"]["adapter"],+ base_identity=base_model,+ )+ model = wrap_h3_fsdp(+ model,+ local_rank=topology.local_rank,+ transformer_block_cls=modules.transformer_block_cls,+ fp32_units=modules.fp32_fsdp_units,+ ignored_parameters=lora.parameters,+ activation_checkpointing=False,+ )+ sample_count, num_waves = _validation_schedule(config["validation"], topology)+ stream = _reader(+ config,+ topology,+ index_field="test_index",+ fixed_validation=True,+ fixed_validation_sample_count=sample_count,+ )+ fixed_cases: list[tuple[int, Any, InferenceCase]] = []+ try:+ silence, silence_profile = load_silence_latents(+ str(config["data"]["silence_latents_path"]),+ pixel_frames=153 if stage == "stage1" else 158,+ )+ _assert_encoder_silence_identity(stream.encoder_profile, silence_profile)+ expected_contract = _checkpoint_contract(+ encoder_profile=stream.encoder_profile,+ silence_profile=silence_profile,+ base_model=base_model,+ geometry=geometry,+ )+ checkpoint_path = str(+ config.get("checkpoint", {}).get("resume_from")+ or config["runtime"].get("weights_checkpoint")+ or ""+ ).strip()+ checkpoint_cfg = config.get("checkpoint", {})+ checkpoint_format = str(checkpoint_cfg.get("format", "solarwm")).strip().lower()+ if checkpoint_path and stage != "stage0p5":+ from .weights import load_initial_weights++ weights_id = _collective_call(+ lambda: load_initial_weights(+ {+ "path": checkpoint_path,+ "stage": stage,+ "weight_source": checkpoint_cfg.get("weight_source", "live"),+ },+ lora,+ ),+ dist=dist,+ topology=topology,+ label="inference checkpoint restore",+ )+ elif checkpoint_path and checkpoint_format == "solarwm_lora":+ weights_id = _collective_call(+ lambda: _load_lora_checkpoint(+ checkpoint_path,+ lora,+ weight_source=str(checkpoint_cfg.get("weight_source", "")),+ broadcast=False,+ ),+ dist=dist,+ topology=topology,+ label="inference checkpoint restore",+ )+ elif checkpoint_path:+ weights_id = _collective_call(+ lambda: _load_adapter_checkpoint(+ checkpoint_path,+ lora,+ torch=torch,+ expected_contract=expected_contract,+ broadcast=False,+ ),+ dist=dist,+ topology=topology,+ label="inference checkpoint restore",+ )+ else:+ weights_id = _base_weights_label(config["model"])+ if hasattr(stream, "prepare"):+ stream.prepare(dist)+ for wave_index in range(num_waves):+ batch = _collective_call(+ stream.next,+ dist=dist,+ topology=topology,+ label=f"inference data wave {wave_index}",+ )+ expected_slot = wave_index * int(topology.dp_world_size) + int(topology.dp_rank)+ identity_error = ""+ if batch.validation_noise_seed is None or batch.validation_slot is None:+ identity_error = (+ f"rank {topology.raw_rank}: H3 inference batch lacks selected slot/noise"+ )+ elif int(batch.validation_slot) != expected_slot:+ identity_error = (+ f"rank {topology.raw_rank}: H3 inference wave {wave_index} "+ f"expected slot {expected_slot}, got {batch.validation_slot}"+ )+ identity_failures = _collective_failures(dist, topology, identity_error)+ if identity_failures:+ raise BackendContractError(+ "H3 inference selection failed: " + " | ".join(identity_failures)+ )+ seed = int(batch.validation_noise_seed)+ case = InferenceCase(+ slot=expected_slot,+ sample_id=batch.sample_id,+ prompt="",+ start_frame=batch.start_frame,+ noise_seed=seed,+ camera_fingerprint=camera_fingerprint(batch),+ metadata={+ "key": batch.sample_id,+ "dataset": batch.dataset_source or batch.sample_id.split("/")[0],+ "plan_fingerprint": batch.plan_fingerprint,+ "source_pixel_frames": 158,+ "output_pixel_frames": 158,+ "train_latent_frames": 45 if stage == "stage1" else 47,+ "rollout_latent_frames": 47 if stage == "stage0p5" else 50,+ "generation_mode": "bidirectional" if stage == "stage0p5" else "autoregressive",+ "sample_solver": {+ "stage0p5": "shifted-euler-data-ward",+ "stage1": "flowmap",+ "stage2": "self_forcing",+ }[stage],+ "camera_translation_transform": "logd4",+ "artifact_valid": True,+ },+ )+ gather_and_assert_sp_identity(+ {+ "sample_id": batch.sample_id,+ "start_frame": batch.start_frame,+ "noise_seed": seed,+ "plan_fingerprint": batch.plan_fingerprint,+ },+ sp_size=get_sp_size(),+ group=get_sp_group(),+ )+ fixed_cases.append((wave_index, batch, case))+ finally:+ stream.close()+ if stage == "stage1":+ from .stage1 import H3Stage1Core++ validation_silence, _ = load_silence_latents(+ str(config["data"]["silence_latents_path"]), pixel_frames=170+ )+ core = H3Stage1Core(+ model, silence, device, config["train"], validation_silence, geometry=geometry+ )+ elif stage == "stage2":+ from .stage2 import H3SGFCore++ core = H3SGFCore(model, silence, device, geometry=geometry)+ else:+ core = H3Stage0p5Core(+ model, silence, device, geometry=geometry, flow_map=_flow_map(config["train"])+ )+ conditioners = None+ comparison_destination = Path(str(config["runtime"]["output_dir"])) / "inference"+ for wave_index, batch, case in fixed_cases:+ generated_latents = _collective_call(+ lambda batch=batch, case=case: core.generate(+ batch,+ noise_seed=case.noise_seed,+ num_inference_steps=int(config["validation"]["num_inference_steps"]),+ ),+ dist=dist,+ topology=topology,+ label=f"inference wave {wave_index} generation",+ )+ output_error = ""+ if get_sp_rank() == 0:+ try:+ if conditioners is None:+ conditioners = load_conditioners(+ config["model"],+ device=device,+ qwen=False,+ video_vae=True,+ audio_vae=False,+ schedulers=False,+ )+ packaged = package_generated(+ generated_latents,+ video_vae=conditioners.video_vae,+ device=device,+ weights_id=weights_id,+ num_inference_steps=int(config["validation"]["num_inference_steps"]),+ reference_latents=batch.target_latents,+ sample_solver=str(case.metadata["sample_solver"]),+ )++ class CachedAdapter:+ family = "minimax_h3"++ def __init__(self, sample: Any) -> None:+ self.sample = sample++ def generate(self, _case: Any, *, weights_id: str) -> Any:+ del _case, weights_id+ return self.sample++ destination = (+ Path(str(config["runtime"]["output_dir"]))+ / "inference-parts"+ / f"wave-{wave_index:03d}"+ / f"dp-rank-{topology.dp_rank:05d}"+ )+ summary = InferenceEngine(CachedAdapter(packaged)).run(+ [case],+ weights_id=weights_id,+ output_dir=destination,+ )+ publish_comparison_partition(+ summary.output_dir,+ comparison_destination,+ step=0,+ pass_name="inference",+ cases=sample_count,+ dp_world_size=int(topology.dp_world_size),+ sp_size=int(topology.sp_size),+ run_root=Path(str(config["runtime"]["output_dir"])),+ logical_world_size_per_round=int(topology.dp_world_size),+ )+ except Exception as exc:+ output_error = (+ f"rank {topology.raw_rank}/dp {topology.dp_rank} wave {wave_index}: "+ f"{type(exc).__name__}: {exc}"+ )+ output_failures = _collective_failures(dist, topology, output_error)+ if output_failures:+ raise BackendContractError(+ "H3 distributed inference output failed: " + " | ".join(output_failures)+ )+ complete_error = ""+ if int(topology.local_rank) == 0:+ try:+ expected_local_slots = tuple(+ wave_index * int(topology.dp_world_size)+ + int(topology.node_id) * int(topology.local_dp_world_size)+ + local_dp_rank+ for wave_index in range(num_waves)+ for local_dp_rank in range(int(topology.local_dp_world_size))+ )+ _publish_h3_validation_complete(+ comparison_destination,+ step=0,+ pass_name="inference",+ expected_local_slots=expected_local_slots,+ global_slots=sample_count,+ )+ except Exception as exc:+ complete_error = f"rank {topology.raw_rank}: {type(exc).__name__}: {exc}"+ complete_failures = _collective_failures(dist, topology, complete_error)+ if complete_failures:+ raise BackendContractError(+ "H3 distributed inference commit failed: " + " | ".join(complete_failures)+ )+ return 0+++__all__ = ["H3TrainingRuntime", "run_inference", "run_training"]diff --git a/src/solarwm/backends/minimax_h3/sgf.py b/src/solarwm/backends/minimax_h3/sgf.pynew file mode 100644index 0000000..ab94732--- /dev/null+++ b/src/solarwm/backends/minimax_h3/sgf.py@@ -0,0 +1,108 @@+"""H3 geometry and native flow math for self-gradient forcing.++The score models see the complete 47-latent physical encode. The causal+student produces ten five-latent chunks, with only the first two latents of+the last chunk supervised. H3's separately encoded image anchor is a+condition: target latent zero remains a generated, supervised video latent.+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any++from .torch_flow import make_shifted_schedule, predict_clean_sample+++@dataclass(frozen=True)+class H3SGFWindow:+ chunk_index: int+ start: int+ current_start: int+ stop: int+ supervised_stop: int++ @property+ def prior_chunks(self) -> tuple[int, ...]:+ return tuple(range(self.start // 5, self.chunk_index))++ @property+ def local_latents(self) -> int:+ return self.stop - self.start++ @property+ def supervised_latents(self) -> int:+ return max(0, self.supervised_stop - self.current_start)+++def h3_sgf_windows(+ *,+ rollout_latents: int = 50,+ supervision_latents: int = 47,+ chunk_latents: int = 5,+ window_chunks: int = 6,+) -> tuple[H3SGFWindow, ...]:+ if (rollout_latents, supervision_latents, chunk_latents, window_chunks) != (50, 47, 5, 6):+ raise ValueError("H3 SGF requires 50 rollout / 47 score latents, five-latent chunks and W6")+ return tuple(+ H3SGFWindow(i, max(0, i - 5) * 5, i * 5, (i + 1) * 5, min((i + 1) * 5, 47))+ for i in range(10)+ )+++def h3_sgf_schedule(+ *, num_steps: int = 4, video_shift: float = 12.0, audio_shift: float = 3.0, device: Any = None+):+ """Four denoiser evaluations, using H3's video/audio shifted sigma grids.++ H3's upstream API counts grid points; SGF configuration counts evaluations.+ Convert that convention once here for both training and validation.+ """+ if num_steps != 4:+ raise ValueError("H3 SGF requires exactly four denoiser evaluations")+ return (+ make_shifted_schedule(num_steps + 1, shift=video_shift, device=device),+ make_shifted_schedule(num_steps + 1, shift=audio_shift, device=device),+ )+++def h3_sgf_score_timesteps(+ *,+ video_shift: float = 12.0,+ audio_shift: float = 3.0,+ min_sigma: float = 0.02,+ max_sigma: float = 0.98,+ device: Any = None,+ generator: Any = None,+):+ """One sampled raw noise level, shifted per H3 modality, in native time."""+ import torch++ from .torch_flow import shift_sigma++ if not 0.0 < min_sigma < max_sigma < 1.0:+ raise ValueError("H3 SGF score sigma bounds must lie strictly within (0,1)")+ raw = torch.randint(0, 1000, (), device=device, generator=generator).float() / 1000.0+ return tuple(+ 1.0 - shift_sigma(raw, shift).clamp(min_sigma, max_sigma)+ for shift in (video_shift, audio_shift)+ )+++def h3_sgf_critic_loss(velocity, *, noise, clean):+ """H3 predicts data-ward ``clean-noise``, the opposite sign from Wan."""+ import torch.nn.functional as F++ if velocity.shape != noise.shape or velocity.shape != clean.shape:+ raise ValueError("H3 critic prediction, noise and clean shapes must agree")+ return F.mse_loss(velocity.float(), clean.float() - noise.float())+++def h3_sgf_clean_prediction(noisy, velocity, timestep):+ return predict_clean_sample(noisy, velocity, timestep)+++def h3_sgf_supervised_video(rollout):+ if rollout.ndim != 5 or rollout.shape[2] != 50:+ raise ValueError("H3 SGF rollout must be [B,C,50,H,W]")+ return rollout[:, :, :47]diff --git a/src/solarwm/backends/minimax_h3/sgf_attention.py b/src/solarwm/backends/minimax_h3/sgf_attention.pynew file mode 100644index 0000000..61eed77--- /dev/null+++ b/src/solarwm/backends/minimax_h3/sgf_attention.py@@ -0,0 +1,202 @@+"""Raw KV caching and per-window MM-RoPE for H3 self-gradient forcing.++Replay is one transformer forward over [condition, audio, clean, noisy]. At+each layer, small query groups attend to exactly their visible key spans.+Each group's raw Q/K receives its own local W6 positions before camera+PRoPE. This also handles H3's joint text/image/audio document: a global+position assignment to shared clean keys cannot express every local window.+"""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any++import torch++from .torch_geometry import native_video_position_grid+from .torch_prope import prope_qkv_separate+++@dataclass+class H3RawKVCache:+ """Each SP rank owns its head shard, before MM-RoPE and camera PRoPE."""++ layers: dict[int, list[tuple[int, torch.Tensor, torch.Tensor]]] = field(default_factory=dict)++ def history(self, layer: int, chunk_index: int):+ entries = self.layers.get(layer, [])+ expected = list(range(max(0, chunk_index - 5), chunk_index))+ if [entry[0] for entry in entries] != expected:+ raise RuntimeError(f"H3 KV history differs at layer={layer}, chunk={chunk_index}")+ return entries++ def commit(self, layer: int, chunk_index: int, key, value):+ if torch.is_grad_enabled():+ raise RuntimeError("H3 raw KV commits must be detached no-grad forwards")+ entries = self.history(layer, chunk_index)+ self.layers[layer] = [+ *entries,+ (chunk_index, key.detach().clone(), value.detach().clone()),+ ][-5:]++ def clear(self):+ self.layers.clear()+++@dataclass(frozen=True)+class H3SGFAttention:+ layout: Any+ # Camera tensors include anchor slot zero and all 50 rollout video slots.+ viewmats: torch.Tensor+ intrinsics: torch.Tensor+ mode: str+ chunk_index: int = 0+ cache: H3RawKVCache | None = None+ commit_cache: bool = False+ # Populated once at the model boundary, before SP shards the input document.+ prefix_rotary: Any = None+ window_rotaries: Any = None+++def prepare_sgf_rotaries(control: H3SGFAttention, rope):+ from dataclasses import replace++ layout = control.layout+ prefix = int(layout.condition_indices.numel()) + int(layout.audio_indices.numel())+ rotations = {}+ for frames in (5, 10, 15, 20, 25, 30):+ positions = native_video_position_grid(+ frames,+ layout.latent_height,+ layout.latent_width,+ patch_size=layout.patch_size,+ origin=float(layout.text_indices.numel()),+ ).to(layout.position_ids.device)+ rotations[frames] = rope(positions)+ return replace(+ control, prefix_rotary=rope(layout.position_ids[:prefix]), window_rotaries=rotations+ )+++def _slice_rotary(rotary, start, stop):+ # Native H3 returns [S, head_dim] cosine/sine buffers.+ return tuple(value[start:stop] for value in rotary)+++def _cat_rotary(*rotaries):+ return tuple(torch.cat(parts, dim=0) for parts in zip(*rotaries, strict=True))+++def sgf_attention(q, k, v, *, control: H3SGFAttention, layer: int, apply_rotary, attend):+ """Attend over full sequence / local heads after Ulysses all-to-all."""+ layout = control.layout+ condition_rows = int(layout.condition_indices.numel())+ prefix = condition_rows + int(layout.audio_indices.numel())+ rows = int(layout.rows_per_video_frame)+ chunk_rows = 5 * rows+ expected_video = 100 if control.mode == "replay" else 5+ if q.shape[1] != prefix + expected_video * rows:+ raise ValueError("H3 SGF packed query shape differs from its rollout/replay layout")+ if control.prefix_rotary is None:+ raise ValueError("H3 SGF rotaries must be prepared before attention")+ if control.mode not in {"rollout", "replay"}:+ raise ValueError("H3 SGF attention mode must be rollout or replay")+ batch = q.shape[0]++ prefix_view = torch.eye(4, device=q.device, dtype=torch.float32).repeat(batch, prefix, 1, 1)+ prefix_K = torch.eye(3, device=q.device, dtype=torch.float32).repeat(batch, prefix, 1, 1)+ anchor_indices = layout.condition_video_indices+ prefix_view[:, anchor_indices] = control.viewmats[:, :1]+ prefix_K[:, anchor_indices] = control.intrinsics[:, :1]++ def camera_for_frames(start, stop):+ return (+ control.viewmats[:, 1 + start : 1 + stop].repeat_interleave(rows, dim=1),+ control.intrinsics[:, 1 + start : 1 + stop].repeat_interleave(rows, dim=1),+ )++ def calculate(query, key, value, qr, kr, qcam, kcam):+ query = apply_rotary(query, *qr)+ key = apply_rotary(key, *kr)+ query, key, value, output_basis = prope_qkv_separate(+ query.transpose(1, 2),+ key.transpose(1, 2),+ value.transpose(1, 2),+ query_viewmats=qcam[0],+ query_K=qcam[1],+ key_viewmats=kcam[0],+ key_K=kcam[1],+ )+ output = attend(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2))+ return output_basis(output.transpose(1, 2)).transpose(1, 2)++ prefix_cam = (prefix_view, prefix_K)+ outputs = [+ calculate(+ q[:, :condition_rows],+ k[:, :condition_rows],+ v[:, :condition_rows],+ _slice_rotary(control.prefix_rotary, 0, condition_rows),+ _slice_rotary(control.prefix_rotary, 0, condition_rows),+ (prefix_view[:, :condition_rows], prefix_K[:, :condition_rows]),+ (prefix_view[:, :condition_rows], prefix_K[:, :condition_rows]),+ )+ ]+ if prefix > condition_rows:+ outputs.append(+ calculate(+ q[:, condition_rows:prefix],+ k[:, :prefix],+ v[:, :prefix],+ _slice_rotary(control.prefix_rotary, condition_rows, prefix),+ control.prefix_rotary,+ (prefix_view[:, condition_rows:prefix], prefix_K[:, condition_rows:prefix]),+ prefix_cam,+ )+ )++ chunk_specs = (+ [(False, i) for i in range(10)] + [(True, i) for i in range(10)]+ if control.mode == "replay"+ else [(True, control.chunk_index)]+ )+ for noisy, index in chunk_specs:+ first = max(0, index - 5)+ local_frames = (index - first + 1) * 5+ if control.mode == "replay":+ qstart = prefix + (50 * rows if noisy else 0) + index * chunk_rows+ clean_start = prefix + first * chunk_rows+ clean_stop = prefix + index * chunk_rows+ history_keys, history_values = (+ k[:, clean_start:clean_stop],+ v[:, clean_start:clean_stop],+ )+ else:+ if control.cache is None:+ raise ValueError("H3 rollout requires an explicit raw KV cache")+ qstart = prefix+ history = control.cache.history(layer, index)+ history_keys = (+ torch.cat([entry[1] for entry in history], dim=1) if history else k[:, :0]+ )+ history_values = (+ torch.cat([entry[2] for entry in history], dim=1) if history else v[:, :0]+ )+ qstop = qstart + chunk_rows+ key = torch.cat((k[:, :prefix], history_keys, k[:, qstart:qstop]), dim=1)+ value = torch.cat((v[:, :prefix], history_values, v[:, qstart:qstop]), dim=1)+ rotary = control.window_rotaries[local_frames]+ qrotary = _slice_rotary(rotary, (local_frames - 5) * rows, local_frames * rows)+ krotary = _cat_rotary(control.prefix_rotary, rotary)+ current_camera = camera_for_frames(index * 5, (index + 1) * 5)+ window_camera = camera_for_frames(first * 5, (index + 1) * 5)+ key_camera = tuple(+ torch.cat((p, w), dim=1) for p, w in zip(prefix_cam, window_camera, strict=True)+ )+ outputs.append(+ calculate(q[:, qstart:qstop], key, value, qrotary, krotary, current_camera, key_camera)+ )+ if control.mode == "rollout" and control.commit_cache:+ control.cache.commit(layer, index, k[:, qstart:qstop], v[:, qstart:qstop])+ return torch.cat(outputs, dim=1)diff --git a/src/solarwm/backends/minimax_h3/sgf_rollout.py b/src/solarwm/backends/minimax_h3/sgf_rollout.pynew file mode 100644index 0000000..eed4468--- /dev/null+++ b/src/solarwm/backends/minimax_h3/sgf_rollout.py@@ -0,0 +1,300 @@+"""H3's two-pass SGF: detached raw-KV rollout, then parallel gradient replay."""++from __future__ import annotations++from dataclasses import dataclass, field+from typing import Any++import torch++from .distributed import broadcast_sp_tensor as broadcast_sequence_parallel_tensor+from .layout import build_stage0p5_layout, build_stage1_layout, patchify_video, unpatchify_video+from .sgf import H3SGFWindow, h3_sgf_schedule, h3_sgf_windows+from .sgf_attention import H3RawKVCache, H3SGFAttention+from .torch_flow import predict_clean_sample, scale_noise+from .torch_geometry import native_video_position_grid+from .torch_layout import build_row_timesteps+++@dataclass+class H3SGFInputs:+ prompt: torch.Tensor+ text_tags: torch.Tensor+ anchor_rows: torch.Tensor+ audio_rows: torch.Tensor+ audio_timestep: torch.Tensor+ camera_viewmats: torch.Tensor # [B,51,4,4], anchor + 50 target cameras+ camera_K: torch.Tensor+ latent_height: int+ latent_width: int+ keyframe_timestep: float = 0.999+ sp_enabled: bool = False+ layouts: dict[str, Any] = field(default_factory=dict)+ # Geometry ControlNet profile and ``{name: [1,24,F,48,84]}`` renders covering+ # every rollout latent (tails repeat the last encoded latent).+ geometry: Any = None+ geometry_latents: dict[str, Any] | None = None++ def layout(self, mode):+ if mode not in self.layouts:+ builder = build_stage1_layout if mode == "replay" else build_stage0p5_layout+ self.layouts[mode] = builder(+ self.text_tags.cpu(),+ 50 if mode == "replay" else 5,+ self.latent_height,+ self.latent_width,+ self.audio_rows.shape[1] // 2,+ ).to(self.prompt.device)+ return self.layouts[mode]+++@dataclass(frozen=True)+class H3SGFRollout:+ cache_target: torch.Tensor+ noisy_at_exit: torch.Tensor+ exit_index: int+ native_exit_timestep: torch.Tensor+++def _random_like(value, *, inputs, generator=None):+ noise = torch.randn(value.shape, device=value.device, dtype=value.dtype, generator=generator)+ if inputs.sp_enabled:+ broadcast_sequence_parallel_tensor(noise)+ return noise+++def _window_positions(layout, chunk_index: int):+ """Rotary grid of chunk ``chunk_index`` as the query of its own sliding window.++ The student's backbone rotates every chunk's queries at these local positions+ (``sgf_attention``); the chunk-local control trunk uses the same ones, so a+ chunk's control is identical in its rollout, its commit and the replay.+ """++ first = max(0, int(chunk_index) - 5)+ local_frames = (int(chunk_index) - first + 1) * 5+ grid = native_video_position_grid(+ local_frames,+ layout.latent_height,+ layout.latent_width,+ patch_size=layout.patch_size,+ origin=float(layout.text_indices.numel()),+ )+ return grid[-5 * int(layout.rows_per_video_frame) :]+++def sgf_geometry_control(inputs: H3SGFInputs, layout, mode: str, chunk_index: int):+ """Chunk-local geometry control for a rollout/commit chunk or the full replay."""++ if inputs.geometry is None:+ return None+ if inputs.geometry_latents is None:+ raise ValueError("H3 SGF geometry run has no geometry latents")+ from .controlnet import geometry_control_for_layout++ positions = layout.position_ids.clone()+ rows = layout.video_indices[layout.num_condition_video_rows :]+ if mode == "rollout":+ frames, copies = slice(chunk_index * 5, (chunk_index + 1) * 5), 1+ local = _window_positions(layout, chunk_index)+ else:+ frames, copies = slice(0, 50), 2+ chunks = torch.cat([_window_positions(layout, index) for index in range(10)])+ local = torch.cat((chunks, chunks))+ positions[rows] = local.to(device=positions.device, dtype=positions.dtype)++ def patches(name: str):+ window = patchify_video(inputs.geometry_latents[name][:, :, frames])+ if int(window.shape[1]) * copies != int(rows.numel()):+ raise ValueError("H3 SGF geometry latents do not cover the rollout window")+ return torch.cat([window] * copies, dim=1)++ profile = inputs.geometry+ return geometry_control_for_layout(+ layout,+ {name: patches(name) for name in profile.modalities},+ profile=profile,+ wide_rows=(+ {name: patches(f"{name}_wide") for name in profile.wide_modalities}+ if profile.wide_enabled+ else None+ ),+ position_ids=positions,+ row_indices=rows,+ attention="segments",+ )+++def h3_student_forward(+ student,+ inputs: H3SGFInputs,+ noisy,+ timestep,+ *,+ clean=None,+ chunk_index=0,+ cache=None,+ commit_cache=False,+):+ mode = "replay" if clean is not None else "rollout"+ layout = inputs.layout(mode)+ frames = 50 if mode == "replay" else 5+ video_parts = [inputs.anchor_rows]+ if clean is not None:+ video_parts.append(patchify_video(clean.detach()))+ video_parts.append(patchify_video(noisy))+ times, indices = build_row_timesteps(+ layout,+ timestep,+ inputs.audio_timestep,+ text_timestep=1.0,+ condition_video_timestep=inputs.keyframe_timestep,+ device=noisy.device,+ )+ control = H3SGFAttention(+ layout,+ inputs.camera_viewmats,+ inputs.camera_K,+ mode,+ chunk_index=chunk_index,+ cache=cache,+ commit_cache=commit_cache,+ )+ geometry = sgf_geometry_control(inputs, layout, mode, chunk_index)+ extra = {} if geometry is None else {"geometry_control": geometry}+ # SGF's camera controls are global immutable metadata. The dedicated+ # attention path selects local query/KV camera rows after Ulysses exchange.+ prediction, _ = student(+ hidden_states=torch.cat(video_parts, dim=1),+ audio_hidden_states=inputs.audio_rows,+ encoder_hidden_states=inputs.prompt,+ timestep=times,+ timestep_indices=indices,+ fused_prope=True,+ packed_sequence_parallel=inputs.sp_enabled,+ sgf_attention=control,+ return_dict=False,+ **layout.transformer_kwargs(),+ **extra,+ )+ velocity = unpatchify_video(+ prediction[:, layout.noisy_video_output_slice],+ frames,+ inputs.latent_height,+ inputs.latent_width,+ )+ return predict_clean_sample(noisy, velocity, timestep)+++def h3_sgf_rollout(+ *,+ student,+ inputs: H3SGFInputs,+ noise,+ exit_index=None,+ per_rank_exit_step=True,+ last_step_only=False,+ generator=None,+ video_shift=12.0,+ progress=None,+ inference_full_length=False,+ output_device=None,+):+ if output_device is not None and not inference_full_length:+ raise ValueError("Output offload is inference-only")+ if inference_full_length:+ count = int(noise.shape[2])+ if count <= 0 or count % 5 or exit_index != 3:+ raise ValueError(+ "Full-length H3 inference requires complete five-latent chunks and exit_index=3"+ )+ if inputs.camera_viewmats.shape[1] != count + 1 or inputs.camera_K.shape[1] != count + 1:+ raise ValueError("Full-length H3 inference requires anchor plus every rollout camera")+ windows = tuple(+ H3SGFWindow(i, max(0, i - 5) * 5, i * 5, (i + 1) * 5, (i + 1) * 5)+ for i in range(count // 5)+ )+ elif noise.shape[2] != 50:+ raise ValueError("H3 SGF requires exactly 50 rollout latents")+ else:+ windows = h3_sgf_windows()+ schedule, _ = h3_sgf_schedule(video_shift=video_shift, device=noise.device)+ if exit_index is None:+ selected = (+ torch.full((), 3, device=noise.device, dtype=torch.long)+ if last_step_only+ else torch.randint(0, 4, (), device=noise.device, generator=generator)+ )+ if not per_rank_exit_step and torch.distributed.is_initialized():+ torch.distributed.broadcast(selected, src=0)+ if inputs.sp_enabled:+ broadcast_sequence_parallel_tensor(selected)+ exit_index = int(selected)+ if not 0 <= exit_index < 4:+ raise ValueError("H3 SGF exit index must lie in [0,4)")+ cache = H3RawKVCache()+ committed, noisy_exits = [], []+ try:+ with torch.no_grad():+ for window in windows:+ current = noise[:, :, window.current_start : window.stop]+ exit_x0 = None+ # Keep four forwards on every rank even when exit choices+ # differ across DP: FSDP collectives require identical counts.+ for step_index, timestep in enumerate(schedule.timesteps):+ if step_index == exit_index:+ noisy_exits.append(+ current.detach().to(output_device).clone()+ if output_device+ else current.detach().clone()+ )+ x0 = h3_student_forward(+ student,+ inputs,+ current,+ timestep,+ chunk_index=window.chunk_index,+ cache=cache,+ )+ if not bool(torch.isfinite(x0).all()):+ raise FloatingPointError(+ f"non-finite H3 SGF chunk={window.chunk_index}, denoise={step_index}"+ )+ if step_index == exit_index:+ exit_x0 = x0.detach().clone()+ if step_index < 3:+ current = scale_noise(+ x0,+ _random_like(x0, inputs=inputs, generator=generator),+ schedule.timesteps[step_index + 1],+ )+ committed.append(exit_x0.to(output_device) if output_device else exit_x0)+ h3_student_forward(+ student,+ inputs,+ exit_x0,+ noise.new_tensor(1.0),+ chunk_index=window.chunk_index,+ cache=cache,+ commit_cache=True,+ )+ if progress is not None:+ progress(window.chunk_index + 1)+ return H3SGFRollout(+ torch.cat(committed, dim=2),+ torch.cat(noisy_exits, dim=2),+ exit_index,+ schedule.timesteps[exit_index],+ )+ finally:+ cache.clear()+++def h3_sgf_replay(*, student, inputs, rollout):+ return h3_student_forward(+ student,+ inputs,+ rollout.noisy_at_exit,+ rollout.native_exit_timestep,+ clean=rollout.cache_target,+ )diff --git a/src/solarwm/backends/minimax_h3/source_length_cache.py b/src/solarwm/backends/minimax_h3/source_length_cache.pynew file mode 100644index 0000000..1145bda--- /dev/null+++ b/src/solarwm/backends/minimax_h3/source_length_cache.py@@ -0,0 +1,179 @@+"""Bind source-length conditions and completed outputs to their actual inputs."""++from __future__ import annotations++import fcntl+import hashlib+import json+import os+import shutil+import tarfile+import uuid+from pathlib import Path+from typing import Any++from solarwm.config.loader import canonical_json+from solarwm.data.index import IndexRow, validate_relative_key+from solarwm.data.transport import LocalResolver++_SOURCE_MEMBERS = {+ "video_member": "source.mp4",+ "camera_member": "source.camera.npz",+ "intrinsics_member": "source.intrinsics.npy",+ "manifest_member": "source.manifest.json",+}+++def atomic_json(path: Path, value: Any) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ part = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")+ try:+ part.write_bytes(canonical_json(value))+ part.replace(path)+ finally:+ part.unlink(missing_ok=True)+++def file_sha256(path: Path) -> str:+ digest = hashlib.sha256()+ with path.open("rb") as handle:+ for block in iter(lambda: handle.read(8 << 20), b""):+ digest.update(block)+ return digest.hexdigest()+++def source_identity(row: dict[str, Any], args: Any) -> dict[str, Any]:+ root = str(args.dataset_root)+ return {+ "root": str(Path(root).resolve()),+ "row": row,+ }+++def preparation_identity(row: dict[str, Any], args: Any) -> str:+ return hashlib.sha256(+ canonical_json(+ {+ "schema": "solarwm.h3-source-condition-inputs.v1",+ "source": source_identity(row, args),+ "model": args.config["model"],+ }+ )+ ).hexdigest()+++def source_members(config: Any) -> dict[str, str]:+ """Index fields to extract, with a geometry ControlNet run's render members."""++ from .controlnet_contract import geometry_member_field, geometry_profile_from_model_config++ members = dict(_SOURCE_MEMBERS)+ geometry = geometry_profile_from_model_config(config["model"])+ if geometry is not None:+ for key in geometry.tensor_keys():+ field = geometry_member_field(key)+ members[field] = f"source.{field.removesuffix('_member')}.npz"+ return members+++def source_files(row: dict[str, Any], args: Any) -> Path:+ resolver = LocalResolver(Path(args.dataset_root))+ local = resolver.resolve(IndexRow.from_mapping(0, row))+ case = Path(args.work_dir) / "cases" / row["key"]+ case.mkdir(parents=True, exist_ok=True)+ identity = source_identity(row, args)+ receipt_path = case / "SOURCE.json"+ with (case / ".source.lock").open("a+b") as lock:+ fcntl.flock(lock, fcntl.LOCK_EX)+ previous = json.loads(receipt_path.read_text()) if receipt_path.is_file() else None+ if previous is not None and previous.get("identity") != identity:+ raise ValueError("Source cache belongs to another dataset or row; use a new work_dir")+ sizes = {}+ with tarfile.open(local, "r:") as archive:+ for field, name in source_members(args.config).items():+ member = archive.getmember(validate_relative_key(row[field], field=field))+ if not member.isfile():+ raise ValueError(f"Source member is not a regular file: {field}")+ sizes[name] = member.size+ target = case / name+ if (+ previous is not None+ and previous.get("sizes", {}).get(name) == member.size+ and target.is_file()+ and target.stat().st_size == member.size+ ):+ continue+ part = target.with_name(f".{name}.{os.getpid()}.tmp")+ try:+ with archive.extractfile(member) as source, part.open("wb") as output:+ shutil.copyfileobj(source, output, 8 << 20)+ if part.stat().st_size != member.size:+ raise ValueError(f"Incomplete source member: {field}")+ part.replace(target)+ finally:+ part.unlink(missing_ok=True)+ atomic_json(receipt_path, {"identity": identity, "sizes": sizes})+ return case+++def prepared_case_receipt(case: Path, row: dict[str, Any], args: Any) -> dict[str, Any]:+ receipt = json.loads((case / "READY.json").read_text())+ if receipt.get("preparation_identity") != preparation_identity(row, args):+ raise ValueError("Condition cache belongs to another source or model; use a new work_dir")+ if receipt.get("condition_sha256") != file_sha256(case / "condition.safetensors"):+ raise ValueError("Frozen condition changed after preparation")+ if row.get("input_kind") == "demo":+ from .demo_conditions import source_digest++ digest = source_digest(case)+ else:+ digest = file_sha256(case / "source.mp4")+ if receipt.get("source_sha256") != digest:+ raise ValueError("Cached source video changed after preparation")+ return receipt+++def result_identity(+ args: Any, condition: dict[str, Any], *, checkpoint_sha256: str, silence_sha256: str, seed: int+) -> str:+ return hashlib.sha256(+ canonical_json(+ {+ "schema": "solarwm.h3-source-result-inputs.v1",+ "preparation_identity": condition["preparation_identity"],+ "condition_sha256": condition["condition_sha256"],+ "checkpoint_receipt_sha256": checkpoint_sha256,+ "weight_source": args.weight_source,+ "silence_sha256": silence_sha256,+ "noise_seed": seed,+ "fps_policy": args.fps_policy,+ "save_latents": bool(args.config["inference"].get("save_latents", False)),+ **(+ {"stream_decode": True}+ if args.config["inference"].get("stream_decode", False)+ else {}+ ),+ }+ )+ ).hexdigest()+++def verify_completed_result(manifest: Path, *, expected_identity: str, save_latents: bool) -> None:+ receipt = json.loads(manifest.read_text())+ if receipt.get("result_identity") != expected_identity:+ raise ValueError("Existing result belongs to another condition or inference configuration")+ suffixes = {+ ".mp4": "output_sha256",+ ".compare.mp4": "compare_sha256",+ ".storyboard.jpg": "storyboard_sha256",+ }+ if receipt.get("source_row", {}).get("input_kind") == "demo":+ suffixes.pop(".compare.mp4")+ if save_latents:+ suffixes[".latents.safetensors"] = "latents_sha256"+ for suffix, field in suffixes.items():+ path = manifest.with_name(manifest.stem + suffix)+ if not path.is_file() or file_sha256(path) != receipt.get(field):+ raise ValueError(+ f"Existing result file differs from its completion receipt: {path.name}"+ )diff --git a/src/solarwm/backends/minimax_h3/stage0p5.py b/src/solarwm/backends/minimax_h3/stage0p5.pynew file mode 100644index 0000000..95b44d6--- /dev/null+++ b/src/solarwm/backends/minimax_h3/stage0p5.py@@ -0,0 +1,942 @@+"""Exact Stage0.5 data-ward flow-matching forward and Euler sampler."""++from __future__ import annotations++import hashlib+from dataclasses import dataclass+from typing import Any++from .artifacts import H3ArtifactBatch+from .distributed import broadcast_sp_tensor, is_sequence_parallel_enabled+from .flow import explicit_schedule, make_shifted_schedule+from .layout import (+ H3PackedLayout,+ H3VisualObservationSpec,+ build_row_timesteps,+ build_stage0p5_layout,+ patchify_video,+ unpatchify_video,+)+++def aligned_control_latents(batch: H3ArtifactBatch) -> Any:+ """``[24,F,h,w]`` of the one observation laid over every target frame on the target's grid.++ The observation-reading trunk takes what the sample lays over the target frame for frame,+ whatever it is. It needs exactly one: with none there is nothing to follow, and several would+ have to be merged by a rule the trunk cannot learn. Frames are put back in target order.+ """++ target = batch.target_latents+ frames = int(target.shape[1])+ covering = [+ item+ for item in (batch.visual_observations or ())+ if item.frames is not None+ and int(item.latents.shape[1]) == len(item.frames)+ and set(range(frames)) <= {int(f) for f in item.frames}+ and tuple(item.latents.shape[2:]) == tuple(target.shape[2:])+ ]+ if len(covering) != 1:+ raise ValueError(+ f"the observation input needs exactly one observation laid over all {frames} target "+ f"frames on the target grid; this sample has {len(covering)}"+ )+ item = covering[0]+ position = {int(f): index for index, f in enumerate(item.frames)}+ return item.latents[:, [position[f] for f in range(frames)]]+++def covers_every_frame(batch: H3ArtifactBatch, item: Any) -> bool:+ """Whether ``item`` is laid over every target frame (on whatever grid)."""++ frames = int(batch.target_latents.shape[1])+ return (+ item.frames is not None+ and int(item.latents.shape[1]) == len(item.frames)+ and set(range(frames)) <= {int(f) for f in item.frames}+ )+++def optional_control_latents(batch: H3ArtifactBatch) -> Any:+ """:func:`aligned_control_latents`, or None for a sample with no observation over every frame - a+ first-frame-only or text-only sample (the trunk on SolarRenderer data). An observation over+ every frame that the trunk cannot take (off the target grid, or several) still fails."""++ if not any(covers_every_frame(batch, item) for item in (batch.visual_observations or ())):+ return None+ return aligned_control_latents(batch)+++def trunk_keep(batch: H3ArtifactBatch) -> Any:+ """Which observations a trunk-reading sample still packs beside the target: every aligned one but the+ structure video the trunk reads (the first frame stays in the sequence); free pictures are not packed."""++ items = tuple(batch.visual_observations or ())++ def keep(index: int) -> bool:+ item = items[index]+ return item.frames is not None and not covers_every_frame(batch, item)++ return keep+++def _keep_none(index: int) -> bool:+ return False+++def batch_geometry_latents(+ batch: H3ArtifactBatch, geometry: Any, device: Any+) -> dict[str, Any] | None:+ """``{modality | modality_wide: [1,24,F,48,84] float32}`` for a geometry run, else None."""++ if geometry is None:+ return None+ import torch++ from .controlnet_contract import geometry_tensor_key++ if geometry.reads_observation:+ latents = optional_control_latents(batch)+ if latents is None: # no structure: the trunk still runs (every rank must), on zeros, at zero gain+ latents = torch.zeros(batch.target_latents.shape, dtype=torch.float32)+ return {+ "observation": latents.to(device, dtype=torch.float32, non_blocking=True).unsqueeze(0)+ }+ stored = batch.geometry_latents+ if stored is None:+ raise ValueError("H3 geometry ControlNet run received a batch without geometry")+ latents = {name: stored[geometry_tensor_key(name)] for name in geometry.modalities}+ for name in geometry.wide_modalities:+ latents[f"{name}_wide"] = stored[geometry_tensor_key(name, wide=True)]+ return {+ name: value.to(device, dtype=torch.float32, non_blocking=True).unsqueeze(0)+ for name, value in latents.items()+ }+++@dataclass(frozen=True)+class H3TorchLayout:+ source: H3PackedLayout+ position_ids: Any+ token_tags: Any+ video_indices: Any+ audio_indices: Any+ text_indices: Any+ camera_video_indices: Any+ camera_frame_ids: Any++ @property+ def target_video_output_slice(self) -> slice:+ return self.source.target_video_output_slice+++@dataclass(frozen=True)+class _Prepared:+ """One training sample's conditioning, drawn by :meth:`H3Stage0p5Core.prepare`."""++ clean: Any+ video_noise: Any+ video_t: Any+ audio_t: Any+ audio_rows: Any+ condition_rows: list+ prompt: Any+ layout: Any+ viewmats: Any+ K: Any+ frame_ids: Any+ geometry: Any+ observation_degrees: Any+ plan: Any+++class H3Stage0p5Core:+ """One model/silence/device implementation shared by train and inference."""++ def __init__(+ self,+ model: Any,+ silence_latents: Any,+ device: Any,+ *,+ geometry: Any = None,+ observations: Any = None,+ flow_map: bool = False,+ ) -> None:+ self.model = model+ self.device = device+ self.silence = silence_latents+ self.geometry = geometry+ # A model with AnyFlow conditioning: every forward names where the target rows go (r).+ self.flow_map = bool(flow_map)+ # An H3ObservationProfile; None keeps the fixed first-frame + camera recipe.+ self.observations = observations+ self._layouts: dict[str, H3TorchLayout] = {}++ def layout(+ self,+ tags: Any,+ *,+ anchors: tuple[str, ...] = ("first",),+ observations: tuple[H3VisualObservationSpec, ...] = (),+ grid: tuple[int, int] = (48, 84),+ frames: int = 47,+ ) -> H3TorchLayout:+ import torch++ tags_cpu = tags.detach().cpu().long().numpy()+ key = hashlib.blake2s(+ tags_cpu.tobytes() + f"|{anchors}|{observations}|{grid}|{frames}".encode()+ ).hexdigest()+ cached = self._layouts.get(key)+ if cached is not None:+ return cached+ source = build_stage0p5_layout(+ tags_cpu,+ frames,+ *grid,+ self.audio_latents(frames),+ keyframe_anchors=anchors,+ observations=observations,+ )+ layout = H3TorchLayout(+ source=source,+ position_ids=torch.from_numpy(source.position_ids).to(self.device),+ token_tags=torch.from_numpy(source.token_tags).long().to(self.device),+ video_indices=torch.from_numpy(source.video_indices).long().to(self.device),+ audio_indices=torch.from_numpy(source.audio_indices).long().to(self.device),+ text_indices=torch.from_numpy(source.text_indices).long().to(self.device),+ camera_video_indices=torch.from_numpy(source.camera_video_indices)+ .long()+ .to(self.device),+ camera_frame_ids=torch.from_numpy(source.camera_frame_ids).long().to(self.device),+ )+ self._layouts[key] = layout+ while len(self._layouts) > 4:+ self._layouts.pop(next(iter(self._layouts)))+ return layout++ def observation_rows(self, batch: H3ArtifactBatch, generator: Any, keep: Any = None) -> Any:+ """``(specs, degrees, [1,N,96] rows)`` of the batch's visual observations.++ Each is noised at its own known degree, like the first-frame anchor. ``keep``+ decides per index whether a sample uses it. The draws come after every other+ draw of the sample, so a batch without observations consumes the same stream.+ """++ import torch++ specs, degrees, rows = [], [], []+ for index, item in enumerate(batch.visual_observations or ()):+ if keep is not None and not keep(index):+ continue+ clean = item.latents.to(self.device, dtype=torch.float32, non_blocking=True)+ clean = clean.unsqueeze(0)+ if clean.ndim != 5 or clean.shape[1] != 24:+ raise ValueError("H3 visual observations must be [24,f,h,w] latents")+ if not 0.0 < float(item.degree) <= 1.0:+ raise ValueError("a visual observation's degree must lie in (0, 1]")+ noise = torch.randn(+ clean.shape, generator=generator, device=self.device, dtype=torch.float32+ )+ self._broadcast_logical_sample(clean, noise)+ degree = float(item.degree)+ rows.append(patchify_video(degree * clean + (1.0 - degree) * noise))+ degrees.append(degree)+ specs.append(+ H3VisualObservationSpec(+ num_frames=int(clean.shape[2]),+ height=int(clean.shape[3]),+ width=int(clean.shape[4]),+ frames=None if item.frames is None else tuple(int(f) for f in item.frames),+ boxes=None+ if item.boxes is None+ else tuple(tuple(float(v) for v in box) for box in item.boxes),+ )+ )+ if not rows:+ return (), None, None+ return tuple(specs), tuple(degrees), torch.cat(rows, dim=1)++ @staticmethod+ def latent_grid(batch: H3ArtifactBatch) -> tuple[int, int]:+ """Landscape ``(48, 84)`` or portrait ``(84, 48)``, read from the sample's latents."""++ source = batch.target_latents if batch.anchor_latents is None else batch.anchor_latents+ grid = (int(source.shape[-2]), int(source.shape[-1]))+ if grid not in {(48, 84), (84, 48)}:+ raise ValueError(f"H3 latents must be 48x84 or 84x48, got {grid}")+ return grid++ @staticmethod+ def latent_frames(batch: H3ArtifactBatch) -> int:+ """Latent frames of the clip: 47 for the 158-frame release, ``5n+2`` in general."""++ frames = 47 if batch.target_latents is None else int(batch.target_latents.shape[1])+ if frames < 2 or (frames - 2) % 5:+ raise ValueError(f"H3 clips hold 5n+2 latent frames, got {frames}")+ return frames++ @staticmethod+ def audio_latents(frames: int) -> int:+ """Audio latents covering the clip: 40 per second of its ``17n+5`` frames at 24 fps."""++ return int(round((17 * ((frames - 2) // 5) + 5) / 24.0 * 40))++ def prompt(self, batch: H3ArtifactBatch, *, use_anchor: bool) -> tuple[Any, Any]:+ """``(embeds [1,N,5120], tags)``: the joint prompt, or the text-only one."""++ import torch++ embeds, tags = batch.prompt_embeds, batch.text_token_tags+ if not use_anchor:+ if batch.text_prompt_embeds is None or batch.text_prompt_tags is None:+ raise ValueError(+ "a sample without its first-frame anchor needs text-only prompt embeds: "+ "the joint prompt was encoded with that frame"+ )+ embeds, tags = batch.text_prompt_embeds, batch.text_prompt_tags+ embeds = embeds.to(self.device, dtype=torch.bfloat16, non_blocking=True).unsqueeze(0)+ tags = tags.to(self.device, dtype=torch.long)+ self._broadcast_logical_sample(tags, embeds)+ return embeds, tags++ def audio_rows(self, frames: int | None = None) -> Any:+ """Silence rows; ``frames`` trims them to a clip shorter than the silence artifact."""++ audio = self.silence.to(+ device=self.device, dtype=__import__("torch").float32, non_blocking=True+ ).unsqueeze(0)+ if frames is not None:+ audio = audio[..., : self.audio_latents(frames)]+ return audio.permute(0, 1, 3, 2).reshape(1, -1, 32).contiguous()++ @staticmethod+ def _broadcast_logical_sample(*values: Any) -> None:+ """Make SP rank zero authoritative for every model input tensor."""++ if is_sequence_parallel_enabled():+ for value in values:+ broadcast_sp_tensor(value)++ def _camera(self, batch: H3ArtifactBatch, layout: H3TorchLayout) -> tuple[Any, Any, Any]:+ import torch++ views = batch.camera_viewmats.to(+ self.device, dtype=torch.float32, non_blocking=True+ ).unsqueeze(0)+ K = batch.camera_K.to(self.device, dtype=torch.float32, non_blocking=True).unsqueeze(0)+ # A clip carries one camera per latent frame, or one per token; both counts follow the+ # clip's own length, which the omni path lets vary.+ frames, grid = self.latent_frames(batch), self.latent_grid(batch)+ rows_per_latent = (grid[0] // 2) * (grid[1] // 2)+ if views.shape[1] == frames:+ return views, K, layout.camera_frame_ids+ if views.shape[1] == frames * rows_per_latent:+ if layout.source.num_observation_rows:+ raise RuntimeError("token-aligned cameras do not cover visual observations")+ if not layout.source.num_condition_video_rows:+ return views, K, None+ anchor_views = views[:, :rows_per_latent]+ anchor_K = K[:, :rows_per_latent]+ return (+ torch.cat((anchor_views, views), dim=1),+ torch.cat((anchor_K, K), dim=1),+ None,+ )+ raise RuntimeError("H3 camera artifact is neither latent nor token aligned")++ @property+ def reads_observation(self) -> bool:+ """Whether the aligned observation reaches the model through the control trunk alone."""++ return self.geometry is not None and self.geometry.reads_observation++ def geometry_latents(self, batch: H3ArtifactBatch) -> dict[str, Any] | None:+ """Device-resident ``[1,24,47,48,84]`` geometry latents, SP-rank-0 authoritative."""++ values = batch_geometry_latents(batch, self.geometry, self.device)+ if values is not None:+ self._broadcast_logical_sample(*values.values())+ return values++ def geometry_control(+ self,+ layout: H3TorchLayout,+ latents: dict[str, Any] | None,+ *,+ frames: Any = None,+ ) -> Any:+ """Bind ``latents[..., frames]`` to the layout's target rows (dense attention)."""++ if latents is None:+ return None+ from .controlnet import geometry_control_for_layout++ def rows(name: str) -> Any:+ value = latents[name] if frames is None else latents[name][:, :, frames]+ return patchify_video(value)++ return geometry_control_for_layout(+ layout.source,+ {name: rows(name) for name in self.geometry.modalities},+ profile=self.geometry,+ wide_rows=(+ {name: rows(f"{name}_wide") for name in self.geometry.wide_modalities}+ if self.geometry.wide_enabled+ else None+ ),+ position_ids=layout.position_ids,+ row_indices=layout.video_indices[layout.source.num_condition_video_rows :],+ )++ @staticmethod+ def shifted_timestep(generator: Any | None, *, shift: float, device: Any) -> Any:+ import torch++ sigma = torch.rand((), generator=generator, device=device, dtype=torch.float32)+ shifted = float(shift) * sigma / (1.0 + (float(shift) - 1.0) * sigma)+ return 1.0 - shifted++ @staticmethod+ def _row_times(+ layout: H3TorchLayout,+ video_t: Any,+ audio_t: Any,+ device: Any,+ text_t: Any = None,+ observation_degrees: Any = None,+ video_r: Any = None,+ ) -> tuple[Any, Any, Any]:+ """``video_t`` is a scalar, or a known-degree field together with its ``text_t``.++ With ``video_r`` (a flow map) the target rows move from ``video_t`` to ``video_r`` and+ every other row stays where it is (r = t); rows are grouped by their (t, r) pair, so an+ equal t with a different destination stays distinct. Returns (t, r or None, index).+ """++ import numpy as np+ import torch++ field = text_t is not None+ times, inverse = build_row_timesteps(+ layout.source,+ video_t.detach().float().cpu().numpy() if field else float(video_t.item()),+ float(audio_t.item()),+ text_timestep=float(text_t.item()) if field else None,+ condition_video_timestep=0.999,+ observation_timesteps=observation_degrees,+ )+ destinations = None+ if video_r is not None:+ if field:+ raise ValueError("a flow-map step moves every target row from one time")+ t_rows = times[inverse]+ r_rows = t_rows.copy()+ r_rows[layout.source.noisy_video_indices] = float(video_r.item())+ pairs, inverse = np.unique(np.stack((t_rows, r_rows), axis=1), axis=0, return_inverse=True)+ times, destinations = pairs[:, 0].copy(), pairs[:, 1].copy()+ destinations = torch.from_numpy(destinations).to(device=device, dtype=torch.float32)+ return (+ torch.from_numpy(np.ascontiguousarray(times)).to(device=device, dtype=torch.float32),+ destinations,+ torch.from_numpy(np.asarray(inverse).reshape(-1)).to(device=device, dtype=torch.long),+ )++ def _forward(+ self,+ *,+ video_rows: Any,+ audio_rows: Any,+ prompt: Any,+ layout: H3TorchLayout,+ video_t: Any,+ audio_t: Any,+ viewmats: Any,+ K: Any,+ frame_ids: Any,+ geometry_control: Any = None,+ text_t: Any = None,+ observation_degrees: Any = None,+ video_r: Any = None,+ model: Any = None,+ ) -> Any:+ """``viewmats=None`` runs without a camera, on the backbone's native attention.++ A flow-map core needs ``video_r``, where this evaluation sends the target rows.+ ``model`` evaluates another copy on the same inputs (a DMD teacher or critic).+ """++ if self.flow_map != (video_r is not None):+ raise ValueError("a flow-map core names r on every forward, and only it does")+ times, destinations, inverse = self._row_times(+ layout, video_t, audio_t, self.device, text_t, observation_degrees, video_r+ )+ extra = {} if geometry_control is None else {"geometry_control": geometry_control}+ if destinations is not None:+ extra["r_timestep"] = destinations+ camera = viewmats is not None+ video, _audio = (self.model if model is None else model)(+ hidden_states=video_rows,+ audio_hidden_states=audio_rows,+ encoder_hidden_states=prompt,+ timestep=times,+ timestep_indices=inverse,+ token_tags=layout.token_tags,+ position_ids=layout.position_ids,+ video_indices=layout.video_indices,+ audio_indices=layout.audio_indices,+ text_indices=layout.text_indices,+ attention_mask=None,+ fused_prope=camera,+ prope_token_indices=layout.camera_video_indices if camera else None,+ prope_frame_ids=frame_ids if camera else None,+ cam_viewmats=viewmats,+ cam_K=K if camera else None,+ stage0p5_sequence_parallel=is_sequence_parallel_enabled(),+ return_dict=False,+ **extra,+ )+ return video[:, layout.target_video_output_slice]++ def observation_plan(+ self,+ generator: Any,+ video_t: Any,+ known_degrees: Any = None,+ grid: tuple[int, int] = (48, 84),+ frames: int = 47,+ ) -> Any:+ """Draw this sample's plan after the ordinary noise draws; ``None`` without a profile."""++ if self.observations is None:+ return None+ import numpy as np+ import torch++ from .observations import sample_observation_plan++ seed = torch.randint(+ 0, 2**31 - 1, (1,), generator=generator, device=self.device, dtype=torch.int64+ )+ self._broadcast_logical_sample(seed)+ given = None+ if known_degrees is not None:+ given = known_degrees.to(self.device, dtype=torch.float32).reshape(-1)+ if given.numel() != frames * 1008:+ raise ValueError("a batch's known_degrees must cover every target row")+ self._broadcast_logical_sample(given)+ given = given.cpu().numpy()+ return sample_observation_plan(+ self.observations,+ frames=frames,+ grid_height=grid[0] // 2,+ grid_width=grid[1] // 2,+ base_time=float(video_t.item()),+ shift=12.0,+ rng=np.random.default_rng(int(seed.item())),+ given=given,+ )++ def prepare(self, batch: H3ArtifactBatch, generator: Any, draw_time: Any) -> _Prepared:+ """Everything one training forward conditions on, drawn in the fixed order.++ ``draw_time(generator)`` gives the sample's data-ward target time; it is called where the+ ordinary recipe draws it, so every other draw lands exactly where it always has.+ """++ import dataclasses++ import torch++ clean = batch.target_latents.to(+ self.device, dtype=torch.float32, non_blocking=True+ ).unsqueeze(0)+ frames = self.latent_frames(batch)+ # A sample without a first-frame latent or without cameras simply has none to use.+ has_anchor, has_camera = batch.anchor_latents is not None, batch.camera_viewmats is not None+ anchor = (+ batch.anchor_latents.to(self.device, dtype=torch.float32, non_blocking=True).unsqueeze(+ 0+ )+ if has_anchor+ else clean[:, :, :1]+ )+ anchor_noise = torch.randn(+ anchor.shape, generator=generator, device=self.device, dtype=torch.float32+ )+ audio_clean = self.audio_rows(frames)+ audio_noise = torch.randn(+ audio_clean.shape, generator=generator, device=self.device, dtype=torch.float32+ )+ audio_t = self.shifted_timestep(generator, shift=3.0, device=self.device)+ video_noise = torch.randn(+ clean.shape, generator=generator, device=self.device, dtype=torch.float32+ )+ video_t = draw_time(generator)+ if is_sequence_parallel_enabled():+ for value in (anchor_noise, audio_noise, video_noise, video_t, audio_t):+ broadcast_sp_tensor(value)+ grid = self.latent_grid(batch)+ plan = self.observation_plan(generator, video_t, batch.known_degrees, grid, frames)+ use_anchor = has_anchor and (plan is None or plan.use_anchor)+ prompt, tags = self.prompt(batch, use_anchor=use_anchor)+ keep = None if plan is None else plan.keeps_observation+ if self.reads_observation:+ # The aligned observation reaches the model through the control trunk alone, and free+ # pictures through the text encoder's rows; only the other aligned observations (the+ # first frame) are packed beside the target.+ keep = trunk_keep(batch)+ specs, degrees, observation_rows = self.observation_rows(batch, generator, keep)+ layout = self.layout(+ tags,+ anchors=("first",) if use_anchor else (),+ observations=specs,+ grid=grid,+ frames=frames,+ )+ viewmats = K = frame_ids = None+ if has_camera:+ viewmats, K, frame_ids = self._camera(batch, layout)+ self._broadcast_logical_sample(viewmats, K)+ self._broadcast_logical_sample(clean, anchor)+ if plan is not None and not plan.use_camera:+ viewmats = K = frame_ids = None+ geometry = self.geometry_control(layout, self.geometry_latents(batch))+ if geometry is not None and plan is not None and not plan.use_control:+ geometry = dataclasses.replace(geometry, gain=0.0)+ if geometry is not None and self.reads_observation and optional_control_latents(batch) is None:+ geometry = dataclasses.replace(geometry, gain=0.0)+ anchor_t = torch.tensor(0.999, device=self.device, dtype=anchor.dtype)+ anchor_noisy = anchor_t * anchor + (1.0 - anchor_t) * anchor_noise+ parts = [patchify_video(anchor_noisy)] if use_anchor else []+ if observation_rows is not None:+ parts.append(observation_rows)+ return _Prepared(+ clean=clean,+ video_noise=video_noise,+ video_t=video_t,+ audio_t=audio_t,+ audio_rows=audio_t * audio_clean + (1.0 - audio_t) * audio_noise,+ condition_rows=parts,+ prompt=prompt,+ layout=layout,+ viewmats=viewmats,+ K=K,+ frame_ids=frame_ids,+ geometry=geometry,+ observation_degrees=degrees,+ plan=plan,+ )++ def forward_loss(self, batch: H3ArtifactBatch, *, noise_seed: int | None) -> Any:+ import torch++ generator = (+ None+ if noise_seed is None+ else torch.Generator(device=self.device).manual_seed(int(noise_seed))+ )+ inputs = self.prepare(+ batch,+ generator,+ lambda draw: self.shifted_timestep(draw, shift=12.0, device=self.device),+ )+ return self.loss_from_inputs(inputs)++ def loss_from_inputs(self, inputs: _Prepared, model: Any = None) -> Any:+ """The flow-matching loss of prepared inputs, on ``model`` (default: this core's)."""++ import torch+ import torch.nn.functional as F++ clean, video_noise, video_t, plan = (+ inputs.clean,+ inputs.video_noise,+ inputs.video_t,+ inputs.plan,+ )+ target_rows = patchify_video(clean - video_noise)+ text_t = row_t = None+ if plan is None:+ noisy_rows = patchify_video(video_t * clean + (1.0 - video_t) * video_noise)+ else:+ # Every target row sits at its own known degree; the target is unchanged.+ row_t = torch.from_numpy(plan.row_times).to(self.device)+ text_t = torch.tensor(plan.base_time, device=self.device)+ degree = row_t.view(1, -1, 1)+ noisy_rows = degree * patchify_video(clean) + (1.0 - degree) * patchify_video(+ video_noise+ )+ prediction = self._forward(+ video_rows=torch.cat((*inputs.condition_rows, noisy_rows), dim=1),+ audio_rows=inputs.audio_rows,+ prompt=inputs.prompt,+ layout=inputs.layout,+ video_t=video_t if plan is None else row_t,+ audio_t=inputs.audio_t,+ viewmats=inputs.viewmats,+ K=inputs.K,+ frame_ids=inputs.frame_ids,+ geometry_control=inputs.geometry,+ text_t=text_t,+ observation_degrees=inputs.observation_degrees,+ model=model,+ )+ if tuple(prediction.shape) != tuple(target_rows.shape):+ raise RuntimeError(+ f"H3 predicted rows {tuple(prediction.shape)} != {tuple(target_rows.shape)}"+ )+ if plan is None:+ return F.mse_loss(prediction.float(), target_rows.float())+ # Rows given as hard observations carry no recoverable noise and no loss.+ weight = torch.from_numpy(plan.loss_rows).to(self.device).view(1, -1, 1).float()+ error = (prediction.float() - target_rows.float()).square() * weight+ return error.sum() / (weight.sum() * prediction.shape[-1])++ def generate(+ self,+ batch: H3ArtifactBatch,+ *,+ noise_seed: int,+ num_inference_steps: int,+ observations: Any = None,+ renoise: bool = False,+ structure_until: float = 1.0,+ video_sigmas: Any = None,+ audio_sigmas: Any = None,+ ) -> Any:+ """Euler sampling; ``observations`` (H3InferenceObservations) makes every input optional.++ ``renoise`` samples as a DMD few-step student was trained to: each step takes the clean+ estimate and noises it afresh to the next time, rather than carrying the old noise on.++ ``video_sigmas`` (and ``audio_sigmas``) replace the shifted grid by a released adapter's own+ sigma list; ``num_inference_steps`` then counts its points.++ ``structure_until`` < 1 keeps the visual observations only for that leading fraction of+ the steps and samples the rest without them, exactly as a training sample whose+ observations were dropped: no rows, no layout entry. The early steps fix layout, motion+ and camera from the structure; the late ones, which shape detail, follow the prompt.+ """++ import dataclasses++ import torch++ if video_sigmas is not None:+ num_inference_steps = len(video_sigmas) # the given list's points (structure_until counts them)+ generator = torch.Generator(device=self.device).manual_seed(int(noise_seed))+ frames, grid = self.latent_frames(batch), self.latent_grid(batch)+ has_anchor, has_camera = batch.anchor_latents is not None, batch.camera_viewmats is not None+ anchor = (+ batch.anchor_latents.to(self.device, dtype=torch.float32, non_blocking=True).unsqueeze(+ 0+ )+ if has_anchor+ else torch.zeros((1, 24, 1, *grid), device=self.device)+ )+ use_anchor = has_anchor and (observations is None or observations.use_anchor)+ prompt, tags = self.prompt(batch, use_anchor=use_anchor)+ anchor_noise = torch.randn(+ anchor.shape, generator=generator, device=self.device, dtype=torch.float32+ )+ audio_clean = self.audio_rows(frames)+ audio_noise = torch.randn(+ audio_clean.shape, generator=generator, device=self.device, dtype=torch.float32+ )+ current = torch.randn(+ (1, 24, frames, *grid),+ generator=generator,+ device=self.device,+ dtype=torch.float32,+ )+ if is_sequence_parallel_enabled():+ for value in (anchor_noise, audio_noise, current):+ broadcast_sp_tensor(value)+ use_items = observations is None or observations.use_visual_observations+ if not use_items:+ keep = _keep_none+ elif self.reads_observation:+ keep = trunk_keep(batch)+ else:+ keep = None+ specs, degrees, observation_rows = self.observation_rows(batch, generator, keep)+ layout = self.layout(+ tags,+ anchors=("first",) if use_anchor else (),+ observations=specs,+ grid=grid,+ frames=frames,+ )+ viewmats = K = frame_ids = None+ if has_camera:+ viewmats, K, frame_ids = self._camera(batch, layout)+ self._broadcast_logical_sample(viewmats, K)+ self._broadcast_logical_sample(anchor)+ if observations is not None and not observations.use_camera:+ viewmats = K = frame_ids = None+ geometry = self.geometry_control(layout, self.geometry_latents(batch))+ if geometry is not None and observations is not None:+ geometry = dataclasses.replace(geometry, gain=float(observations.control_gain))+ if geometry is not None and self.reads_observation and optional_control_latents(batch) is None:+ geometry = dataclasses.replace(geometry, gain=0.0)+ anchor_rows = patchify_video(0.999 * anchor + 0.001 * anchor_noise)+ condition_rows = [anchor_rows] if use_anchor else []+ if observation_rows is not None:+ condition_rows.append(observation_rows)+ # The free tail: the same sample with its visual observations dropped, as training drops+ # them. Built only when asked for, so the default draws exactly what it always did.+ release_at = num_inference_steps+ free = None+ if structure_until < 1.0 and observation_rows is not None:+ if has_camera or geometry is not None:+ raise ValueError("releasing the structure is defined for samples with no camera or control trunk")+ release_at = max(0, int(round(float(structure_until) * num_inference_steps))) # 0: text only throughout+ free_specs, free_degrees, _ = self.observation_rows(batch, generator, _keep_none)+ free_layout = self.layout(+ tags,+ anchors=("first",) if use_anchor else (),+ observations=free_specs,+ grid=grid,+ frames=frames,+ )+ free = ([anchor_rows] if use_anchor else [], free_layout, free_degrees)+ fixed = held = degree = start = None+ if observations is not None and observations.latents is not None:+ rows = frames * (grid[0] // 2) * (grid[1] // 2)+ given = observations.latents.to(self.device, dtype=torch.float32).unsqueeze(0)+ if tuple(given.shape) != tuple(current.shape):+ raise ValueError("in-place observations must cover the target latent grid")++ def field(value: Any) -> Any:+ if value is None:+ return torch.zeros(rows, device=self.device)+ result = torch.as_tensor(value, dtype=torch.float32).to(self.device).reshape(-1)+ if result.numel() != rows or bool(((result < 0) | (result > 1)).any().item()):+ raise ValueError("known degrees cover every target row and lie in [0, 1]")+ return result++ degree, start = field(observations.row_degrees), field(observations.start_degrees)+ self._broadcast_logical_sample(degree, start, given)+ if bool((degree > 0).all().item()) or bool(((degree > 0) & (start > 0)).any().item()):+ raise ValueError("fixed rows must leave rows to generate and cannot also start")+ # Fixed rows hold ``d*y + (1-d)*noise`` for the whole trajectory; starting rows+ # enter at their degree and join the schedule once it reaches them.+ fixed = (degree > 0).view(1, -1, 1)+ level = torch.maximum(degree, start).view(1, -1, 1)+ entered = level * patchify_video(given) + (1.0 - level) * patchify_video(current)+ held = entered+ current = unpatchify_video(+ torch.where(level > 0, entered, patchify_video(current)), frames, *grid, channels=24+ )+ if self.flow_map and fixed is not None:+ raise ValueError("flow-map sampling moves every target row; in-place rows are not held")+ if free is not None and fixed is not None:+ raise ValueError("releasing the structure is defined for samples with no in-place rows")+ if renoise and (fixed is not None or self.flow_map):+ raise ValueError("fresh-noise sampling moves every target row along the plain schedule")+ if video_sigmas is not None:+ video_schedule = explicit_schedule(video_sigmas)+ audio_schedule = (+ explicit_schedule(audio_sigmas)+ if audio_sigmas is not None+ else make_shifted_schedule(len(video_schedule.sigmas), shift=3.0)+ )+ else:+ video_schedule = make_shifted_schedule(num_inference_steps, shift=12.0)+ audio_schedule = make_shifted_schedule(num_inference_steps, shift=3.0)+ with torch.no_grad():+ for index, (video_time, audio_time) in enumerate(+ zip(+ video_schedule.timesteps,+ audio_schedule.timesteps,+ strict=True,+ )+ ):+ video_t = torch.tensor(float(video_time), device=self.device)+ audio_t = torch.tensor(float(audio_time), device=self.device)+ audio_rows = audio_t * audio_clean + (1.0 - audio_t) * audio_noise+ target_rows = patchify_video(current)+ row_t = text_t = None+ if fixed is not None:+ target_rows = torch.where(fixed, held, target_rows)+ row_t = torch.where(degree > 0, degree, torch.clamp(start, min=video_t))+ text_t = video_t+ step_rows, step_layout, step_degrees = condition_rows, layout, degrees+ if free is not None and index >= release_at:+ step_rows, step_layout, step_degrees = free+ prediction_rows = self._forward(+ video_rows=torch.cat((*step_rows, target_rows), dim=1),+ audio_rows=audio_rows,+ prompt=prompt,+ layout=step_layout,+ video_t=video_t if row_t is None else row_t,+ audio_t=audio_t,+ viewmats=viewmats,+ K=K,+ frame_ids=frame_ids,+ geometry_control=geometry,+ text_t=text_t,+ observation_degrees=step_degrees,+ # A flow map jumps straight to the next schedule time; its average velocity+ # over the step takes the same Euler update below.+ video_r=torch.tensor(+ 1.0 - float(video_schedule.sigmas[index + 1]), device=self.device+ )+ if self.flow_map+ else None,+ )+ velocity = unpatchify_video(prediction_rows, frames, *grid, channels=24).float()+ if tuple(velocity.shape) != tuple(current.shape):+ raise RuntimeError(+ f"H3 inference velocity {tuple(velocity.shape)} != "+ f"sample {tuple(current.shape)}"+ )+ if not bool(torch.isfinite(velocity).all().item()):+ raise FloatingPointError("H3 Stage0.5 inference predicted non-finite velocity")+ sigma = float(video_schedule.sigmas[index])+ sigma_next = float(video_schedule.sigmas[index + 1])+ if sigma == 0:+ raise RuntimeError("H3 Euler schedule reached sigma zero early")+ if fixed is None and renoise:+ denoised = current + sigma * velocity+ fresh = torch.randn(+ current.shape, generator=generator, device=self.device, dtype=current.dtype+ )+ current = (1.0 - sigma_next) * denoised + sigma_next * fresh+ elif fixed is None:+ denoised = current + sigma * velocity+ ratio = sigma_next / sigma+ current = ratio * current + (1.0 - ratio) * denoised+ else:+ # Every row steps from its own level; a row ahead of the schedule waits.+ row_sigma = (1.0 - row_t).clamp_min(1e-6).view(1, -1, 1)+ ratio = (sigma_next / row_sigma).clamp(max=1.0)+ ratio = torch.where(fixed, torch.ones_like(ratio), ratio)+ rows_now = target_rows + (1.0 - ratio) * row_sigma * prediction_rows.float()+ current = unpatchify_video(rows_now, frames, *grid, channels=24)+ if fixed is not None:+ # An observed row returns its clean estimate x0 = x_d + (1-d) * v.+ estimate = held + (1.0 - degree.view(1, -1, 1)) * prediction_rows.float()+ current = unpatchify_video(+ torch.where(fixed, estimate, patchify_video(current)),+ frames,+ *grid,+ channels=24,+ )+ if not bool(torch.isfinite(current).all().item()):+ raise FloatingPointError("H3 inference generated non-finite latents")+ return current.contiguous()+++__all__ = [+ "H3Stage0p5Core",+ "H3TorchLayout",+ "aligned_control_latents",+ "batch_geometry_latents",+ "covers_every_frame",+ "optional_control_latents",+ "trunk_keep",+]diff --git a/src/solarwm/backends/minimax_h3/stage1.py b/src/solarwm/backends/minimax_h3/stage1.pynew file mode 100644index 0000000..109dc95--- /dev/null+++ b/src/solarwm/backends/minimax_h3/stage1.py@@ -0,0 +1,194 @@+"""Teacher-forced W6 AnyFlow v1.5 with H3's native data-ward velocity."""++from __future__ import annotations++from collections import OrderedDict+from typing import Any++from solarwm.training.anyflow import apply_timestep_shift, sample_anyflow_time_pairs++from .anyflow_loss import h3_anyflow_v15_loss+from .controlnet import teacher_forced_geometry_control+from .distributed import get_dp_group, get_dp_rank, get_dp_world_size+from .layout import build_stage1_layout, patchify_video+from .stage0p5 import H3Stage0p5Core+from .torch_layout import build_stage1_row_time_pairs+from .torch_mask import build_stage1_w6_block_mask+++class H3Stage1Core(H3Stage0p5Core):+ """Evaluate detached targets before the trainable AnyFlow prediction."""++ def __init__(+ self,+ model: Any,+ silence_latents: Any,+ device: Any,+ train: Any,+ validation_silence: Any = None,+ *,+ geometry: Any = None,+ ) -> None:+ super().__init__(model, silence_latents, device, geometry=geometry)+ self.train = train+ self.validation_silence = validation_silence+ self._masks: OrderedDict[tuple[int, int, int], Any] = OrderedDict()++ def layout(self, tags: Any) -> Any:+ key = tuple(tags.detach().cpu().tolist())+ if key not in self._layouts:+ self._layouts[key] = build_stage1_layout(+ key,+ 45,+ 48,+ 84,+ self.silence.shape[-1],+ ).to(self.device)+ while len(self._layouts) > 4:+ self._layouts.pop(next(iter(self._layouts)))+ return self._layouts[key]++ def _camera(self, batch: Any, layout: Any) -> tuple[Any, Any, Any]:+ import torch++ views = batch.camera_viewmats.to(self.device, torch.float32).unsqueeze(0)+ intrinsics = batch.camera_K.to(self.device, torch.float32).unsqueeze(0)+ if views.shape[1] == 47:+ return views[:, :45], intrinsics[:, :45], layout.camera_frame_ids+ if views.shape[1] != 47 * 1008:+ raise ValueError("Stage1 camera must describe the complete 47-latent artifact")+ views, intrinsics = views[:, : 45 * 1008], intrinsics[:, : 45 * 1008]+ return (+ torch.cat((views[:, :1008], views, views), dim=1),+ torch.cat((intrinsics[:, :1008], intrinsics, intrinsics), dim=1),+ None,+ )++ def forward_loss(self, batch: Any, *, noise_seed: int | None) -> Any:+ import torch++ generator = (+ None+ if noise_seed is None+ else torch.Generator(device=self.device).manual_seed(int(noise_seed))+ )+ clean = batch.target_latents.to(self.device, torch.float32).unsqueeze(0)+ anchor = batch.anchor_latents.to(self.device, torch.float32).unsqueeze(0)+ prompt = batch.prompt_embeds.to(self.device, torch.bfloat16).unsqueeze(0)+ tags = batch.text_token_tags.to(self.device, torch.long)+ self._broadcast_logical_sample(tags, clean, anchor, prompt)+ clean = clean[:, :, :45].contiguous()+ layout = self.layout(tags)+ views, intrinsics, frame_ids = self._camera(batch, layout)+ self._broadcast_logical_sample(views, intrinsics)+ # One chunk-local control payload serves all four AnyFlow evaluations.+ geometry = teacher_forced_geometry_control(+ layout, self.geometry_latents(batch), profile=self.geometry, frames=slice(0, 45)+ )+ extra = {} if geometry is None else {"geometry_control": geometry}++ def noise_like(value: Any) -> Any:+ return torch.randn(+ value.shape,+ device=self.device,+ dtype=torch.float32,+ generator=generator,+ )++ anchor_noise = noise_like(anchor)+ self._broadcast_logical_sample(anchor_noise)+ anchor_t = torch.tensor(0.999, device=self.device, dtype=anchor.dtype)+ anchor_rows = patchify_video(anchor_t * anchor + (1.0 - anchor_t) * anchor_noise)+ audio_clean = self.audio_rows()+ audio_noise = noise_like(audio_clean)+ audio_t = self.shifted_timestep(generator, shift=3.0, device=self.device)+ self._broadcast_logical_sample(audio_noise, audio_t)+ audio = audio_t * audio_clean + (1.0 - audio_t) * audio_noise+ noise = noise_like(clean)+ self._broadcast_logical_sample(noise)+ pairs = sample_anyflow_time_pairs(+ 1,+ logical_dp_rank=get_dp_rank(),+ logical_dp_world_size=get_dp_world_size(),+ diffusion_ratio=float(self.train["diffusion_ratio"]),+ consistency_ratio=float(self.train["consistency_ratio"]),+ generator=generator,+ device=self.device,+ )+ self._broadcast_logical_sample(*pairs)+ sigma_t = apply_timestep_shift(pairs.t, 12.0)+ sigma_r = apply_timestep_shift(pairs.r, 12.0)+ clean_rows, noise_rows = patchify_video(clean), patchify_video(noise)+ key = (layout.sequence_length, layout.rows_per_video_frame, layout.window_chunks)+ if key not in self._masks:+ self._masks[key] = build_stage1_w6_block_mask(+ layout,+ batch_size=1,+ num_heads=None,+ device=self.device,+ )+ while len(self._masks) > 4:+ self._masks.popitem(last=False)+ self._masks.move_to_end(key)+ mask = self._masks[key]++ def velocity(noisy: Any, video_t: Any, video_r: Any) -> Any:+ times, targets, indices = build_stage1_row_time_pairs(+ layout,+ video_t,+ video_r,+ audio_t,+ keyframe_timestep=0.999,+ device=self.device,+ )+ prediction, _ = self.model(+ hidden_states=torch.cat((anchor_rows, clean_rows, noisy), dim=1),+ audio_hidden_states=audio,+ encoder_hidden_states=prompt,+ timestep=times,+ r_timestep=targets,+ timestep_indices=indices,+ attention_mask=mask,+ fused_prope=True,+ prope_token_indices=layout.camera_video_indices,+ prope_frame_ids=frame_ids,+ cam_viewmats=views,+ cam_K=intrinsics,+ packed_sequence_parallel=self._sp_enabled(),+ return_dict=False,+ **layout.transformer_kwargs(),+ **extra,+ )+ return prediction[:, layout.noisy_video_output_slice]++ return h3_anyflow_v15_loss(+ velocity,+ clean_rows,+ noise_rows,+ sigma_t,+ sigma_r,+ pairs.is_diffusion,+ epsilon=float(self.train["finite_difference_epsilon"]),+ shift=12.0,+ process_group=get_dp_group(),+ )++ @staticmethod+ def _sp_enabled() -> bool:+ from .distributed import is_sequence_parallel_enabled++ return is_sequence_parallel_enabled()++ def generate(self, batch: Any, *, noise_seed: int, num_inference_steps: int) -> Any:+ from .stage1_sampling import generate_stage1++ if num_inference_steps != 4 or self.validation_silence is None:+ raise ValueError("H3 Stage1 validation needs NFE4 and the encoded 170-frame silence")+ return generate_stage1(+ self.model,+ batch,+ self.validation_silence,+ device=self.device,+ seed=noise_seed,+ geometry=self.geometry,+ )diff --git a/src/solarwm/backends/minimax_h3/stage1_sampling.py b/src/solarwm/backends/minimax_h3/stage1_sampling.pynew file mode 100644index 0000000..a0689a6--- /dev/null+++ b/src/solarwm/backends/minimax_h3/stage1_sampling.py@@ -0,0 +1,127 @@+"""Stage1's absolute-coordinate W6 sampler with four finite-map evaluations."""++from __future__ import annotations++from typing import Any++from .distributed import broadcast_sp_tensor, is_sequence_parallel_enabled+from .layout import build_stage1_layout, patchify_video, unpatchify_video+from .torch_flow import make_shifted_schedule, scale_noise+from .torch_geometry import native_video_position_grid+from .torch_layout import _unique_row_interval_plan+from .torch_mask import build_stage1_w6_block_mask+++def generate_stage1(+ model: Any, batch: Any, silence: Any, *, device: Any, seed: int, geometry: Any = None+) -> Any:+ import torch++ from .controlnet import pad_control_latents, teacher_forced_geometry_control+ from .stage0p5 import batch_geometry_latents++ if batch.rollout_camera_viewmats is None or batch.rollout_camera_K is None:+ raise ValueError("Stage1 validation requires the real 170-frame source camera trajectory")+ generator = torch.Generator(device=device).manual_seed(int(seed))+ # The encoded renders cover 47 latents; the 50-latent rollout repeats the+ # last one, the same tail policy as its cameras.+ control_latents = batch_geometry_latents(batch, geometry, device)+ if control_latents is not None:+ for value in control_latents.values():+ broadcast_sp_tensor(value)+ control_latents = {+ name: pad_control_latents(value, 50) for name, value in control_latents.items()+ }++ def noise_like(value: Any) -> Any:+ noise = torch.randn(value.shape, device=device, dtype=torch.float32, generator=generator)+ return broadcast_sp_tensor(noise)++ prompt = batch.prompt_embeds.unsqueeze(0).to(device, torch.bfloat16)+ anchor = batch.anchor_latents.unsqueeze(0).to(device, torch.float32)+ tags = batch.text_token_tags.to(device, torch.long)+ views = batch.rollout_camera_viewmats.to(device, torch.float32)+ intrinsics = batch.rollout_camera_K.to(device, torch.float32)+ for value in (prompt, anchor, tags, views, intrinsics):+ broadcast_sp_tensor(value)+ if views.shape != (50, 4, 4) or intrinsics.shape != (50, 3, 3):+ raise ValueError("Stage1 rollout cameras must describe all 50 latent positions")+ anchor_rows = patchify_video(scale_noise(anchor, noise_like(anchor), 0.999))+ audio_clean = silence.to(device, torch.float32).unsqueeze(0)+ audio_clean = audio_clean.permute(0, 1, 3, 2).reshape(1, -1, 32).contiguous()+ audio_noise = noise_like(audio_clean)+ sigmas = torch.linspace(1.0, 0.0, 5, device=device, dtype=torch.float32)+ audio_schedule = make_shifted_schedule(5, shift=3.0, device=device)+ positions = native_video_position_grid(50, 48, 84, origin=float(tags.numel())).to(device)+ generated = []+ masks = {}+ with torch.no_grad():+ for chunk in range(10):+ first = max(0, chunk - 5)+ frames = (chunk - first + 1) * 5+ layout = build_stage1_layout(tags.cpu(), frames, 48, 84, 283).to(device)+ selected_positions = positions[first * 5 * 1008 : (chunk + 1) * 5 * 1008]+ layout.position_ids.index_copy_(0, layout.clean_video_indices, selected_positions)+ layout.position_ids.index_copy_(0, layout.noisy_video_indices, selected_positions)+ if frames not in masks:+ masks[frames] = build_stage1_w6_block_mask(+ layout, batch_size=1, num_heads=None, device=device+ )+ placeholder = torch.zeros((1, 24, 5, 48, 84), device=device, dtype=torch.float32)+ clean = torch.cat((*generated[first:chunk], placeholder), dim=2)+ noisy = torch.zeros_like(clean)+ current = noise_like(placeholder)+ local_views = torch.cat((views[:1], views[first * 5 : (chunk + 1) * 5]), 0).unsqueeze(0)+ local_K = torch.cat(+ (intrinsics[:1], intrinsics[first * 5 : (chunk + 1) * 5]), 0+ ).unsqueeze(0)+ frame_ids = layout.camera_frame_ids.clone()+ frame_ids[layout.num_condition_video_rows :] += 1+ control = teacher_forced_geometry_control(+ layout,+ control_latents,+ profile=geometry,+ frames=slice(first * 5, (chunk + 1) * 5),+ )+ extra = {} if control is None else {"geometry_control": control}+ for step in range(4):+ noisy[:, :, -5:] = current+ audio_t = audio_schedule.timesteps[step]+ audio = scale_noise(audio_clean, audio_noise, audio_t)+ times, targets, indices = _unique_row_interval_plan(+ layout,+ current_local_chunk=chunk - first,+ video_timestep=1.0 - sigmas[step],+ video_r_timestep=1.0 - sigmas[step + 1],+ audio_timestep=audio_t,+ keyframe_timestep=0.999,+ device=device,+ )+ prediction, _ = model(+ hidden_states=torch.cat(+ (anchor_rows, patchify_video(clean), patchify_video(noisy)), 1+ ),+ audio_hidden_states=audio,+ encoder_hidden_states=prompt,+ timestep=times,+ r_timestep=targets,+ timestep_indices=indices,+ attention_mask=masks[frames],+ fused_prope=True,+ prope_token_indices=layout.camera_video_indices,+ prope_frame_ids=frame_ids,+ cam_viewmats=local_views,+ cam_K=local_K,+ packed_sequence_parallel=is_sequence_parallel_enabled(),+ return_dict=False,+ **layout.transformer_kwargs(),+ **extra,+ )+ velocity = unpatchify_video(+ prediction[:, layout.noisy_video_output_slice][:, -5 * 1008 :], 5, 48, 84+ )+ current = current + (sigmas[step] - sigmas[step + 1]) * velocity.float()+ if not bool(torch.isfinite(current).all()):+ raise FloatingPointError(f"non-finite H3 Stage1 validation chunk {chunk}")+ generated.append(current.contiguous())+ return torch.cat(generated, 2)[:, :, :47]diff --git a/src/solarwm/backends/minimax_h3/stage2.py b/src/solarwm/backends/minimax_h3/stage2.pynew file mode 100644index 0000000..37789c4--- /dev/null+++ b/src/solarwm/backends/minimax_h3/stage2.py@@ -0,0 +1,181 @@+"""H3 SGF conditioning, bidirectional scoring and causal generation."""++from __future__ import annotations++from typing import Any++from .distributed import broadcast_sp_tensor, is_sequence_parallel_enabled+from .layout import build_stage0p5_layout, patchify_video, unpatchify_video+from .sgf import h3_sgf_score_timesteps+from .sgf_rollout import H3SGFInputs, h3_sgf_rollout+from .stage0p5 import H3Stage0p5Core+from .torch_flow import sample_shifted_timestep, scale_noise+from .torch_layout import build_row_timesteps+++class H3SGFCore(H3Stage0p5Core):+ """Share the exact fixed-audio input construction between rollout and validation."""++ def rollout_geometry_latents(self, batch: Any, frames: int = 50) -> Any:+ """Encoded renders for every rollout latent, the tail repeating like cameras."""++ from .controlnet import pad_control_latents++ latents = self.geometry_latents(batch)+ if latents is None:+ return None+ return {name: pad_control_latents(value, frames) for name, value in latents.items()}++ def random_like(self, value: Any, *, generator: Any = None) -> Any:+ import torch++ noise = torch.randn(+ value.shape, device=value.device, dtype=value.dtype, generator=generator+ )+ return broadcast_sp_tensor(noise)++ def prepare_inputs(self, batch: Any, *, generator: Any = None) -> H3SGFInputs:+ import torch++ prompt = batch.prompt_embeds.unsqueeze(0).to(self.device, torch.bfloat16)+ tags = batch.text_token_tags.to(self.device, torch.long)+ anchor = batch.anchor_latents.unsqueeze(0).to(self.device, torch.float32)+ views = batch.camera_viewmats.unsqueeze(0).to(self.device, torch.float32)+ intrinsics = batch.camera_K.unsqueeze(0).to(self.device, torch.float32)+ self._broadcast_logical_sample(prompt, tags, anchor, views, intrinsics)+ if views.shape[1] == 47 * 1008:+ for value in (views, intrinsics):+ grouped = value.unflatten(1, (47, 1008))+ if not torch.equal(grouped, grouped[:, :, :1].expand_as(grouped)):+ raise ValueError("H3 SGF needs one camera per latent; spatial cameras differ")+ views, intrinsics = views[:, ::1008], intrinsics[:, ::1008]+ if views.shape[1] != 47 or intrinsics.shape[1] != 47:+ raise ValueError("H3 SGF requires the exact 47 source latent cameras")+ views = torch.cat((views[:, :1], views, views[:, -1:].expand(-1, 3, -1, -1)), 1)+ intrinsics = torch.cat(+ (intrinsics[:, :1], intrinsics, intrinsics[:, -1:].expand(-1, 3, -1, -1)), 1+ )+ anchor = scale_noise(anchor, self.random_like(anchor, generator=generator), 0.999)+ audio = self.audio_rows()+ audio_time = sample_shifted_timestep(+ 1,+ 3.0,+ self.device,+ torch.float32,+ generator=generator,+ ).reshape(())+ broadcast_sp_tensor(audio_time)+ audio = scale_noise(audio, self.random_like(audio, generator=generator), audio_time)+ return H3SGFInputs(+ prompt,+ tags,+ patchify_video(anchor),+ audio,+ audio_time,+ views,+ intrinsics,+ 48,+ 84,+ 0.999,+ is_sequence_parallel_enabled(),+ geometry=self.geometry,+ geometry_latents=self.rollout_geometry_latents(batch),+ )++ def rollout(+ self,+ inputs: H3SGFInputs,+ *,+ generator: Any = None,+ exit_index: int | None = None,+ progress: Any = None,+ ) -> Any:+ import torch++ noise = torch.empty((1, 24, 50, 48, 84), device=self.device, dtype=torch.float32)+ return h3_sgf_rollout(+ student=self.model,+ inputs=inputs,+ noise=self.random_like(noise, generator=generator),+ exit_index=exit_index,+ generator=generator,+ progress=progress,+ )++ def score_inputs(self, clean: Any) -> tuple[Any, ...]:+ video_time, audio_time = h3_sgf_score_timesteps(device=self.device)+ self._broadcast_logical_sample(video_time, audio_time)+ noise = self.random_like(clean)+ noisy = scale_noise(clean, noise, video_time)+ silence = self.audio_rows()+ audio = scale_noise(silence, self.random_like(silence), audio_time)+ return noisy, noise, video_time, audio, audio_time++ def score_forward(+ self,+ model: Any,+ inputs: H3SGFInputs,+ noisy: Any,+ video_time: Any,+ audio: Any,+ audio_time: Any,+ ) -> Any:+ import torch++ layout = build_stage0p5_layout(inputs.text_tags.cpu(), 47, 48, 84, audio.shape[1] // 2).to(+ self.device+ )+ times, indices = build_row_timesteps(+ layout,+ video_time,+ audio_time,+ condition_video_timestep=0.999,+ device=self.device,+ )+ frame_ids = layout.camera_frame_ids.clone()+ frame_ids[layout.num_condition_video_rows :] += 1+ extra = {}+ if inputs.geometry is not None:+ from .controlnet import geometry_control_for_layout++ # Teacher and critic score the whole clip bidirectionally, like Stage0.5.+ def patches(name: str) -> Any:+ return patchify_video(inputs.geometry_latents[name][:, :, :47])++ extra["geometry_control"] = geometry_control_for_layout(+ layout,+ {name: patches(name) for name in inputs.geometry.modalities},+ profile=inputs.geometry,+ wide_rows=(+ {name: patches(f"{name}_wide") for name in inputs.geometry.wide_modalities}+ if inputs.geometry.wide_enabled+ else None+ ),+ )+ velocity, _ = model(+ hidden_states=torch.cat((inputs.anchor_rows, patchify_video(noisy)), 1),+ audio_hidden_states=audio,+ encoder_hidden_states=inputs.prompt,+ timestep=times,+ timestep_indices=indices,+ fused_prope=True,+ prope_token_indices=layout.camera_video_indices,+ prope_frame_ids=frame_ids,+ cam_viewmats=inputs.camera_viewmats[:, :48],+ cam_K=inputs.camera_K[:, :48],+ packed_sequence_parallel=is_sequence_parallel_enabled(),+ return_dict=False,+ **layout.transformer_kwargs(),+ **extra,+ )+ return unpatchify_video(velocity[:, layout.noisy_video_output_slice], 47, 48, 84)++ def generate(self, batch: Any, *, noise_seed: int, num_inference_steps: int) -> Any:+ import torch++ if num_inference_steps != 4:+ raise ValueError("H3 SGF validation requires four denoiser evaluations")+ generator = torch.Generator(device=self.device).manual_seed(int(noise_seed))+ with torch.no_grad():+ inputs = self.prepare_inputs(batch, generator=generator)+ return self.rollout(inputs, generator=generator, exit_index=3).cache_target[:, :, :47]diff --git a/src/solarwm/backends/minimax_h3/stage2_runtime.py b/src/solarwm/backends/minimax_h3/stage2_runtime.pynew file mode 100644index 0000000..104ed00--- /dev/null+++ b/src/solarwm/backends/minimax_h3/stage2_runtime.py@@ -0,0 +1,266 @@+"""Three-role H3 SGF updates using the public checkpoint and validation lifecycle."""++from __future__ import annotations++import time+from dataclasses import replace+from pathlib import Path+from typing import Any++from solarwm.training import JsonlEventSink, StepPolicy+from solarwm.training.sgf import compute_sgf_kl_gradient, sgf_student_loss, should_update_student++from .distributed import get_sp_size, sync_lora_gradients+from .runtime import H3TrainingRuntime, _base_model_load_receipt+from .sgf import h3_sgf_critic_loss+from .sgf_rollout import h3_sgf_replay+from .stage2 import H3SGFCore+from .torch_flow import predict_clean_sample+++class H3SGFTrainingRuntime(H3TrainingRuntime):+ def __init__(self, config: Any) -> None:+ super().__init__(config, defer_resume=True)+ self.initialization = {"student": self._weights_id}+ torch = self.torch+ cpu_rng, cuda_rng = torch.get_rng_state(), torch.cuda.get_rng_state(self.device)+ try:+ self.teacher, _ = self._build_score_role(trainable=False)+ critic, critic_lora = self._build_score_role(trainable=True)+ finally:+ torch.set_rng_state(cpu_rng)+ torch.cuda.set_rng_state(cuda_rng, self.device)+ from .optimizer import build_adapter_optimizer++ optimizer = build_adapter_optimizer(+ critic_lora,+ self.train_cfg["critic_optimizer"],+ self.train_cfg.get("critic_geometry_controlnet_optimizer"),+ )+ scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda _: 1.0)+ self.extra_roles["critic"] = {+ "model": critic,+ "lora": critic_lora,+ "optimizer": optimizer,+ "scheduler": scheduler,+ }+ self.model.eval()+ self.contract = replace(+ self.contract,+ extras={**self.contract.extras, "initialization": dict(self.initialization)},+ )+ resume = str(self.checkpoint_cfg.get("resume_from") or "")+ if resume:+ self.load_checkpoint(resume)+ self._check_ema_boundary()++ def _make_core(self) -> H3SGFCore:+ return H3SGFCore(self.model, self.silence, self.device, geometry=self.geometry)++ def _build_score_role(self, *, trainable: bool) -> tuple[Any, Any]:+ from .fsdp import wrap_h3_fsdp+ from .lora import inject_h3_lora+ from .optional import load_transformer+ from .weights import load_initial_weights++ seed = 0x48330000 + int(self.config["data"]["seed"])+ self.torch.manual_seed(seed)+ self.torch.cuda.manual_seed_all(seed)+ modules = load_transformer(self.model_cfg, device=self.device)+ modules.transformer.requires_grad_(False)+ if self.geometry is not None:+ modules.transformer.geometry_controlnet.activation_checkpointing = trainable+ model, lora = inject_h3_lora(+ modules.transformer,+ self.model_cfg["adapter"],+ base_identity=_base_model_load_receipt(self.model_cfg, modules.transformer),+ )+ model = wrap_h3_fsdp(+ model,+ local_rank=self.topology.local_rank,+ transformer_block_cls=modules.transformer_block_cls,+ fp32_units=modules.fp32_fsdp_units,+ ignored_parameters=lora.parameters,+ activation_checkpointing=trainable,+ frozen_base_shard_size=8,+ )+ role = "critic" if trainable else "teacher"+ identity = load_initial_weights(self.checkpoint_cfg["initialization"][role], lora)+ self.initialization[role] = identity+ if self.is_main:+ print(f"[h3-sgf-init] {role}={identity}", flush=True)+ if not trainable:+ model.requires_grad_(False)+ return model.eval(), lora++ def _check_ema_boundary(self) -> None:+ expected = self.student_step >= int(self.checkpoint_cfg["ema"]["start_step"])+ if expected != (self.ema is not None):+ raise ValueError("H3 SGF EMA presence differs from the saved student-update boundary")++ def save_checkpoint(self, step: int) -> str:+ # The replay leaves large inactive allocations. Release them before+ # checkpoint collectives, retaining live parameters, optimizer and EMA.+ self.torch.cuda.synchronize(self.device)+ self.torch.cuda.empty_cache()+ return super().save_checkpoint(step)++ def _student_ema_update(self) -> None:+ from .ema import H3ShardedEMA++ cfg = self.checkpoint_cfg["ema"]+ if self.ema is None and self.student_step >= int(cfg["start_step"]):+ self.ema = H3ShardedEMA(self.model, decay=float(cfg["decay"]), device=self.device)+ if self.is_main:+ print(f"[h3-sgf-ema] created student_step={self.student_step}", flush=True)+ if self.ema is not None:+ self.ema.update(self.model)++ def _finish_update(self, model: Any, lora: Any, optimizer: Any, scheduler: Any) -> float:+ sync_lora_gradients(lora.trainable_parameters)+ finite, norm = self._finite_clip_norm(+ (parameter for parameter in model.parameters() if parameter.requires_grad),+ 10.0,+ )+ if not finite:+ raise FloatingPointError(f"non-finite H3 SGF gradient norm={norm}")+ optimizer.step()+ scheduler.step()+ optimizer.zero_grad(set_to_none=True)+ return norm++ def _phase(self, name: str) -> None:+ if self.is_main:+ print(+ f"[h3-sgf-phase] outer_step={self.global_step + 1} {name} "+ f"elapsed_s={time.monotonic() - self._step_started:.1f}",+ flush=True,+ )++ def _rollout(self, inputs: Any) -> Any:+ return self.core.rollout(+ inputs, progress=lambda chunk: self._phase(f"rollout_chunk={chunk}/10")+ )++ def train_outer_step(self) -> dict[str, float]:+ torch, core = self.torch, self.core+ self._step_started = time.monotonic()+ critic = self.extra_roles["critic"]+ update_student = should_update_student(self.global_step, 5)+ stats = {+ "student_updated": float(update_student),+ "loss_student": 0.0,+ "student_grad_norm": 0.0,+ }+ if update_student:+ self._phase("student_rollout")+ inputs = core.prepare_inputs(self._next_batch())+ rollout = self._rollout(inputs)+ self._phase("student_replay")+ output = h3_sgf_replay(student=self.model, inputs=inputs, rollout=rollout)[:, :, :47]+ with torch.no_grad():+ self._phase("student_scores")+ noisy, noise, vt, audio, at = core.score_inputs(output.detach())+ fake_velocity = core.score_forward(critic["model"], inputs, noisy, vt, audio, at)+ real_velocity = core.score_forward(self.teacher, inputs, noisy, vt, audio, at)+ fake = predict_clean_sample(noisy, fake_velocity, vt)+ real = predict_clean_sample(noisy, real_velocity, vt)+ if not bool(torch.isfinite(fake).all() and torch.isfinite(real).all()):+ raise FloatingPointError("non-finite H3 SGF score output")+ gradient = compute_sgf_kl_gradient(+ fake_x0=fake, real_x0=real, student_output=output.detach()+ )+ loss = sgf_student_loss(output, gradient)+ self._phase("student_backward")+ (loss / get_sp_size()).backward()+ stats["loss_student"] = float(loss.detach())+ stats["student_grad_norm"] = self._finish_update(+ self.model, self.lora, self.optimizer, self.scheduler+ )+ self.student_step += 1+ self._student_ema_update()+ del (+ inputs,+ rollout,+ output,+ noisy,+ noise,+ audio,+ fake_velocity,+ real_velocity,+ fake,+ real,+ gradient,+ loss,+ )+ inputs = core.prepare_inputs(self._next_batch())+ self._phase("critic_rollout")+ rollout = self._rollout(inputs)+ self._phase("critic_replay")+ with torch.no_grad():+ clean = h3_sgf_replay(student=self.model, inputs=inputs, rollout=rollout)[:, :, :47]+ noisy, noise, vt, audio, at = core.score_inputs(clean)+ self._phase("critic_forward")+ velocity = core.score_forward(critic["model"], inputs, noisy, vt, audio, at)+ loss = h3_sgf_critic_loss(velocity, noise=noise, clean=clean)+ if not bool(torch.isfinite(loss)):+ raise FloatingPointError("non-finite H3 SGF critic loss")+ self._phase("critic_backward")+ (loss / get_sp_size()).backward()+ stats["critic_grad_norm"] = self._finish_update(+ critic["model"],+ critic["lora"],+ critic["optimizer"],+ critic["scheduler"],+ )+ self._global_step += 1+ stats.update(+ loss_critic=float(loss.detach()),+ student_step=float(self.student_step),+ loss=stats["loss_student"] if update_student else float(loss.detach()),+ lr=float(self.optimizer.param_groups[0]["lr"]),+ critic_lr=float(critic["optimizer"].param_groups[0]["lr"]),+ compute_time_s=time.monotonic() - self._step_started,+ )+ if self.is_main:+ print(+ f"[h3-sgf] outer_step={self.global_step} student_step={self.student_step} "+ f"student_loss={stats['loss_student']:.6f} critic_loss={stats['loss_critic']:.6f} "+ f"compute_time_s={stats['compute_time_s']:.2f} "+ f"ema={'present' if self.ema is not None else 'not_started'}",+ flush=True,+ )+ return stats+++def run_sgf_training(config: Any) -> int:+ runtime = H3SGFTrainingRuntime(config)+ policy = StepPolicy(+ max_steps=int(config["train"]["max_steps"]),+ save_every=int(config["checkpoint"]["save_every_steps"]),+ save_steps=tuple(config["checkpoint"].get("save_steps", ())),+ validate_every=int(config["validation"]["validate_every_steps"]),+ validation_steps=(int(config["validation"]["smoke_step"]),)+ if int(config["validation"]["smoke_step"]) > 0+ else (),+ )+ sink = (+ JsonlEventSink(Path(str(config["runtime"]["output_dir"])) / "training-events.jsonl")+ if runtime.is_main+ else None+ )+ try:+ while runtime.global_step < policy.max_steps:+ stats = runtime.train_outer_step()+ step = runtime.global_step+ if sink is not None:+ sink({"event": "train_step", "step": step, **stats})+ if policy.should_save(step):+ runtime.save_checkpoint(step)+ if policy.should_validate(step):+ report = runtime.validate(step)+ if sink is not None:+ sink({"event": "validation", "step": step, "report": report})+ finally:+ runtime.reader.close()+ return 0diff --git a/src/solarwm/backends/minimax_h3/stream_decode.py b/src/solarwm/backends/minimax_h3/stream_decode.pynew file mode 100644index 0000000..6cd6c10--- /dev/null+++ b/src/solarwm/backends/minimax_h3/stream_decode.py@@ -0,0 +1,79 @@+"""Bounded-memory form of the official H3 temporal decoder.++Keep the official spatial tiles, temporal clips and overlap arithmetic. Only+the collection of decoded clips changes: completed frames are yielded rather+than retaining the whole pixel video on the GPU.+"""++from __future__ import annotations+++def decoded_chunks(vae, normalized_latents, *, device):+ import torch++ z = normalized_latents+ size, drop, ratio = vae.tokens_chunk_size, vae.config.token_drop, vae.temporal_compression_ratio+ tokens = z.shape[2] + drop+ padding = (-tokens) % size+ count = (tokens + padding) // size - int(drop > 0)+ if count < 1:+ raise ValueError("H3 streaming decoder needs at least one complete temporal clip")+ mean = torch.tensor(vae.config.latents_mean, device=device).view(1, -1, 1, 1, 1)+ std = torch.tensor(vae.config.latents_std, device=device).view(1, -1, 1, 1, 1)+ intra_tail = vae.config.clip_length % ratio+ trim = sum(+ intra_tail if intra_tail and (z.shape[2] + k) % size == 0 else ratio for k in range(padding)+ )+ overlap = None+ emitted = 0+ pending_cpu = None++ def completed(chunk):+ # Padding can remove frames from more than the last overlap chunk.+ # Withhold exactly that tail across chunks, like slicing the official+ # concatenated output, while keeping only a bounded CPU frame buffer.+ nonlocal pending_cpu+ chunk = chunk.cpu()+ if pending_cpu is not None:+ chunk = torch.cat((pending_cpu, chunk), dim=2)+ keep = min(trim, chunk.shape[2])+ if chunk.shape[2] > keep:+ yield chunk[:, :, :-keep] if keep else chunk+ pending_cpu = chunk[:, :, -keep:] if keep else None++ for index in range(count):+ start = index * size+ # The official decoder slices after padding only to a chunk boundary;+ # its final clip can be shorter than size + token_overlap.+ stop = min(index * size + size + vae.token_overlap, z.shape[2] + padding)+ part = z[:, :, start : min(stop, z.shape[2])].to(device).float()+ if stop > z.shape[2]:+ part = torch.cat((part, part[:, :, -1:].repeat(1, 1, stop - z.shape[2], 1, 1)), dim=2)+ with (+ torch.no_grad(),+ torch.autocast(+ device_type=str(device).split(":")[0],+ dtype=torch.float16,+ enabled=str(device).startswith("cuda"),+ ),+ ):+ clip = vae._decode_clip(part * std + mean)+ for j in range(int(drop > 0) + 1):+ chunk = clip[:, :, j * size * ratio : (j + 1) * size * ratio][+ :, :, vae.frame_pre_padding :+ ]+ if j == 0:+ if overlap is not None:+ chunk = vae._blend(overlap, chunk, vae.frame_overlap, dim=-3)+ for ready in completed(chunk):+ emitted += ready.shape[2]+ yield ready+ else:+ overlap = chunk+ del clip, part+ if overlap is not None:+ for ready in completed(overlap):+ emitted += ready.shape[2]+ yield ready+ if emitted < 1:+ raise ValueError("H3 temporal decode produced no frames")diff --git a/src/solarwm/backends/minimax_h3/torch_flow.py b/src/solarwm/backends/minimax_h3/torch_flow.pynew file mode 100644index 0000000..981613e--- /dev/null+++ b/src/solarwm/backends/minimax_h3/torch_flow.py@@ -0,0 +1,147 @@+"""MiniMax-H3 rectified-flow convention and shifted sigma schedules."""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any+++@dataclass(frozen=True)+class H3FlowSchedule:+ """A shifted H3 sigma grid and its model timesteps (terminal sigma excluded)."""++ sigmas: Any+ timesteps: Any+++def _broadcast_time(value: Any, sample: Any) -> Any:+ import torch++ value = torch.as_tensor(value, device=sample.device, dtype=sample.dtype)+ while value.ndim < sample.ndim:+ value = value.unsqueeze(-1)+ return value+++def scale_noise(clean_sample: Any, noise: Any, timestep: Any) -> Any:+ """Apply H3 forward flow: ``x_t=t*x0+(1-t)*noise``; ``t=1`` is clean."""++ if clean_sample.shape != noise.shape:+ raise ValueError(+ f"clean/noise shapes must match, got {clean_sample.shape} and {noise.shape}"+ )+ t = _broadcast_time(timestep, clean_sample)+ return t * clean_sample + (1.0 - t) * noise+++def add_noise(clean: Any, noise: Any, t: Any) -> Any:+ """Trainer-facing alias for :func:`scale_noise`."""++ return scale_noise(clean, noise, t)+++def data_velocity_target(clean_sample: Any, noise: Any) -> Any:+ """Return H3's data-ward velocity target ``x0-noise``."""++ if clean_sample.shape != noise.shape:+ raise ValueError(+ f"clean/noise shapes must match, got {clean_sample.shape} and {noise.shape}"+ )+ return clean_sample - noise+++def velocity_target(clean: Any, noise: Any) -> Any:+ """Trainer-facing alias returning H3's ``clean-noise`` target."""++ return data_velocity_target(clean, noise)+++def sample_shifted_timestep(+ batch: int | tuple[int, ...],+ shift: float,+ device: Any,+ dtype: Any,+ generator: Any = None,+) -> Any:+ """Sample training timesteps from a uniformly sampled, shifted sigma.++ The random variable is a base ``sigma ~ U(0,1)``. H3's exponential+ shift is applied to sigma and the transformer convention is returned as+ ``t = 1-sigma_shifted``.+ """++ if shift <= 0:+ raise ValueError(f"shift must be positive, got {shift}")+ import torch++ shape = (batch,) if isinstance(batch, int) else tuple(batch)+ if not shape or any(int(size) <= 0 for size in shape):+ raise ValueError(f"batch must describe a positive shape, got {batch!r}")+ base_sigma = torch.rand(shape, device=device, dtype=dtype, generator=generator)+ return 1.0 - shift_sigma(base_sigma, shift)+++def predict_clean_sample(noisy_sample: Any, velocity: Any, timestep: Any) -> Any:+ """Recover ``x0 = x_t + (1-t)*v`` from H3 data-ward velocity."""++ sigma = 1.0 - _broadcast_time(timestep, noisy_sample)+ return noisy_sample + sigma * velocity+++def shift_sigma(sigma: Any, shift: float) -> Any:+ """Apply H3's exponential shift ``s*sigma/(1+(s-1)*sigma)``."""++ if shift <= 0:+ raise ValueError(f"shift must be positive, got {shift}")+ return shift * sigma / (1.0 + (shift - 1.0) * sigma)+++def make_shifted_schedule(+ num_inference_steps: int,+ *,+ shift: float = 12.0,+ device: Any = None,+) -> H3FlowSchedule:+ """Build the exact H3 grid, including terminal sigma zero.++ ``num_inference_steps`` names grid points in upstream H3, therefore the+ transformer runs ``len(sigmas)-1`` evaluations.+ """++ if num_inference_steps < 2:+ raise ValueError("num_inference_steps must be at least 2")+ import torch++ base = torch.linspace(1.0, 0.0, int(num_inference_steps), dtype=torch.float32)+ sigmas = torch.unique_consecutive(shift_sigma(base, shift)).to(device=device)+ return H3FlowSchedule(sigmas=sigmas, timesteps=(1.0 - sigmas[:-1]))+++def euler_step(sample: Any, velocity: Any, timestep: Any, sigma: Any, sigma_next: Any) -> Any:+ """Take one deterministic H3 Euler step in float32 for half inputs."""++ import torch++ denoised = predict_clean_sample(sample, velocity, timestep)+ compute_dtype = (+ torch.float32 if sample.dtype in (torch.float16, torch.bfloat16) else sample.dtype+ )+ sigma = torch.as_tensor(sigma, device=sample.device, dtype=compute_dtype)+ sigma_next = torch.as_tensor(sigma_next, device=sample.device, dtype=compute_dtype)+ ratio = sigma_next / sigma+ result = ratio * sample.to(compute_dtype) + (1.0 - ratio) * denoised.to(compute_dtype)+ return result.to(sample.dtype)+++__all__ = [+ "H3FlowSchedule",+ "add_noise",+ "data_velocity_target",+ "euler_step",+ "make_shifted_schedule",+ "predict_clean_sample",+ "sample_shifted_timestep",+ "scale_noise",+ "shift_sigma",+ "velocity_target",+]diff --git a/src/solarwm/backends/minimax_h3/torch_geometry.py b/src/solarwm/backends/minimax_h3/torch_geometry.pynew file mode 100644index 0000000..0ea932b--- /dev/null+++ b/src/solarwm/backends/minimax_h3/torch_geometry.py@@ -0,0 +1,84 @@+"""Native torch position grids used by causal H3 execution."""++from __future__ import annotations++import math+from typing import Any++from .geometry import DEFAULT_GEOMETRY+++def spatial_position_axis(dim: int, patch: int, sqrt_area: float) -> Any:+ """Build one upstream-compatible float64, endpoint-excluded spatial axis."""++ if dim <= 0 or patch <= 0 or dim % patch:+ raise ValueError(f"dim={dim} must be positive and divisible by patch={patch}")+ import numpy as np+ import torch++ ratio = dim / sqrt_area+ left = (1.0 - ratio) / 2.0+ values = np.linspace(left, left + ratio, dim // patch, endpoint=False) * 32+ return torch.from_numpy(values).to(torch.float64)+++def frame_position_grid(+ latent_height: int,+ latent_width: int,+ patch_h: int = 2,+ patch_w: int = 2,+) -> tuple[Any, Any]:+ """Return flattened ``(h,w)`` coordinates and the width axis for one frame."""++ import torch++ sqrt_area = math.sqrt(latent_height * latent_width)+ height_grid = spatial_position_axis(latent_height, patch_h, sqrt_area)+ width_grid = spatial_position_axis(latent_width, patch_w, sqrt_area)+ grids = torch.meshgrid(height_grid, width_grid, indexing="ij")+ return torch.stack([grid.reshape(-1) for grid in grids], dim=-1), width_grid+++def temporal_position_grid(num_latent_frames: int, origin: float = 0.0) -> Any:+ """Return H3's non-uniform ``5/3*(1,4,4,4,4)`` rotary-time grid."""++ if num_latent_frames < 0:+ raise ValueError(f"num_latent_frames must be non-negative, got {num_latent_frames}")+ import torch++ spans = torch.tensor(+ [+ DEFAULT_GEOMETRY.rope_frame_rescale+ * DEFAULT_GEOMETRY.rope_frames_per_latent[+ i % len(DEFAULT_GEOMETRY.rope_frames_per_latent)+ ]+ for i in range(num_latent_frames)+ ],+ dtype=torch.float64,+ )+ if num_latent_frames == 0:+ return spans+ return float(origin) + torch.cat((torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)))+++def native_video_position_grid(+ num_latent_frames: int,+ latent_height: int,+ latent_width: int,+ *,+ patch_size: tuple[int, int, int] = (1, 2, 2),+ origin: float = 0.0,+) -> Any:+ """Return flattened native H3 ``(t,h,w)`` IDs in frame-major row order."""++ import torch++ patch_t, patch_h, patch_w = patch_size+ if patch_t != 1:+ raise ValueError("released H3 MM-RoPE layout requires temporal patch_size=1")+ frame_grid, _ = frame_position_grid(latent_height, latent_width, patch_h, patch_w)+ times = temporal_position_grid(num_latent_frames, origin)+ result = torch.empty(num_latent_frames, frame_grid.shape[0], 3, dtype=torch.float64)+ result[:, :, 0] = times[:, None]+ result[:, :, 1:] = frame_grid[None]+ return result.reshape(-1, 3)diff --git a/src/solarwm/backends/minimax_h3/torch_layout.py b/src/solarwm/backends/minimax_h3/torch_layout.pynew file mode 100644index 0000000..89333f3--- /dev/null+++ b/src/solarwm/backends/minimax_h3/torch_layout.py@@ -0,0 +1,190 @@+"""Torch timestep grouping over the shared public H3 row layout."""++from __future__ import annotations++from typing import Any++import torch++from .layout import H3PackedLayout+++def build_row_timesteps(+ layout: H3PackedLayout,+ video_timestep: Any,+ audio_timestep: Any,+ *,+ text_timestep: Any | None = None,+ condition_video_timestep: Any = 0.999,+ clean_video_timestep: Any = 1.0,+ device: Any = None,+) -> tuple[Any, Any]:+ """Build H3's ``(distinct_timesteps, row_to_timestep_index)`` pair.++ Stage0.5 requires a scalar ``video_timestep``. Stage1 accepts a scalar, one+ value per chunk, or one value per latent frame; only noisy-video rows use+ those sampled values. Clean rows use ``t=1``, keyframe anchors use their+ fixed augmentation level, and audio uses an independent scalar shift-3+ sample. ``text_timestep`` defaults to video time in Stage0.5 and clean+ ``t=1`` in Stage1.+ """++ import torch++ def tensor(value: Any) -> Any:+ return torch.as_tensor(value, dtype=torch.float32, device=device).flatten()++ def scalar(value: Any, name: str) -> Any:+ tensor = torch.as_tensor(value, dtype=torch.float32, device=device)+ if tensor.numel() != 1:+ raise ValueError(f"{name} must be scalar, got {tuple(tensor.shape)}")+ return tensor.reshape(())++ video_values = tensor(video_timestep)+ if layout.stage == "stage0p5":+ if video_values.numel() != 1:+ raise ValueError("Stage0.5 video_timestep must be scalar")+ noisy_row_values = video_values.expand(layout.noisy_video_indices.numel())+ default_text = video_values[0]+ elif layout.stage == "stage1":+ num_frames = layout.noisy_video_indices.numel() // layout.rows_per_video_frame+ num_chunks = num_frames // int(layout.chunk_latent_frames)+ if video_values.numel() == 1:+ frame_values = video_values.expand(num_frames)+ elif video_values.numel() == num_chunks:+ frame_values = video_values.repeat_interleave(int(layout.chunk_latent_frames))+ elif video_values.numel() == num_frames:+ frame_values = video_values+ else:+ raise ValueError(+ "Stage1 video_timestep must be scalar, [num_chunks] or [num_latent_frames]; "+ f"got {video_values.numel()} values for {num_chunks} chunks/{num_frames} frames"+ )+ noisy_row_values = frame_values.repeat_interleave(layout.rows_per_video_frame)+ default_text = torch.tensor(1.0, dtype=torch.float32, device=device)+ else:+ raise ValueError(f"unsupported layout stage {layout.stage!r}")++ row_timesteps = torch.zeros(layout.sequence_length, dtype=torch.float32, device=device)+ row_timesteps[layout.text_indices.to(device=device)] = scalar(+ default_text if text_timestep is None else text_timestep, "text_timestep"+ )+ row_timesteps[layout.condition_video_indices.to(device=device)] = scalar(+ condition_video_timestep, "condition_video_timestep"+ )+ row_timesteps[layout.audio_indices.to(device=device)] = scalar(audio_timestep, "audio_timestep")+ if int(layout.clean_video_indices.numel()):+ row_timesteps[layout.clean_video_indices.to(device=device)] = scalar(+ clean_video_timestep, "clean_video_timestep"+ )+ row_timesteps[layout.noisy_video_indices.to(device=device)] = noisy_row_values+ return torch.unique(row_timesteps, sorted=True, return_inverse=True)+++def _unique_timestep_plan(row_timesteps: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:+ if row_timesteps.ndim != 1:+ raise ValueError("row_timesteps must be one-dimensional")+ return torch.unique(row_timesteps.float(), sorted=True, return_inverse=True)+++def build_stage1_row_timesteps(+ layout: Any,+ video_chunk_timesteps: torch.Tensor,+ audio_timestep: torch.Tensor,+ *,+ keyframe_timestep: float = 0.999,+ device: torch.device,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Assign per-chunk Stage1 timesteps to the explicit clean/noisy layout."""++ chunks = video_chunk_timesteps.reshape(-1).to(device=device, dtype=torch.float32)+ expected_chunks = int(layout.target_video_chunk_ids.max().item()) + 1+ if chunks.numel() != expected_chunks:+ raise ValueError(+ f"expected {expected_chunks} Stage1 video chunk timesteps, got {chunks.numel()}"+ )+ rows = torch.ones(layout.sequence_length, dtype=torch.float32, device=device)+ rows[layout.condition_video_indices] = float(keyframe_timestep)+ rows[layout.audio_indices] = audio_timestep.reshape(())+ noisy_chunks = layout.target_video_chunk_ids.index_select(0, layout.noisy_video_indices)+ rows[layout.noisy_video_indices] = chunks.index_select(0, noisy_chunks)+ # Text and clean teacher-forcing rows remain clean conditions at t=1.+ return _unique_timestep_plan(rows)+++def build_stage1_row_time_pairs(layout, video_t, video_r, audio_t, *, keyframe_timestep, device):+ """Group by the pair, so equal t with different destination r stays distinct."""+ chunks = int(layout.target_video_chunk_ids.max().item()) + 1+ t, t_index = build_stage1_row_timesteps(+ layout,+ video_t.expand(chunks),+ audio_t,+ keyframe_timestep=keyframe_timestep,+ device=device,+ )+ t_rows = t[t_index]+ r_rows = t_rows.clone()+ r_rows[layout.noisy_video_indices] = video_r.reshape(())+ pairs, indices = torch.unique(+ torch.stack((t_rows, r_rows), dim=1), sorted=True, dim=0, return_inverse=True+ )+ return pairs[:, 0], pairs[:, 1], indices+++def _unique_row_timestep_plan(+ layout: Any,+ *,+ current_local_chunk: int,+ video_timestep: Any,+ audio_timestep: Any,+ keyframe_timestep: float,+ device: Any,+) -> tuple[Any, Any]:+ import torch++ rows = torch.ones(layout.sequence_length, dtype=torch.float32, device=device)+ rows[layout.condition_video_indices] = float(keyframe_timestep)+ rows[layout.audio_indices] = torch.as_tensor(audio_timestep, device=device).reshape(())+ noisy_chunks = layout.target_video_chunk_ids.index_select(0, layout.noisy_video_indices)+ current_rows = layout.noisy_video_indices[noisy_chunks == int(current_local_chunk)]+ if int(current_rows.numel()) != 5 * int(layout.rows_per_video_frame):+ raise ValueError("current noisy chunk does not contain exactly five latent frames")+ rows[current_rows] = torch.as_tensor(video_timestep, device=device).reshape(())+ return torch.unique(rows, sorted=True, return_inverse=True)+++def _unique_row_interval_plan(+ layout: Any,+ *,+ current_local_chunk: int,+ video_timestep: Any,+ video_r_timestep: Any,+ audio_timestep: Any,+ keyframe_timestep: float,+ device: Any,+) -> tuple[Any, Any, Any]:+ """Group native H3 ``(t,r)`` pairs without merging unequal intervals.++ Only current noisy-video rows carry a finite-map interval. Conditions,+ audio and clean history retain their original diagonal conditioning.+ """++ import torch++ times, inverse = _unique_row_timestep_plan(+ layout,+ current_local_chunk=current_local_chunk,+ video_timestep=video_timestep,+ audio_timestep=audio_timestep,+ keyframe_timestep=keyframe_timestep,+ device=device,+ )+ rows = times[inverse]+ r_rows = rows.clone()+ noisy_chunks = layout.target_video_chunk_ids.index_select(0, layout.noisy_video_indices)+ current_rows = layout.noisy_video_indices[noisy_chunks == int(current_local_chunk)]+ r_rows[current_rows] = torch.as_tensor(video_r_timestep, device=device).reshape(())+ pairs, pair_inverse = torch.unique(+ torch.stack((rows, r_rows), dim=1), dim=0, sorted=True, return_inverse=True+ )+ return pairs[:, 0], pairs[:, 1], pair_inversediff --git a/src/solarwm/backends/minimax_h3/torch_mask.py b/src/solarwm/backends/minimax_h3/torch_mask.pynew file mode 100644index 0000000..5fe46f9--- /dev/null+++ b/src/solarwm/backends/minimax_h3/torch_mask.py@@ -0,0 +1,270 @@+"""Direct sparse W6 masks for compiled H3 teacher-forcing attention."""++from __future__ import annotations++import math+from typing import Any++from .layout import AUDIO_ROLE, CLEAN_VIDEO_ROLE, CONDITION_ROLE, NOISY_VIDEO_ROLE, H3PackedLayout+from .mask import stage1_window_allows as stage1_w6_allows+++def _validate_layout(layout: H3PackedLayout) -> None:+ if layout.stage != "stage1" or layout.window_chunks is None:+ raise ValueError("W6 masks require a Stage1 layout")+ if not int(layout.clean_video_indices.numel()) or not int(layout.noisy_video_indices.numel()):+ raise ValueError("Stage1 W6 TF requires explicit clean and noisy video copies")+++def build_stage1_w6_block_mask(+ layout: H3PackedLayout,+ *,+ batch_size: int | None = None,+ num_heads: int | None = None,+ device: Any = None,+ block_size: int | tuple[int, int] = 128,+) -> Any:+ """Build the exact W6 ``BlockMask`` without a token-level dense mask.++ ``create_block_mask`` evaluates a logical ``[Q,K]`` grid and is unsafe for+ H3's 29k/72k-row Stage1 documents unless its compilation succeeds. This+ builder instead derives full and partial tiles directly from the small set+ of contiguous role/chunk runs. Peak construction memory is therefore+ ``O(ceil(Q/128)^2)`` block metadata, never ``O(Q^2)`` token metadata.+ """++ _validate_layout(layout)+ import torch+ from torch.nn.attention.flex_attention import BlockMask++ if batch_size not in (None, 1):+ raise ValueError("H3 W6 currently requires batch_size=1")+ if num_heads is not None:+ raise ValueError("H3 W6 BlockMask must be shared across all attention heads")+ if isinstance(block_size, tuple):+ if tuple(block_size) != (128, 128):+ raise ValueError("H3 W6 requires FlexAttention block_size=(128,128)")+ tile_size = 128+ else:+ tile_size = int(block_size)+ if tile_size != 128:+ raise ValueError("H3 W6 requires FlexAttention block_size=128")++ roles_cpu = layout.row_roles.detach().to(device="cpu", dtype=torch.long)+ chunks_cpu = layout.target_video_chunk_ids.detach().to(device="cpu", dtype=torch.long)+ sequence_length = int(layout.sequence_length)+ if tuple(roles_cpu.shape) != (sequence_length,) or tuple(chunks_cpu.shape) != (+ sequence_length,+ ):+ raise ValueError("H3 W6 layout role/chunk rows do not match sequence_length")+ window = int(layout.window_chunks)++ # Find the handful of contiguous (role, chunk) runs. A typical seven-+ # chunk Stage1 document has condition, audio, seven clean and seven noisy+ # runs even though it contains tens of thousands of rows.+ changes = (roles_cpu[1:] != roles_cpu[:-1]) | (chunks_cpu[1:] != chunks_cpu[:-1])+ boundaries = [int(value) + 1 for value in torch.nonzero(changes).flatten()]+ starts = [0, *boundaries]+ ends = [*boundaries, sequence_length]+ runs = [+ (+ start,+ end,+ int(roles_cpu[start].item()),+ int(chunks_cpu[start].item()),+ )+ for start, end in zip(starts, ends, strict=True)+ ]++ def merge_intervals(intervals: list[tuple[int, int]]) -> list[tuple[int, int]]:+ merged: list[tuple[int, int]] = []+ for start, end in intervals:+ if merged and start == merged[-1][1]:+ merged[-1] = (merged[-1][0], end)+ else:+ merged.append((start, end))+ return merged++ allowed_by_query: dict[tuple[int, int], list[tuple[int, int]]] = {}+ for _start, _end, query_role, query_chunk in runs:+ query_key = (query_role, query_chunk)+ if query_key in allowed_by_query:+ continue+ allowed_by_query[query_key] = merge_intervals(+ [+ (key_start, key_end)+ for key_start, key_end, key_role, key_chunk in runs+ if stage1_w6_allows(+ query_role,+ query_chunk,+ key_role,+ key_chunk,+ window_chunks=window,+ )+ ]+ )++ num_blocks = math.ceil(sequence_length / tile_size)+ kv_starts = torch.arange(num_blocks, dtype=torch.long) * tile_size+ kv_ends = torch.clamp(kv_starts + tile_size, max=sequence_length)+ full_tiles = torch.zeros((num_blocks, num_blocks), dtype=torch.bool)+ any_tiles = torch.zeros_like(full_tiles)++ for query_block in range(num_blocks):+ query_start = query_block * tile_size+ query_end = min(query_start + tile_size, sequence_length)+ query_segments = [+ (max(query_start, start), min(query_end, end), role, chunk)+ for start, end, role, chunk in runs+ if start < query_end and query_start < end+ ]+ if not query_segments:+ raise RuntimeError(+ f"H3 W6 direct mask found no query run for [{query_start},{query_end})"+ )+ row_full = torch.ones(num_blocks, dtype=torch.bool)+ row_any = torch.zeros(num_blocks, dtype=torch.bool)+ for _segment_start, _segment_end, query_role, query_chunk in query_segments:+ segment_full = torch.zeros(num_blocks, dtype=torch.bool)+ segment_any = torch.zeros(num_blocks, dtype=torch.bool)+ for allowed_start, allowed_end in allowed_by_query[(query_role, query_chunk)]:+ segment_full |= (allowed_start <= kv_starts) & (kv_ends <= allowed_end)+ segment_any |= (kv_starts < allowed_end) & (allowed_start < kv_ends)+ row_full &= segment_full+ row_any |= segment_any+ if not bool(row_any.any().item()):+ raise RuntimeError(f"H3 W6 query block {query_block} has no visible key block")+ full_tiles[query_block] = row_full+ any_tiles[query_block] = row_any++ partial_tiles = any_tiles & ~full_tiles+ metadata_blocks = 1 << (num_blocks - 1).bit_length()++ def ordered_metadata(tile_mask: Any) -> tuple[Any, Any]:+ # Mirror torch 2.6's internal _dense_to_ordered conversion: its CPU+ # argsort path is defined for int32 metadata, not every bool backend.+ ordered = torch.zeros((metadata_blocks, metadata_blocks), dtype=torch.int32)+ ordered[:num_blocks, :num_blocks].copy_(tile_mask)+ counts = ordered.sum(dim=-1).to(dtype=torch.int32, memory_format=torch.contiguous_format)+ indices = torch.argsort(ordered, dim=-1, descending=True, stable=True).to(+ dtype=torch.int32, memory_format=torch.contiguous_format+ )+ return counts[None, None].to(device), indices[None, None].to(device)++ partial_counts, partial_indices = ordered_metadata(partial_tiles)+ full_counts, full_indices = ordered_metadata(full_tiles)++ # Capture only device scalars. PyTorch 2.6 dynamic Flex lifts the size of+ # a captured lookup table into a symbolic autotuning input it cannot key.+ # Canonical packing permits the identical role/chunk lookup by intervals,+ # without specializing caption lengths or allocating a token-level mask.+ condition_end = int(layout.condition_indices.numel())+ clean_start = condition_end + int(layout.audio_indices.numel())+ clean_rows = int(layout.clean_video_indices.numel())+ noisy_rows = int(layout.noisy_video_indices.numel())+ noisy_start = clean_start + clean_rows+ chunk_rows = int(layout.rows_per_video_frame) * int(layout.chunk_latent_frames or 0)+ if (+ chunk_rows <= 0+ or clean_rows % chunk_rows+ or noisy_rows % chunk_rows+ or noisy_start + noisy_rows != sequence_length+ ):+ raise ValueError("H3 W6 requires canonical whole-chunk [C|A|clean|noisy] packing")+ clean_chunk_start = int(chunks_cpu[clean_start])+ noisy_chunk_start = int(chunks_cpu[noisy_start])+ expected_runs = []+ if condition_end:+ expected_runs.append((0, condition_end, CONDITION_ROLE, -1))+ if clean_start > condition_end:+ expected_runs.append((condition_end, clean_start, AUDIO_ROLE, -1))+ for start, length, role, first_chunk in (+ (clean_start, clean_rows, CLEAN_VIDEO_ROLE, clean_chunk_start),+ (noisy_start, noisy_rows, NOISY_VIDEO_ROLE, noisy_chunk_start),+ ):+ expected_runs.extend(+ (+ start + offset,+ start + offset + chunk_rows,+ role,+ first_chunk + offset // chunk_rows,+ )+ for offset in range(0, length, chunk_rows)+ )+ if runs != expected_runs:+ raise ValueError("H3 W6 requires canonical contiguous role/chunk runs")+ (+ condition_end,+ clean_start,+ noisy_start,+ sequence_end,+ chunk_rows,+ clean_chunk_start,+ noisy_chunk_start,+ window_bound,+ ) = (+ torch.tensor(value, dtype=torch.long, device=device)+ for value in (+ condition_end,+ clean_start,+ noisy_start,+ sequence_length,+ chunk_rows,+ clean_chunk_start,+ noisy_chunk_start,+ window,+ )+ )++ def role_and_chunk(index: Any) -> tuple[Any, Any, Any, Any, Any]:+ valid = (index >= 0) & (index < sequence_end)+ condition = valid & (index < condition_end)+ audio = valid & (index >= condition_end) & (index < clean_start)+ clean = valid & (index >= clean_start) & (index < noisy_start)+ noisy = valid & (index >= noisy_start)+ video_start = torch.where(index >= noisy_start, noisy_start, clean_start)+ first_chunk = torch.where(index >= noisy_start, noisy_chunk_start, clean_chunk_start)+ chunk = torch.div(index - video_start, chunk_rows, rounding_mode="floor") + first_chunk+ return condition, audio, clean, noisy, chunk++ def mask_mod(_batch: Any, _head: Any, query_index: Any, key_index: Any) -> Any:+ condition_query, audio_query, clean_query, noisy_query, query_chunk = role_and_chunk(+ query_index+ )+ condition_key, audio_key, clean_key, noisy_key, key_chunk = role_and_chunk(key_index)+ delta = query_chunk - key_chunk+ common_key = condition_key | audio_key+ condition_visibility = condition_query & condition_key+ audio_visibility = audio_query & common_key+ clean_visibility = clean_query & (common_key | (clean_key & (delta >= 0)))+ noisy_visibility = noisy_query & (+ common_key+ | (clean_key & (delta >= 1) & (delta < window_bound))+ | (noisy_key & (delta == 0))+ )+ return condition_visibility | audio_visibility | clean_visibility | noisy_visibility++ block_mask = BlockMask.from_kv_blocks(+ partial_counts,+ partial_indices,+ full_counts,+ full_indices,+ BLOCK_SIZE=tile_size,+ mask_mod=mask_mod,+ seq_lengths=(sequence_length, sequence_length),+ )+ # PyTorch 2.6 Flex autotuning requires concrete metadata dimensions.+ # Power-of-two zero-count buckets keep only these sparse tensors static;+ # real token lengths and the visibility of every actual tile are unchanged.+ for metadata in (+ block_mask.kv_num_blocks,+ block_mask.kv_indices,+ block_mask.full_kv_num_blocks,+ block_mask.full_kv_indices,+ block_mask.q_num_blocks,+ block_mask.q_indices,+ block_mask.full_q_num_blocks,+ block_mask.full_q_indices,+ ):+ torch._dynamo.mark_static(metadata)+ return block_maskdiff --git a/src/solarwm/backends/minimax_h3/torch_prope.py b/src/solarwm/backends/minimax_h3/torch_prope.pynew file mode 100644index 0000000..06b9c59--- /dev/null+++ b/src/solarwm/backends/minimax_h3/torch_prope.py@@ -0,0 +1,199 @@+# SPDX-License-Identifier: Apache-2.0 AND MIT+# Adapted from PRoPE (Projective Positional Encoding for Multiview+# Transformers), MIT, Copyright (c) the PRoPE authors. Modified by SolarWM+# for this backend's attention layout.+"""CUDA/torch implementation of the H3 camera-PRoPE suffix."""++from __future__ import annotations++from collections.abc import Callable+from functools import partial+from typing import Any++from .camera import (+ H3_ATTENTION_HEAD_DIM,+ H3_CAMERA_PROPE_DIM_END,+ H3_CAMERA_PROPE_DIM_START,+ WAN_FIXED_FX,+ WAN_FIXED_FY,+)+++def logd4_relative_viewmats(viewmats: Any) -> Any:+ import torch++ if not isinstance(viewmats, torch.Tensor):+ raise TypeError("viewmats must be a torch.Tensor")+ if tuple(viewmats.shape[-2:]) != (4, 4) or not viewmats.is_floating_point():+ raise ValueError("viewmats must be floating tensors ending in [4,4]")+ compute_dtype = torch.float64 if viewmats.dtype == torch.float64 else torch.float32+ translation = viewmats[..., :3, 3].to(compute_dtype)+ norm = torch.linalg.vector_norm(translation, dim=-1, keepdim=True)+ safe = norm.clamp_min(torch.finfo(compute_dtype).tiny)+ scale = torch.where(+ norm > 0,+ torch.log1p(norm) / (4.0 * safe),+ torch.zeros_like(norm),+ )+ output = viewmats.clone()+ output[..., :3, 3] = (translation * scale).to(viewmats.dtype)+ return output+++def _fixed_focal_K(reference: Any) -> Any:+ output = reference.new_zeros(reference.shape)+ output[..., 0, 0] = WAN_FIXED_FX+ output[..., 1, 1] = WAN_FIXED_FY+ output[..., 2, 2] = 1.0+ return output+++def _invert_se3(value: Any) -> Any:+ output = value.new_zeros(value.shape)+ rotation = value[..., :3, :3].transpose(-1, -2)+ output[..., :3, :3] = rotation+ output[..., :3, 3] = -__import__("torch").einsum(+ "...ij,...j->...i", rotation, value[..., :3, 3]+ )+ output[..., 3, 3] = 1.0+ return output+++def _lift_K(value: Any) -> Any:+ output = value.new_zeros((*value.shape[:-2], 4, 4))+ output[..., :3, :3] = value+ output[..., 3, 3] = 1.0+ return output+++def _invert_K(value: Any) -> Any:+ output = value.new_zeros(value.shape)+ output[..., 0, 0] = 1.0 / value[..., 0, 0]+ output[..., 1, 1] = 1.0 / value[..., 1, 1]+ output[..., 0, 2] = -value[..., 0, 2] / value[..., 0, 0]+ output[..., 1, 2] = -value[..., 1, 2] / value[..., 1, 1]+ output[..., 2, 2] = 1.0+ return output+++def _project(features: Any, *, matrix: Any) -> Any:+ import torch++ batch, heads, sequence, width = features.shape+ projective = int(matrix.shape[-1])+ if width % projective or tuple(matrix.shape) != (+ batch,+ sequence,+ projective,+ projective,+ ):+ raise ValueError("camera projection is not aligned with attention rows")+ matrix = matrix.to(device=features.device, dtype=features.dtype)+ tiled = features.reshape(batch, heads, sequence, width // projective, projective)+ return torch.einsum("bsij,bhspj->bhspi", matrix, tiled).reshape(features.shape)+++def prope_qkv(+ q: Any,+ k: Any,+ v: Any,+ *,+ viewmats: Any,+ Ks: Any,+ **_unused: object,+) -> tuple[Any, Any, Any, Callable[[Any], Any]]:+ """Apply logd4 camera PRoPE only to H3 head dimensions ``[96:128)``."""++ import torch++ if q.ndim != 4 or q.shape != k.shape or q.shape != v.shape:+ raise ValueError("q/k/v must be equal [B,H,S,D] tensors")+ batch, _heads, sequence, head_dim = q.shape+ if head_dim != H3_ATTENTION_HEAD_DIM:+ raise ValueError("H3 camera PRoPE requires head_dim=128")+ if tuple(viewmats.shape) != (batch, sequence, 4, 4):+ raise ValueError("viewmats must be token-aligned [B,S,4,4]")+ if tuple(Ks.shape) != (batch, sequence, 3, 3):+ raise ValueError("K must be token-aligned [B,S,3,3]")+ views = logd4_relative_viewmats(viewmats)+ fixed = _fixed_focal_K(Ks)+ projection = torch.einsum("...ij,...jk->...ik", _lift_K(fixed), views)+ query_matrix = projection.transpose(-1, -2).to(views.dtype)+ kv_matrix = torch.einsum(+ "...ij,...jk->...ik", _invert_se3(views), _lift_K(_invert_K(fixed))+ ).to(views.dtype)++ def suffix(features: Any, transform: Callable[[Any], Any]) -> Any:+ native = features[..., :H3_CAMERA_PROPE_DIM_START]+ camera = features[..., H3_CAMERA_PROPE_DIM_START:H3_CAMERA_PROPE_DIM_END]+ return torch.cat((native, transform(camera)), dim=-1)++ apply_q = partial(_project, matrix=query_matrix)+ apply_kv = partial(_project, matrix=kv_matrix)+ apply_out = partial(_project, matrix=projection.to(views.dtype))+ return (+ suffix(q, apply_q),+ suffix(k, apply_kv),+ suffix(v, apply_kv),+ partial(suffix, transform=apply_out),+ )+++__all__ = ["logd4_relative_viewmats", "prope_qkv"]+++def _prepare_suffix_transforms(+ *,+ viewmats: Any,+ Ks: Any,+) -> tuple[+ Callable[[Any], Any],+ Callable[[Any], Any],+ Callable[[Any], Any],+]:+ import torch++ fixed_K = _fixed_focal_K(Ks)+ normalized_K = torch.zeros_like(fixed_K)+ normalized_K[..., 0, 0] = fixed_K[..., 0, 0]+ normalized_K[..., 1, 1] = fixed_K[..., 1, 1]+ normalized_K[..., 2, 2] = 1.0++ projection = torch.einsum("...ij,...jk->...ik", _lift_K(normalized_K), viewmats)+ projection_t = projection.transpose(-1, -2).to(dtype=viewmats.dtype)+ projection_inv = torch.einsum(+ "...ij,...jk->...ik",+ _invert_se3(viewmats),+ _lift_K(_invert_K(normalized_K)),+ ).to(dtype=viewmats.dtype)+ return (+ partial(_project, matrix=projection_t),+ partial(_project, matrix=projection_inv),+ partial(_project, matrix=projection),+ )+++def prope_qkv_separate(q, k, v, *, query_viewmats, query_K, key_viewmats, key_K):+ """H3's existing head-sliced PRoPE with different query and cached KV rows."""+ import torch++ if q.ndim != 4 or k.ndim != 4 or k.shape != v.shape:+ raise ValueError("H3 cached PRoPE requires BHSD queries and equal BHSD keys/values")+ if q.shape[:2] != k.shape[:2] or q.shape[-1] != 128 or k.shape[-1] != 128:+ raise ValueError("H3 cached PRoPE requires matching batch/heads and head_dim=128")+ apply_q, _, apply_output = _prepare_suffix_transforms(+ viewmats=logd4_relative_viewmats(query_viewmats.to(q.dtype)), Ks=query_K.to(q.dtype)+ )+ _, apply_kv, _ = _prepare_suffix_transforms(+ viewmats=logd4_relative_viewmats(key_viewmats.to(k.dtype)), Ks=key_K.to(k.dtype)+ )++ def suffix(features, transform):+ return torch.cat((features[..., :96], transform(features[..., 96:128])), dim=-1)++ return (+ suffix(q, apply_q),+ suffix(k, apply_kv),+ suffix(v, apply_kv),+ partial(suffix, transform=apply_output),+ )diff --git a/src/solarwm/backends/minimax_h3/validation_inputs.py b/src/solarwm/backends/minimax_h3/validation_inputs.pynew file mode 100644index 0000000..ea26344--- /dev/null+++ b/src/solarwm/backends/minimax_h3/validation_inputs.py@@ -0,0 +1,204 @@+"""Portable fixed H3 validation plans over preencoded conditions and cameras.++Paths in each row may be absolute or relative to the plan. Stage1 supplies+170 consecutive source cameras; Stage2 supplies the exact 47 latent cameras.+The train reader is independent of these small, immutable validation inputs.+"""++from __future__ import annotations++import hashlib+import json+from collections import Counter+from pathlib import Path+from typing import Any++import numpy as np++from solarwm.errors import DataContractError++from .artifacts import (+ H3ArtifactBatch,+ read_encoder_contract,+ read_geometry_latents,+ require_geometry_encoder_profile,+)+from .camera import validate_absolute_c2w, validate_normalized_intrinsics+from .codec import _validate_video_text_tensors+from .controlnet_contract import geometry_profile_from_model_config+++def _path(root: Path, value: str) -> Path:+ path = Path(value)+ return path if path.is_absolute() else root / path+++def _load_cameras(path: Path, row: dict[str, Any], *, stage: str) -> tuple[Any, Any]:+ from safetensors.torch import load_file++ if path.suffix == ".safetensors":+ values = load_file(str(path), device="cpu")+ elif path.suffix == ".npz":+ with np.load(path, allow_pickle=False) as archive:+ values = {key: archive[key] for key in archive.files}+ else:+ raise DataContractError("H3 validation cameras must be safetensors or NPZ")+ return validation_cameras(values, row, stage=stage)+++def validation_cameras(values: Any, row: Any, *, stage: str) -> tuple[Any, Any]:+ """Align complete camera trajectories to the causal validation rollout."""+ import torch++ convention = row.get("camera_convention")+ if stage == "stage2":+ if convention != "relative_w2c+normalized_K" or row.get("camera_alignment") != "latent":+ raise DataContractError("H3 SGF validation needs declared relative latent cameras")+ from .geometry import latent_aligned_pixel_indices++ expected = torch.tensor(latent_aligned_pixel_indices(158)) + int(+ row["validation_start_frame"]+ )+ if not torch.equal(torch.as_tensor(values["source_latent_frame_indices"]), expected):+ raise DataContractError("H3 SGF camera indices differ from the 158-frame encode")+ views, intrinsics = values["viewmats"], values["K"]+ count = 47+ else:+ expected = torch.arange(+ int(row["validation_start_frame"]), int(row["validation_start_frame"]) + 170+ )+ if not torch.equal(torch.as_tensor(values["source_frame_indices"]), expected):+ raise DataContractError("H3 Stage1 needs 170 consecutive source camera frames")+ intrinsics = values.get("camera_K", values.get("K", values.get("intrinsics")))+ if convention == "absolute_c2w+normalized_K":+ from .torch_prope import _invert_se3++ poses = torch.as_tensor(values.get("camera_c2w", values.get("c2w"))).float()+ if poses.shape != (170, 4, 4):+ raise DataContractError("H3 Stage1 absolute camera shape differs")+ relative = torch.matmul(_invert_se3(poses[:1]), poses)+ relative[0] = torch.eye(4)+ views = _invert_se3(relative)+ elif convention == "relative_w2c+normalized_K":+ views = values["viewmats"]+ else:+ raise DataContractError("H3 validation camera convention is missing or unsupported")+ views, intrinsics = torch.as_tensor(views).float(), torch.as_tensor(intrinsics).float()+ if views.shape != (170, 4, 4) or intrinsics.shape != (170, 3, 3):+ raise DataContractError("H3 Stage1 camera must cover the full 170-frame trajectory")+ indices = [0]+ for index in range(49):+ indices.append(indices[-1] + (1, 4, 4, 4, 4)[index % 5])+ views, intrinsics = views[indices], intrinsics[indices]+ count = 50+ views, intrinsics = torch.as_tensor(views).float(), torch.as_tensor(intrinsics).float()+ if views.shape != (count, 4, 4) or intrinsics.shape != (count, 3, 3):+ raise DataContractError("H3 validation camera tensor shape differs")+ if not bool(torch.isfinite(views).all() and torch.isfinite(intrinsics).all()):+ raise DataContractError("H3 validation cameras contain non-finite values")+ if not torch.allclose(views[0], torch.eye(4), atol=1e-4, rtol=0):+ raise DataContractError("H3 relative camera must anchor the first frame at identity")+ try:+ validate_absolute_c2w(views.numpy())+ validate_normalized_intrinsics(intrinsics.numpy())+ except (TypeError, ValueError) as exc:+ raise DataContractError(f"invalid H3 validation camera: {exc}") from exc+ return views.contiguous(), intrinsics.contiguous()+++class H3PreparedValidationStream:+ """Read a fixed plan in complete logical-DP waves, with no sample substitution."""++ def __init__(self, config: Any, topology: Any) -> None:+ self.encoder_contract, self.encoder_profile = read_encoder_contract(+ config["data"]["encoder_contract_path"]+ )+ self.geometry = geometry_profile_from_model_config(config["model"])+ require_geometry_encoder_profile(self.encoder_profile, self.geometry)+ self.path = Path(config["validation"]["prepared_plan"])+ payload = self.path.read_bytes()+ document = json.loads(payload)+ self.digest = hashlib.sha256(payload).hexdigest()+ self.rows = document["rows"]+ self.stage = config["train"]["stage"]+ self.topology = topology+ self.cursor = topology.dp_rank+ expected_count = int(config["validation"]["sample_count"])+ if len(self.rows) != expected_count or [+ row["validation_slot"] for row in self.rows+ ] != list(range(expected_count)):+ raise DataContractError(+ "H3 prepared validation plan must have exactly the ordered configured slots"+ )+ if len({row["sample_id"] for row in self.rows}) != expected_count:+ raise DataContractError("H3 prepared validation plan repeats a sample")+ if config["validation"].get("balance_by_dataset", False):+ counts = Counter(+ row.get("dataset_source", row["sample_id"].split("/")[0]) for row in self.rows+ )+ if max(counts.values()) - min(counts.values()) > 1:+ raise DataContractError(f"H3 validation datasets are unbalanced: {dict(counts)}")++ def next(self) -> H3ArtifactBatch:+ import torch+ from safetensors.torch import load_file++ if self.cursor >= len(self.rows):+ raise StopIteration+ row = self.rows[self.cursor]+ self.cursor += self.topology.dp_world_size+ paths = {+ key: _path(self.path.parent, row[key]) for key in ("condition_safetensors", "camera")+ }+ for key, path in paths.items():+ expected = row.get(key + "_sha256")+ if expected and hashlib.sha256(path.read_bytes()).hexdigest() != expected:+ raise DataContractError(f"H3 prepared validation artifact changed: {path}")+ values = load_file(str(paths["condition_safetensors"]), device="cpu")+ start = int(row["validation_start_frame"])+ if not torch.equal(values["source_frame_indices"], torch.arange(start, start + 158)):+ raise DataContractError("H3 validation condition source indices differ from its plan")+ geometry_keys = self.geometry.tensor_keys() if getattr(self, "geometry", None) else ()+ for key in ("target_latents", "anchor_latents", *geometry_keys):+ if key in values and values[key].ndim == 5 and values[key].shape[0] == 1:+ values[key] = values[key][0]+ prompt, tags = values["prompt_embeds"], values["text_token_tags"]+ if prompt.ndim == 3 and prompt.shape[0] == 1:+ prompt = prompt[0]+ tags = tags.reshape(-1)+ try:+ _validate_video_text_tensors(+ values["target_latents"], values["anchor_latents"], prompt, tags+ )+ except (TypeError, ValueError) as exc:+ raise DataContractError(f"invalid H3 validation condition: {exc}") from exc+ if not all(+ bool(tensor.isfinite().all())+ for tensor in (values["target_latents"], values["anchor_latents"], prompt)+ ):+ raise DataContractError("H3 validation condition contains non-finite values")+ views, intrinsics = _load_cameras(paths["camera"], row, stage=self.stage)+ # Geometry renders travel in the same condition file as the latents they drive.+ geometry_latents = read_geometry_latents(values, getattr(self, "geometry", None))+ return H3ArtifactBatch(+ sample_id=str(row["sample_id"]),+ start_frame=start,+ plan_fingerprint=self.digest,+ target_latents=values["target_latents"],+ anchor_latents=values["anchor_latents"],+ prompt_embeds=prompt,+ text_token_tags=tags,+ source_frame_indices=values["source_frame_indices"],+ camera_viewmats=views[:47],+ camera_K=intrinsics[:47],+ source_fps=None,+ validation_slot=int(row["validation_slot"]),+ validation_noise_seed=int(row["validation_noise_seed"]),+ rollout_camera_viewmats=views if self.stage == "stage1" else None,+ rollout_camera_K=intrinsics if self.stage == "stage1" else None,+ dataset_source=str(row.get("dataset_source", row["sample_id"].split("/")[0])),+ geometry_latents=geometry_latents,+ )++ def close(self) -> None:+ """The prepared stream owns no background workers or open descriptors."""diff --git a/src/solarwm/backends/minimax_h3/validation_stream.py b/src/solarwm/backends/minimax_h3/validation_stream.pynew file mode 100644index 0000000..1b869e1--- /dev/null+++ b/src/solarwm/backends/minimax_h3/validation_stream.py@@ -0,0 +1,319 @@+"""Fixed, source-balanced H3 validation directly from the released test indexes."""++from __future__ import annotations++import hashlib+import json+from collections import defaultdict+from collections.abc import Mapping, Sequence+from dataclasses import replace+from typing import Any++import numpy as np++from solarwm.data import IndexRow, read_index, resolve_index_path+from solarwm.data.sampling import SamplePlan+from solarwm.errors import DataContractError+from solarwm.runtime import Topology++from .artifacts import H3CameraFilterError, H3PreencodedStream+from .controlnet_contract import geometry_profile_from_model_config+from .raw_data import decode_camera+from .validation_inputs import validation_cameras+++def source_video_id(row: IndexRow) -> str:+ return str(row.values.get("source_sample_id") or row.sample_id.split("/h3-158f-occ-")[0])+++def validation_candidates(+ rows: Sequence[IndexRow], *, sample_count: int, seed: int, balance: bool+) -> tuple[tuple[IndexRow, ...], ...]:+ """Give each slot disjoint source videos and same-dataset reserve candidates."""+ groups: dict[str, dict[str, IndexRow]] = defaultdict(dict)+ for row in rows:+ source = row.sample_id.split("/")[0] if balance else "all"+ groups[source].setdefault(source_video_id(row), row)+ sources = sorted(groups)+ if not sources or sample_count < len(sources):+ raise DataContractError("H3 validation needs at least one slot per eligible dataset")+ slots = []+ for slot in range(sample_count):+ source = sources[slot % len(sources)]+ quota = sample_count // len(sources) + (sources.index(source) < sample_count % len(sources))+ ordered = sorted(+ groups[source].values(),+ key=lambda row: hashlib.sha256(f"{seed}:{row.sample_id}".encode()).hexdigest(),+ )+ if len(ordered) < quota:+ raise DataContractError(f"H3 validation has too few distinct source videos in {source}")+ slots.append(tuple(ordered[slot // len(sources) :: quota]))+ return tuple(slots)+++class H3IndexedValidationStream:+ """Materialize complete logical-DP waves without preparing separate tensor files."""++ def __init__(self, config: Mapping[str, Any], topology: Any, *, frozen_ids=None) -> None:+ self.config = config+ data, validation = config["data"], config["validation"]+ transport = data["transport"]+ self.stage = config["train"]["stage"]+ self.topology = topology+ self.cursor = int(topology.dp_rank)+ self.count = int(validation["sample_count"])+ if self.count < 1 or self.count % int(topology.dp_world_size):+ raise DataContractError(+ "H3 validation sample count must form complete logical-DP waves"+ )+ self.noise_seed = int(validation["noise_seed"])+ if self.noise_seed < 0 or self.noise_seed + (self.count - 1) * 1009 >= 2**63:+ raise DataContractError("H3 validation noise seed is outside [0, 2**63)")+ self.reader = H3PreencodedStream(+ root=str(transport["root"]),+ index=str(resolve_index_path(data, "test_index")),+ # The validation slots own sampling; this reader only materializes artifacts.+ topology=Topology(1, 0, 1, 0, sp_size=1),+ seed=int(data.get("seed", 42)),+ encoder_contract_path=str(data["encoder_contract_path"]),+ cache_dir=transport.get("cache_dir"),+ cache_max_gib=float(transport.get("cache_max_gib", 256)),+ num_workers=1,+ camera_audit_latents=47,+ geometry=geometry_profile_from_model_config(config["model"]),+ )+ self.encoder_contract = self.reader.encoder_contract+ self.encoder_profile = self.reader.encoder_profile+ try:+ self.raw_rows = {}+ rows = self.reader.rows+ if self.stage == "stage1":+ self.raw_rows = {+ row.sample_id: row+ for row in read_index(resolve_index_path(data, "raw_test_index"))+ }+ missing = [+ row.sample_id for row in rows if source_video_id(row) not in self.raw_rows+ ]+ if missing:+ raise DataContractError(+ f"H3 raw test index lacks encoded sources: {missing[:3]}"+ )+ rows = tuple(+ row+ for row in rows+ if int(self.raw_rows[source_video_id(row)].values["num_frames"])+ >= int(row.values["start_frame"]) + 170+ )+ self.frozen = frozen_ids is not None+ if self.frozen:+ by_id = {row.sample_id: row for row in rows}+ if len(frozen_ids) != self.count or len(set(frozen_ids)) != self.count:+ raise DataContractError(+ "H3 frozen validation IDs differ from the configured slots"+ )+ try:+ self.candidates = tuple((by_id[sample_id],) for sample_id in frozen_ids)+ except KeyError as exc:+ raise DataContractError(+ "H3 frozen validation source left the eligible test index"+ ) from exc+ else:+ self.candidates = validation_candidates(+ rows,+ sample_count=self.count,+ seed=int(validation["selection_seed"]),+ balance=bool(validation.get("balance_by_dataset", False)),+ )+ self.digest = hashlib.sha256(+ json.dumps(+ [[dict(row.values) for row in reserve] for reserve in self.candidates],+ sort_keys=True,+ separators=(",", ":"),+ ).encode()+ ).hexdigest()+ except Exception:+ self.close()+ raise++ def _stage1_camera(self, batch: Any, row: IndexRow) -> Any:+ import torch++ raw = self.raw_rows[source_video_id(row)]+ manifest = json.loads(self.reader.shards.read(row, str(row.values["manifest_member"])))+ encoding = manifest.get("h3_preencoding", {})+ transform = encoding.get("resize_transform")+ if not isinstance(transform, Mapping):+ # Reconstruct the center crop from source dimensions when metadata omits it.+ height, width = int(raw.values["height"]), int(raw.values["width"])+ scale = max(1344 / width, 768 / height)+ resized_w, resized_h = max(1344, round(width * scale)), max(768, round(height * scale))+ transform = dict(+ source_h=height,+ source_w=width,+ resized_h=resized_h,+ resized_w=resized_w,+ crop_left=(resized_w - 1344) // 2,+ crop_top=(resized_h - 768) // 2,+ target_h=768,+ target_w=1344,+ )+ intrinsics_member = raw.values.get("intrinsics_member")+ camera_bytes = self.reader.shards.read(raw, str(raw.values["camera_member"]))+ intrinsics = (+ self.reader.shards.read(raw, str(intrinsics_member)) if intrinsics_member else None+ )+ indices = tuple(range(batch.start_frame, batch.start_frame + 170))+ c2w, K = decode_camera(camera_bytes, indices, transform, intrinsics_bytes=intrinsics)+ views, K = validation_cameras(+ {"camera_c2w": c2w, "camera_K": K, "source_frame_indices": np.asarray(indices)},+ {+ "validation_start_frame": batch.start_frame,+ "camera_convention": "absolute_c2w+normalized_K",+ },+ stage="stage1",+ )+ encoded_views, encoded_K = batch.camera_viewmats, batch.camera_K+ if encoded_views.shape[0] == 47 * 1008:+ for value in (encoded_views, encoded_K):+ grouped = value.unflatten(0, (47, 1008))+ if not torch.equal(grouped, grouped[:, :1].expand_as(grouped)):+ raise DataContractError(+ "H3 Stage1 validation requires spatially constant cameras"+ )+ encoded_views, encoded_K = encoded_views[::1008], encoded_K[::1008]+ if not torch.allclose(views[:47], encoded_views, atol=1e-5, rtol=1e-5):+ raise DataContractError("H3 raw validation cameras disagree with the encoded window")+ if not torch.allclose(K[:47], encoded_K, atol=1e-6, rtol=1e-5):+ raise DataContractError("H3 raw validation intrinsics disagree with the encoded window")+ if (+ float(views.abs().max()) > 20+ or float(torch.linalg.vector_norm(views[:, :3, 3], dim=-1).max()) > 20+ ):+ raise H3CameraFilterError("H3 rollout camera exceeds the magnitude guards")+ return replace(+ batch,+ camera_viewmats=views[:47],+ camera_K=K[:47],+ rollout_camera_viewmats=views,+ rollout_camera_K=K,+ )++ def _select(self, slot: int, candidates: Sequence[IndexRow]) -> Any:+ for row in candidates:+ start = int(row.values["start_frame"])+ plan = SamplePlan(+ row.sample_id,+ row.key,+ row.shard,+ row.ordinal,+ 0,+ 0,+ start,+ tuple(range(start, start + 158)),+ self.topology.dp_rank,+ 0,+ )+ try:+ batch = self.reader.read_artifact(row, plan)+ if self.stage == "stage1":+ batch = self._stage1_camera(batch, row)+ except H3CameraFilterError:+ if self.frozen:+ raise+ continue+ return replace(+ batch,+ validation_slot=slot,+ validation_noise_seed=self.noise_seed + slot * 1009,+ plan_fingerprint=self.digest,+ )+ return None++ def prepare(self, dist: Any) -> None:+ """Freeze all slots, refilling exhausted reserves without reusing other videos."""+ from solarwm.runtime.distributed import collective_call++ def gather(local: Any) -> list[Any]:+ if not dist.is_initialized():+ if int(self.topology.dp_world_size) != 1:+ raise DataContractError("H3 distributed validation requires a process group")+ return [local]+ output = [None] * dist.get_world_size()+ dist.all_gather_object(output, local)+ return output++ slots = range(int(self.topology.dp_rank), self.count, int(self.topology.dp_world_size))+ self.prepared = collective_call(+ lambda: {slot: self._select(slot, self.candidates[slot]) for slot in slots},+ dist=dist,+ label="H3 validation candidate selection",+ )+ records = gather(+ {slot: batch.sample_id if batch else None for slot, batch in self.prepared.items()}+ )+ accepted = {}+ for record in records:+ for slot, sample_id in record.items():+ if slot in accepted and accepted[slot] != sample_id:+ raise DataContractError("H3 SP peers selected different validation sources")+ accepted[slot] = sample_id+ if set(accepted) != set(range(self.count)):+ raise DataContractError("H3 validation selection did not cover all slots")+ used = {value.split("/h3-158f-occ-")[0] for value in accepted.values() if value}+ for slot in sorted(slot for slot, value in accepted.items() if value is None):+ if self.frozen:+ raise DataContractError(+ "H3 frozen validation source no longer passes camera guards"+ )+ source = self.candidates[slot][0].sample_id.split("/")[0]+ candidates = sorted(+ (+ row+ for pool in self.candidates+ for row in pool+ if source_video_id(row) not in used+ and (+ not self.config["validation"].get("balance_by_dataset", False)+ or row.sample_id.split("/")[0] == source+ )+ ),+ key=lambda row: hashlib.sha256(+ f"{self.config['validation']['selection_seed']}:{row.sample_id}".encode()+ ).hexdigest(),+ )++ def refill(slot: int = slot, candidates: Any = candidates) -> Any:+ if slot not in self.prepared:+ return None+ batch = self._select(slot, candidates)+ if batch is None:+ raise DataContractError(+ f"H3 validation has no unused camera-safe source for slot {slot}"+ )+ self.prepared[slot] = batch+ return batch.sample_id++ sample_id = collective_call(refill, dist=dist, label="H3 validation reserve refill")+ selected = {value for value in gather(sample_id) if value is not None}+ if len(selected) != 1:+ raise DataContractError("H3 validation refill differs between SP peers")+ used.add(selected.pop().split("/h3-158f-occ-")[0])++ def next(self) -> Any:+ slot = self.cursor+ if slot >= self.count:+ raise DataContractError("H3 validation requested more cases than configured")+ prepared = getattr(self, "prepared", None)+ batch = (+ prepared.pop(slot)+ if prepared is not None+ else self._select(slot, self.candidates[slot])+ )+ if batch is None:+ raise DataContractError(f"H3 validation slot {slot} needs collective reserve refill")+ self.cursor += int(self.topology.dp_world_size)+ return batch++ def close(self) -> None:+ self.reader.close()diff --git a/src/solarwm/backends/minimax_h3/weights.py b/src/solarwm/backends/minimax_h3/weights.pynew file mode 100644index 0000000..3f2e147--- /dev/null+++ b/src/solarwm/backends/minimax_h3/weights.py@@ -0,0 +1,238 @@+"""Weights-only transitions between published H3 stages.++Supports SolarWM checkpoints and PEFT safetensors sidecars with a model.pt+configuration. Only wrapper/named-adapter prefixes are translated; optimizer,+RNG, scheduler, step and EMA history are never inherited here.+"""++from __future__ import annotations++import json+from collections.abc import Mapping+from pathlib import Path+from typing import Any++from solarwm.errors import BackendContractError+++def canonical_lora_key(key: str) -> str:+ parts = (+ part+ for part in key.split(".")+ if part+ not in {+ "_fsdp_wrapped_module",+ "_checkpoint_wrapped_module",+ }+ )+ return (+ ".".join(parts)+ .replace(".lora_A.default.weight", ".lora_A.weight")+ .replace(".lora_B.default.weight", ".lora_B.weight")+ )+++def load_initial_weights(spec: Mapping[str, Any], lora: Any) -> str:+ """Validate the stage/camera/geometry contract before copying any tensor."""+ import torch++ root = Path(str(spec["path"]))+ if root.is_file():+ root = root.parent+ stage = str(spec["stage"])+ source = str(spec["weight_source"])+ if stage not in {"stage0p5", "stage1", "stage2"} or source not in {"live", "ema"}:+ raise BackendContractError("unsupported H3 initialization stage or weight source")+ if not (root / "COMPLETE.json").is_file():+ raise BackendContractError(f"H3 initialization checkpoint is incomplete: {root}")+ geometry_policy = str(spec.get("geometry_controlnet", "required"))+ if geometry_policy not in {"required", "fresh"}:+ raise BackendContractError(+ "H3 initialization geometry_controlnet must be required or fresh"+ )+ source_geometry = None+ manifest_path = root / "checkpoint-manifest.json"+ if manifest_path.is_file():+ manifest = json.loads(manifest_path.read_text(encoding="utf-8"))+ contract = manifest.get("contract", {})+ required = {+ "family": "minimax_h3",+ "stage": stage,+ "camera_translation_transform": "logd4",+ "parameterization": "peft-lora-r384-alpha384",+ }+ profile = contract.get("extras", {}).get("encoder_profile", {})+ if any(contract.get(key) != value for key, value in required.items()) or (+ profile.get("pixel_frames"),+ profile.get("height"),+ profile.get("width"),+ ) != (158, 768, 1344):+ raise BackendContractError("H3 initialization stage/camera/encoder contract differs")+ if stage in {"stage1", "stage2"}:+ expected = {+ "chunk_latents": 5,+ "window_chunks": 6,+ "target_latents": 45 if stage == "stage1" else 47,+ "rollout_latents": 50,+ "student_rope_mode": "native_absolute" if stage == "stage1" else "sliding_local",+ }+ if any(contract.get("extras", {}).get(key) != value for key, value in expected.items()):+ raise BackendContractError("H3 initialization causal window/RoPE contract differs")+ source_geometry = contract.get("extras", {}).get("geometry_controlnet")+ component = "ema.pt" if source == "ema" else "adapter.pt"+ payload = torch.load(root / component, map_location="cpu", mmap=True, weights_only=True)+ if source == "ema":+ if payload is None or payload.get("schema") != "solarwm.minimax-h3-ema.v1":+ raise BackendContractError("H3 source checkpoint has no compatible EMA")+ values = payload["shadow"]+ frozen_lora = (+ isinstance(source_geometry, Mapping)+ and source_geometry.get("lora_trainable") is False+ )+ if frozen_lora:+ # A frozen-LoRA run's EMA tracked only the trained ControlNet; its+ # LoRA never moved, so the live adapter value is the EMA value.+ adapter = torch.load(+ root / "adapter.pt", map_location="cpu", mmap=True, weights_only=True+ )+ shadowed = {canonical_lora_key(key) for key in values}+ values = {+ **{+ key: value+ for key, value in adapter["state"].items()+ if canonical_lora_key(key) not in shadowed+ },+ **values,+ }+ else:+ values = payload["state"]+ step = int(manifest["step"])+ else:+ # PEFT sidecars use model.pt for the configuration contract.+ payload = torch.load(root / "model.pt", map_location="cpu", mmap=True, weights_only=True)+ contract = payload.get("config", {}).get("h3_runtime_contract", {})+ expected_stage = {"stage0p5": "stage0p5", "stage1": "stage1_tf", "stage2": "stage2_sgf"}[+ stage+ ]+ required = {+ "model_family": "minimax_h3",+ "stage": expected_stage,+ "pixel_frames": 158,+ "encoded_latents": 47,+ "target_latents": 45 if stage == "stage1" else 47,+ "target_height": 768,+ "target_width": 1344,+ "video_timestep_shift": 12.0,+ "audio_timestep_shift": 3.0,+ "keyframe_noise_aug": 0.999,+ }+ if any(contract.get(key) != value for key, value in required.items()):+ raise BackendContractError("H3 PEFT checkpoint contract differs")+ camera = contract.get("camera_prope", {})+ if camera.get("relative_translation_transform") != "logd4":+ raise BackendContractError("H3 PEFT checkpoint must use logd4 camera conditioning")+ if stage == "stage1" and any(+ contract.get(key) != value+ for key, value in {+ "chunk_latents": 5,+ "window_chunks": 6,+ "flow_objective": "anyflow_forward_map",+ }.items()+ ):+ raise BackendContractError("H3 Stage1 initialization requires AnyFlow W6")+ step = int(payload.get("global_step", payload.get("step", -1)))+ del payload+ from safetensors.torch import load_file++ component = (+ "adapter_model_ema.safetensors" if source == "ema" else "adapter_model.safetensors"+ )+ values = load_file(str(root / component), device="cpu")+ translated = {canonical_lora_key(key): value for key, value in values.items()}+ expected = {+ key: parameter+ for key, parameter in lora.parameter_by_key.items()+ if key in _required_initialization_keys(lora, source_geometry, geometry_policy)+ }+ if len(translated) != len(values) or set(translated) != set(expected):+ raise BackendContractError("H3 initialization LoRA tensor keys differ")+ for key, parameter in expected.items():+ value = translated[key]+ if value.shape != parameter.shape or value.dtype not in {torch.float32, torch.bfloat16}:+ raise BackendContractError(f"H3 initialization tensor descriptor differs: {key}")+ with torch.no_grad():+ for key, parameter in expected.items():+ parameter.copy_(translated[key].to(device=parameter.device, dtype=parameter.dtype))+ return f"{stage}:{source}:step={step}"+++def load_geometry_controlnet_weights(path: str, lora: Any) -> str:+ """Overlay a converted ``geometry_controlnet.*`` file onto the adapter set.++ Used to start a run from a ControlNet trained elsewhere (FastVideo); the key+ set and every shape must match the run's trunk exactly.+ """++ import torch+ from safetensors import safe_open+ from safetensors.torch import load_file++ from .controlnet_contract import H3_GEOMETRY_MODULE, H3_GEOMETRY_STATE_PREFIX+ from .fastvideo_import import H3_GEOMETRY_WEIGHTS_SCHEMA++ if getattr(lora, "geometry_profile", None) is None:+ raise BackendContractError("geometry ControlNet weights need a run that builds the trunk")+ with safe_open(str(path), framework="pt") as handle:+ schema = (handle.metadata() or {}).get("schema")+ if schema != H3_GEOMETRY_WEIGHTS_SCHEMA:+ raise BackendContractError(f"{path} is not a SolarWM geometry ControlNet weights file")+ prefix = f"{H3_GEOMETRY_MODULE}."+ values = {+ f"{H3_GEOMETRY_STATE_PREFIX}{key[len(prefix) :]}": value+ for key, value in load_file(str(path), device="cpu").items()+ if key.startswith(prefix)+ }+ expected = set(lora.geometry_keys)+ if set(values) != expected:+ raise BackendContractError(+ "geometry ControlNet weights differ from the run's trunk: "+ f"missing={sorted(expected - set(values))[:8]} "+ f"extra={sorted(set(values) - expected)[:8]}"+ )+ for key, value in values.items():+ if tuple(value.shape) != tuple(lora.parameter_by_key[key].shape):+ raise BackendContractError(f"geometry ControlNet weight shape differs for {key!r}")+ with torch.no_grad():+ for key, value in values.items():+ parameter = lora.parameter_by_key[key]+ parameter.copy_(value.to(device=parameter.device, dtype=parameter.dtype))+ return f"geometry_controlnet:{Path(path).name}"+++def _required_initialization_keys(lora: Any, source_geometry: Any, policy: str) -> set[str]:+ """Keys a source must provide, after matching geometry ControlNet identities.++ ``fresh`` is the one way to seed a ControlNet run from a checkpoint that has+ none: the trunk keeps its zero-initialised (exact no-op) state.+ """++ keys = set(lora.parameter_by_key)+ profile = getattr(lora, "geometry_profile", None)+ if profile is None:+ if source_geometry is not None:+ raise BackendContractError(+ "H3 initialization carries a geometry ControlNet this run does not build"+ )+ return keys+ if source_geometry is None:+ if policy != "fresh":+ raise BackendContractError(+ "H3 initialization source has no geometry ControlNet; set "+ "geometry_controlnet: fresh to start the trunk from its no-op initialisation"+ )+ return keys - set(lora.geometry_keys)+ if not isinstance(source_geometry, Mapping) or (+ source_geometry.get("architecture") != profile.architecture()+ ):+ raise BackendContractError("H3 initialization geometry ControlNet architecture differs")+ return keysdiff --git a/src/solarwm/backends/registry.py b/src/solarwm/backends/registry.pynew file mode 100644index 0000000..31c81db--- /dev/null+++ b/src/solarwm/backends/registry.py@@ -0,0 +1,58 @@+"""Backend protocol and lazy builtin discovery."""++from __future__ import annotations++import importlib+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any, Protocol, runtime_checkable++from solarwm.errors import BackendContractError+++@dataclass(frozen=True)+class BackendSpec:+ family: str+ display_name: str+ module: str+ factory: str = "create_backend"+++@runtime_checkable+class Backend(Protocol):+ family: str++ def validate_config(self, config: Mapping[str, Any]) -> None: ...++ def train(self, config: Mapping[str, Any]) -> int: ...++ def infer(self, config: Mapping[str, Any]) -> int: ...++ def preencode(self, config: Mapping[str, Any]) -> int: ...+++_BUILTINS = {+ "minimax_h3": BackendSpec("minimax_h3", "MiniMax-H3", "solarwm.backends.minimax_h3"),+}+++def backend_spec(family: str) -> BackendSpec:+ try:+ return _BUILTINS[family]+ except KeyError as exc:+ raise BackendContractError(f"unknown backend family {family!r}") from exc+++def load_backend(family: str) -> Backend:+ spec = backend_spec(family)+ try:+ module = importlib.import_module(spec.module)+ factory = getattr(module, spec.factory)+ backend = factory(family=family)+ except Exception as exc:+ raise BackendContractError(+ f"failed to load {spec.display_name} backend from {spec.module}: {exc}"+ ) from exc+ if not isinstance(backend, Backend):+ raise BackendContractError(f"{spec.module}.{spec.factory} returned an invalid backend")+ return backenddiff --git a/src/solarwm/checkpoint/__init__.py b/src/solarwm/checkpoint/__init__.pynew file mode 100644index 0000000..81d9de0--- /dev/null+++ b/src/solarwm/checkpoint/__init__.py@@ -0,0 +1,17 @@+"""Transactional v2 checkpoints and exact-resume compatibility."""++from .store import (+ CheckpointContract,+ CheckpointTransaction,+ VerifiedCheckpoint,+ assert_resume_compatible,+ verify_checkpoint,+)++__all__ = [+ "CheckpointContract",+ "CheckpointTransaction",+ "VerifiedCheckpoint",+ "assert_resume_compatible",+ "verify_checkpoint",+]diff --git a/src/solarwm/checkpoint/store.py b/src/solarwm/checkpoint/store.pynew file mode 100644index 0000000..c26e1b5--- /dev/null+++ b/src/solarwm/checkpoint/store.py@@ -0,0 +1,268 @@+"""Local checkpoint commit protocol with explicit component inventories."""++from __future__ import annotations++import hashlib+import json+import os+import uuid+from collections.abc import Mapping, Sequence+from dataclasses import asdict, dataclass, field+from pathlib import Path+from typing import Any++from solarwm.errors import CheckpointError+from solarwm.runtime.serialization import canonical_json_bytes++_MANIFEST = "checkpoint-manifest.json"+_COMPLETE = "COMPLETE.json"+++@dataclass(frozen=True)+class CheckpointContract:+ """Algorithm state that must match for an exact full resume."""++ family: str+ stage: str+ causal_mode: str+ objective: str+ objective_variant: str+ camera_translation_transform: str+ parameterization: str+ sp_size: int+ data_generation: str+ extras: Mapping[str, Any] = field(default_factory=dict)++ def __post_init__(self) -> None:+ for name in (+ "family",+ "stage",+ "causal_mode",+ "objective",+ "camera_translation_transform",+ "parameterization",+ "data_generation",+ ):+ if not str(getattr(self, name)).strip():+ raise CheckpointError(f"checkpoint contract {name} must be non-empty")+ if self.sp_size < 1:+ raise CheckpointError("checkpoint contract sp_size must be positive")++ def as_dict(self) -> dict[str, Any]:+ return asdict(self)++ @classmethod+ def from_dict(cls, value: Mapping[str, Any]) -> CheckpointContract:+ try:+ return cls(+ family=str(value["family"]),+ stage=str(value["stage"]),+ causal_mode=str(value["causal_mode"]),+ objective=str(value["objective"]),+ objective_variant=str(value.get("objective_variant", "")),+ camera_translation_transform=str(value["camera_translation_transform"]),+ parameterization=str(value["parameterization"]),+ sp_size=int(value["sp_size"]),+ data_generation=str(value["data_generation"]),+ extras=dict(value.get("extras", {})),+ )+ except (KeyError, TypeError, ValueError) as exc:+ raise CheckpointError("invalid checkpoint contract") from exc+++@dataclass(frozen=True)+class ComponentFile:+ path: str+ size: int++ def __post_init__(self) -> None:+ if not isinstance(self.path, str) or not self.path:+ raise CheckpointError("checkpoint component path must be non-empty")+ if type(self.size) is not int or self.size < 1:+ raise CheckpointError(f"checkpoint component size must be positive: {self.path!r}")+++@dataclass(frozen=True)+class VerifiedCheckpoint:+ path: Path+ step: int+ contract: CheckpointContract+ files: tuple[ComponentFile, ...]+ metadata: Mapping[str, Any]+ manifest_digest: str+++def _atomic_write(path: Path, data: bytes) -> None:+ temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")+ with temporary.open("wb") as handle:+ handle.write(data)+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temporary, path)+++def _component_files(root: Path, required: Sequence[str]) -> tuple[ComponentFile, ...]:+ required_set = set(required)+ if not required_set or "" in required_set:+ raise CheckpointError("required checkpoint components must be non-empty")+ if len(required_set) != len(required):+ raise CheckpointError("required checkpoint components contain duplicates")+ for component in sorted(required_set):+ if "/" in component or component in {".", "..", _MANIFEST, _COMPLETE}:+ raise CheckpointError(f"invalid checkpoint component {component!r}")+ target = root / component+ if not target.exists():+ raise CheckpointError(f"checkpoint component is missing: {component}")+ if target.is_symlink():+ raise CheckpointError(f"checkpoint component may not be a symlink: {component}")++ records: list[ComponentFile] = []+ for component in sorted(required_set):+ target = root / component+ candidates = (target,) if target.is_file() else tuple(sorted(target.rglob("*")))+ for candidate in candidates:+ if candidate.is_symlink():+ raise CheckpointError(+ f"checkpoint contents may not be symlinks: {candidate.relative_to(root)}"+ )+ if not candidate.is_file():+ continue+ records.append(+ ComponentFile(+ path=candidate.relative_to(root).as_posix(),+ size=candidate.stat().st_size,+ )+ )+ if not records:+ raise CheckpointError("checkpoint contains no component files")+ return tuple(records)+++class CheckpointTransaction:+ """Build a checkpoint privately and publish it with one atomic rename.++ Version 2 manifests record the exact component inventory and byte sizes.+ They deliberately avoid rereading large payloads to compute content digests.+ """++ def __init__(self, target: str | Path) -> None:+ self.target = Path(target).resolve()+ self.path = self.target.with_name(f".{self.target.name}.{uuid.uuid4().hex}.partial")+ self._committed = False++ def __enter__(self) -> CheckpointTransaction:+ if self.target.exists():+ raise CheckpointError(f"checkpoint target already exists: {self.target}")+ self.target.parent.mkdir(parents=True, exist_ok=True)+ self.path.mkdir(mode=0o755)+ return self++ def commit(+ self,+ *,+ step: int,+ contract: CheckpointContract,+ required_components: Sequence[str],+ metadata: Mapping[str, Any],+ ) -> VerifiedCheckpoint:+ if self._committed:+ raise CheckpointError("checkpoint transaction was already committed")+ if step < 0:+ raise CheckpointError("checkpoint step must be non-negative")+ files = _component_files(self.path, required_components)+ manifest = {+ "schema": "solarwm.checkpoint.v2",+ "step": int(step),+ "contract": contract.as_dict(),+ "required_components": sorted(required_components),+ "files": [{"path": record.path, "size": record.size} for record in files],+ "metadata": dict(metadata),+ }+ manifest_bytes = canonical_json_bytes(manifest)+ manifest_digest = hashlib.blake2s(manifest_bytes).hexdigest()+ _atomic_write(self.path / _MANIFEST, manifest_bytes)+ _atomic_write(+ self.path / _COMPLETE,+ canonical_json_bytes(+ {+ "schema": "solarwm.checkpoint-complete.v2",+ "step": int(step),+ "manifest_digest": manifest_digest,+ }+ ),+ )+ if self.target.exists():+ raise CheckpointError(f"checkpoint target appeared during commit: {self.target}")+ os.replace(self.path, self.target)+ directory_fd = os.open(self.target.parent, os.O_RDONLY)+ try:+ os.fsync(directory_fd)+ finally:+ os.close(directory_fd)+ self._committed = True+ return verify_checkpoint(self.target)++ def __exit__(self, *_: object) -> None:+ # An incomplete directory is intentionally retained for diagnosis. It+ # has a .partial suffix and is never discoverable as a valid checkpoint.+ return None+++def verify_checkpoint(path: str | Path) -> VerifiedCheckpoint:+ """Validate a v2 checkpoint's small metadata, inventory, paths, and sizes."""+ root = Path(path).resolve()+ complete_path = root / _COMPLETE+ manifest_path = root / _MANIFEST+ if not complete_path.is_file() or not manifest_path.is_file():+ raise CheckpointError(f"checkpoint is not complete: {root}")+ try:+ complete = json.loads(complete_path.read_bytes())+ manifest_bytes = manifest_path.read_bytes()+ manifest = json.loads(manifest_bytes)+ except (OSError, json.JSONDecodeError) as exc:+ raise CheckpointError(f"checkpoint metadata is invalid: {root}") from exc+ if (+ complete.get("schema") != "solarwm.checkpoint-complete.v2"+ or manifest.get("schema") != "solarwm.checkpoint.v2"+ ):+ raise CheckpointError("unknown or mismatched checkpoint schemas")+ manifest_digest = hashlib.blake2s(manifest_bytes).hexdigest()+ if complete.get("manifest_digest") != manifest_digest:+ raise CheckpointError("checkpoint manifest identity differs from COMPLETE")+ if complete.get("step") != manifest.get("step"):+ raise CheckpointError("checkpoint step differs between manifest and COMPLETE")++ try:+ files = tuple(ComponentFile(**value) for value in manifest["files"])+ contract = CheckpointContract.from_dict(manifest["contract"])+ metadata = dict(manifest["metadata"])+ step = int(manifest["step"])+ except (KeyError, TypeError, ValueError) as exc:+ raise CheckpointError("checkpoint manifest fields are invalid") from exc+ if not files:+ raise CheckpointError("checkpoint manifest has no files")+ if len({record.path for record in files}) != len(files):+ raise CheckpointError("checkpoint manifest has duplicate file paths")+ for record in files:+ relative = Path(record.path)+ if relative.is_absolute() or ".." in relative.parts:+ raise CheckpointError(f"checkpoint manifest has unsafe path {record.path!r}")+ target = root / relative+ if not target.is_file() or target.is_symlink():+ raise CheckpointError(f"checkpoint file is missing: {record.path}")+ if target.stat().st_size != record.size:+ raise CheckpointError(f"checkpoint file size differs: {record.path}")+ return VerifiedCheckpoint(root, step, contract, files, metadata, manifest_digest)+++def assert_resume_compatible(expected: CheckpointContract, actual: CheckpointContract) -> None:+ """Reject full resume unless every algorithm-bearing field is identical."""++ if expected.as_dict() == actual.as_dict():+ return+ differences = {+ key: {"expected": expected.as_dict()[key], "actual": actual.as_dict()[key]}+ for key in expected.as_dict()+ if expected.as_dict()[key] != actual.as_dict()[key]+ }+ raise CheckpointError(f"exact-resume contract mismatch: {differences}")diff --git a/src/solarwm/cli.py b/src/solarwm/cli.pynew file mode 100644index 0000000..bc728fc--- /dev/null+++ b/src/solarwm/cli.py@@ -0,0 +1,331 @@+"""Command-line interface for training, inference, preencoding, and data inspection."""++from __future__ import annotations++import argparse+import hashlib+import json+import os+import sys+import tempfile+from collections.abc import Sequence+from pathlib import Path+from typing import Any++from solarwm import __version__+from solarwm.config import load_config+from solarwm.config.routes import supported_routes, validate_route+from solarwm.data.index import inventory, read_index+from solarwm.data.sampling import (+ CanonicalSampler,+ ReaderIdentity,+ SamplingConfig,+ plan_fingerprint,+)+from solarwm.errors import SolarWMError+from solarwm.runtime.create_only import write_file_create_only+from solarwm.runtime.images import probe_python_runtime+from solarwm.runtime.output_layout import (+ camera_inference_output_layout,+ invocation_output_dir,+)+from solarwm.runtime.provenance import build_launch_manifest, reject_inline_secrets+++def _common_config(parser: argparse.ArgumentParser) -> None:+ parser.add_argument("--config", required=True, type=Path)+ parser.add_argument(+ "--set",+ action="append",+ default=[],+ metavar="KEY=VALUE",+ help="override one dotted config value; may be repeated",+ )+++def build_parser() -> argparse.ArgumentParser:+ parser = argparse.ArgumentParser(prog="solarwm")+ parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")+ commands = parser.add_subparsers(dest="command", required=True)++ for action in ("train", "infer", "preencode"):+ child = commands.add_parser(action, help=f"run the unified {action} entrypoint")+ _common_config(child)++ config = commands.add_parser("config", help="resolve and inspect a run config")+ config_commands = config.add_subparsers(dest="config_command", required=True)+ resolve = config_commands.add_parser("resolve")+ _common_config(resolve)+ resolve.add_argument("--output", type=Path)+ config_commands.add_parser("routes")++ data = commands.add_parser("data", help="inspect canonical index semantics")+ data_commands = data.add_subparsers(dest="data_command", required=True)+ inspect = data_commands.add_parser("inspect")+ inspect.add_argument("index", type=Path)+ plan = data_commands.add_parser("plan")+ plan.add_argument("index", type=Path)+ plan.add_argument("--seed", type=int, required=True)+ plan.add_argument("--pixel-frames", type=int, required=True)+ plan.add_argument("--rank", type=int, default=0)+ plan.add_argument("--world-size", type=int, default=1)+ plan.add_argument("--worker-id", type=int, default=0)+ plan.add_argument("--num-workers", type=int, default=1)+ plan.add_argument("--epoch", type=int, default=0)+ plan.add_argument("--limit", type=int, default=16)+ plan.add_argument("--shuffle-buffer", type=int, default=4096)++ environment = commands.add_parser("environment", help="inspect the current runtime")+ environment_commands = environment.add_subparsers(dest="environment_command", required=True)+ environment_commands.add_parser("probe")++ return parser+++def _write_json(value: Any, output: Path | None = None) -> None:+ payload = json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n"+ if output is None:+ sys.stdout.write(payload)+ return+ output.parent.mkdir(parents=True, exist_ok=True)+ temporary: str | None = None+ try:+ with tempfile.NamedTemporaryFile(+ mode="w",+ encoding="utf-8",+ dir=output.parent,+ prefix=f".{output.name}.",+ suffix=".tmp",+ delete=False,+ ) as handle:+ temporary = handle.name+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temporary, output)+ temporary = None+ finally:+ if temporary is not None:+ Path(temporary).unlink(missing_ok=True)+++def _run_backend(args: argparse.Namespace) -> int:+ resolved = load_config(args.config, args.set)+ reject_inline_secrets(resolved.values)+ configured_action = str(resolved.values["action"])+ if configured_action != args.command:+ raise SolarWMError(+ f"command {args.command!r} does not match config action {configured_action!r}"+ )+ route = validate_route(resolved.values)+ runtime = resolved.values.get("runtime", {})++ from solarwm.backends import load_backend++ backend = load_backend(route.family)+ backend.validate_config(resolved.values)+ if not str(runtime.get("output_dir", "")).strip():+ raise SolarWMError("runtime.output_dir is required")+ publication_layout = camera_inference_output_layout(resolved.values)+ output_dir = invocation_output_dir(resolved.values)+ launcher_rank = int(os.environ.get("RANK", "0"))+ if launcher_rank == 0:+ if publication_layout is not None:+ publication_layout.publish_root.mkdir(parents=True, exist_ok=True)+ output_dir.parent.mkdir(parents=True, exist_ok=True)+ try:+ output_dir.mkdir()+ except FileExistsError as exc:+ raise SolarWMError(f"camera inference run already exists: {output_dir}") from exc+ else:+ output_dir.mkdir(parents=True, exist_ok=True)+ # A terminal result describes only the current invocation. Remove+ # a result left by an earlier legacy invocation before work starts.+ (output_dir / "run-result.json").unlink(missing_ok=True)+ resolved.write_json(output_dir / "resolved-config.json")+ manifest = build_launch_manifest(+ config=resolved.values,+ source_config=resolved.path,+ source_digest=resolved.source_digest,+ resolved_digest=resolved.resolved_digest,+ route=route.key,+ repository=Path(__file__).resolve().parents[2],+ )+ _write_json(manifest, output_dir / "launch-manifest.json")+ operation = getattr(backend, args.command)+ try:+ result = int(operation(resolved.values) or 0)+ except Exception as exc:+ if launcher_rank == 0:+ _write_json(+ {+ "schema": "solarwm.run-result.v1",+ "status": "failed",+ "error_type": type(exc).__name__,+ "message": str(exc),+ },+ output_dir / "run-result.json",+ )+ raise+ if launcher_rank == 0:+ _write_json(+ {"schema": "solarwm.run-result.v1", "status": "complete", "exit_code": result},+ output_dir / "run-result.json",+ )+ if publication_layout is not None and result == 0:+ publication_complete = output_dir / "publication" / "COMPLETE.json"+ if not publication_complete.is_file():+ raise SolarWMError(+ "camera inference returned without a publication completion marker"+ )+ publication_payload = publication_complete.read_bytes()+ run_result_payload = (output_dir / "run-result.json").read_bytes()+ complete_payload = (+ json.dumps(+ {+ "schema": "solarwm.camera-inference-run-complete.v1",+ "layout": publication_layout.layout,+ "run_id": publication_layout.run_id,+ "resolved_config_digest": resolved.resolved_digest,+ "publication_complete_digest": hashlib.blake2s(+ publication_payload+ ).hexdigest(),+ "run_result_digest": hashlib.blake2s(run_result_payload).hexdigest(),+ },+ allow_nan=False,+ sort_keys=True,+ separators=(",", ":"),+ ensure_ascii=False,+ )+ + "\n"+ ).encode("utf-8")+ write_file_create_only(+ output_dir / "COMPLETE.json",+ complete_payload,+ error_type=SolarWMError,+ label="camera inference run completion marker",+ )+ return result+++def _config_command(args: argparse.Namespace) -> int:+ if args.config_command == "routes":+ _write_json([route.key for route in supported_routes()])+ return 0+ resolved = load_config(args.config, args.set)+ reject_inline_secrets(resolved.values)+ route = validate_route(resolved.values)+ from solarwm.backends import load_backend++ load_backend(route.family).validate_config(resolved.values)+ payload = {+ "source": str(resolved.path),+ "source_digest": resolved.source_digest,+ "resolved_digest": resolved.resolved_digest,+ "route": route.key,+ "config": resolved.values,+ }+ _write_json(payload, args.output)+ return 0+++def _data_command(args: argparse.Namespace) -> int:+ rows = read_index(args.index)+ report = inventory(args.index, rows)+ if args.data_command == "inspect":+ _write_json(+ {+ "index": str(args.index.resolve()),+ "rows": report.rows,+ "virtual_occurrences": report.virtual_occurrences,+ "ordered_sample_id_digest": report.ordered_sample_id_digest,+ "ordered_row_digest": report.ordered_row_digest,+ "decompressed_digest": report.decompressed_digest,+ }+ )+ return 0++ config = SamplingConfig(+ seed=args.seed,+ pixel_frames=args.pixel_frames,+ shuffle_buffer=args.shuffle_buffer,+ )+ identity = ReaderIdentity(+ rank=args.rank,+ world_size=args.world_size,+ worker_id=args.worker_id,+ num_workers=args.num_workers,+ )+ plans = list(CanonicalSampler(rows, config, identity).iter_epoch(args.epoch))+ selected = plans[: max(0, args.limit)]+ _write_json(+ {+ "full_plan_fingerprint": plan_fingerprint(plans),+ "full_plan_count": len(plans),+ "shown": [+ {+ "sample_id": plan.sample_id,+ "key": plan.key,+ "shard": plan.shard,+ "repeat_ordinal": plan.repeat_ordinal,+ "start_frame": plan.start_frame,+ "source_frame_indices": plan.source_frame_indices,+ }+ for plan in selected+ ],+ }+ )+ return 0+++def _environment_command(args: argparse.Namespace) -> int:+ if args.environment_command == "probe":+ _write_json(probe_python_runtime())+ return 0+ raise SolarWMError(f"unhandled environment command {args.environment_command!r}")+++def _run_backend_or_leave(args: argparse.Namespace) -> int:+ """Run a backend command; a rank of a distributed launch that fails exits outright.++ A rank that returns from a failure still joins its threads at interpreter shutdown, and one+ that never ends leaves it alive: the launcher never sees it fail, and every other+ rank waits out the collective timeout. Leaving at once hands the failure to the launcher.+ """++ if int(os.environ.get("WORLD_SIZE", "1")) <= 1:+ return _run_backend(args)+ try:+ return _run_backend(args)+ except (SolarWMError, OSError, ValueError) as exc:+ print(f"solarwm: error: {exc}", file=sys.stderr)+ status = 2+ except BaseException:+ import traceback++ traceback.print_exc()+ status = 1+ sys.stdout.flush()+ sys.stderr.flush()+ os._exit(status)+++def main(argv: Sequence[str] | None = None) -> int:+ args = build_parser().parse_args(argv)+ try:+ if args.command in {"train", "infer", "preencode"}:+ return _run_backend_or_leave(args)+ if args.command == "config":+ return _config_command(args)+ if args.command == "data":+ return _data_command(args)+ if args.command == "environment":+ return _environment_command(args)+ raise SolarWMError(f"unhandled command {args.command!r}")+ except (SolarWMError, OSError, ValueError) as exc:+ print(f"solarwm: error: {exc}", file=sys.stderr)+ return 2+++if __name__ == "__main__": # pragma: no cover+ raise SystemExit(main())diff --git a/src/solarwm/config/__init__.py b/src/solarwm/config/__init__.pynew file mode 100644index 0000000..b68b81a--- /dev/null+++ b/src/solarwm/config/__init__.py@@ -0,0 +1,11 @@+"""Versioned configuration loading and validation."""++from .loader import ResolvedConfig, load_config+from .routes import Route, validate_route++__all__ = [+ "ResolvedConfig",+ "Route",+ "load_config",+ "validate_route",+]diff --git a/src/solarwm/config/loader.py b/src/solarwm/config/loader.pynew file mode 100644index 0000000..4399c92--- /dev/null+++ b/src/solarwm/config/loader.py@@ -0,0 +1,243 @@+"""Safe YAML loading, environment expansion, overrides, and provenance."""++from __future__ import annotations++import copy+import hashlib+import json+import math+import os+import re+import tempfile+from collections.abc import Mapping, MutableMapping, Sequence+from dataclasses import dataclass+from pathlib import Path+from typing import Any++import yaml++from solarwm.errors import ConfigurationError++_ENV = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}")+_TOP_LEVEL_KEYS = frozenset(+ {+ "schema",+ "action",+ "name",+ "model",+ "data",+ "distributed",+ "train",+ "validation",+ "checkpoint",+ "preencode",+ "inference",+ "runtime",+ "metadata",+ }+)+++class _StrictSafeLoader(yaml.SafeLoader):+ """Safe YAML loader that rejects duplicate and non-string mapping keys."""+++def _construct_mapping(+ loader: _StrictSafeLoader, node: yaml.nodes.MappingNode, deep: bool = False+) -> dict[str, Any]:+ loader.flatten_mapping(node)+ result: dict[str, Any] = {}+ for key_node, value_node in node.value:+ key = loader.construct_object(key_node, deep=deep)+ if not isinstance(key, str):+ raise ConfigurationError(+ f"YAML mapping keys must be strings at line {key_node.start_mark.line + 1}"+ )+ if key in result:+ raise ConfigurationError(+ f"duplicate YAML key {key!r} at line {key_node.start_mark.line + 1}"+ )+ result[key] = loader.construct_object(value_node, deep=deep)+ return result+++_StrictSafeLoader.add_constructor(+ yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_mapping+)+++def _construct_json_mapping(pairs: list[tuple[str, Any]]) -> dict[str, Any]:+ result: dict[str, Any] = {}+ for key, value in pairs:+ if key in result:+ raise ConfigurationError(f"duplicate JSON key {key!r}")+ result[key] = value+ return result+++def _expand_env(value: Any) -> Any:+ if isinstance(value, dict):+ return {key: _expand_env(item) for key, item in value.items()}+ if isinstance(value, list):+ return [_expand_env(item) for item in value]+ if not isinstance(value, str):+ return value++ def replace(match: re.Match[str]) -> str:+ name = match.group(1)+ if name not in os.environ:+ raise ConfigurationError(f"environment variable {name!r} is not set")+ return os.environ[name]++ return _ENV.sub(replace, value)+++def _parse_override(raw: str) -> tuple[list[str], Any]:+ if "=" not in raw:+ raise ConfigurationError(f"override must be KEY=VALUE, got {raw!r}")+ dotted, value = raw.split("=", 1)+ keys = [part for part in dotted.split(".") if part]+ if not keys:+ raise ConfigurationError(f"override has an empty key: {raw!r}")+ try:+ parsed = yaml.load(value, Loader=_StrictSafeLoader)+ except (yaml.YAMLError, ConfigurationError) as exc:+ raise ConfigurationError(f"invalid override {raw!r}: {exc}") from exc+ return keys, parsed+++def _apply_override(config: MutableMapping[str, Any], raw: str) -> None:+ keys, value = _parse_override(raw)+ node: MutableMapping[str, Any] = config+ for key in keys[:-1]:+ child = node.get(key)+ if child is None:+ child = {}+ node[key] = child+ if not isinstance(child, MutableMapping):+ raise ConfigurationError(f"override {raw!r} crosses non-mapping key {key!r}")+ node = child+ node[keys[-1]] = value+++def canonical_json(config: Mapping[str, Any]) -> bytes:+ """Return stable bytes used by manifests and checkpoint provenance."""++ return (+ json.dumps(+ config,+ allow_nan=False,+ sort_keys=True,+ separators=(",", ":"),+ ensure_ascii=False,+ )+ + "\n"+ ).encode("utf-8")+++@dataclass(frozen=True)+class ResolvedConfig:+ """A validated config plus its exact source and resolved identities."""++ path: Path+ source_digest: str+ resolved_digest: str+ values: Mapping[str, Any]++ def mutable_copy(self) -> dict[str, Any]:+ return copy.deepcopy(dict(self.values))++ def write_json(self, path: str | Path) -> None:+ target = Path(path)+ target.parent.mkdir(parents=True, exist_ok=True)+ temporary: str | None = None+ try:+ with tempfile.NamedTemporaryFile(+ mode="wb",+ dir=target.parent,+ prefix=f".{target.name}.",+ suffix=".tmp",+ delete=False,+ ) as handle:+ temporary = handle.name+ handle.write(canonical_json(self.values))+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temporary, target)+ temporary = None+ finally:+ if temporary is not None:+ Path(temporary).unlink(missing_ok=True)+++def _validate_json_value(value: Any, path: tuple[str, ...] = ()) -> None:+ label = ".".join(path) or "<root>"+ if value is None or isinstance(value, (str, bool, int)):+ return+ if isinstance(value, float):+ if not math.isfinite(value):+ raise ConfigurationError(f"non-finite number at {label}")+ return+ if isinstance(value, list):+ for index, child in enumerate(value):+ _validate_json_value(child, (*path, str(index)))+ return+ if isinstance(value, dict):+ for key, child in value.items():+ if not isinstance(key, str):+ raise ConfigurationError(f"non-string mapping key at {label}")+ _validate_json_value(child, (*path, key))+ return+ raise ConfigurationError(+ f"unsupported YAML value {type(value).__name__} at {label}; "+ "configs must be canonical JSON-compatible values"+ )+++def _validate_top_level(config: Mapping[str, Any]) -> None:+ unknown = sorted(set(config) - _TOP_LEVEL_KEYS)+ if unknown:+ raise ConfigurationError(f"unknown top-level config keys: {unknown}")+ if config.get("schema") != "solarwm.run.v1":+ raise ConfigurationError("schema must be exactly 'solarwm.run.v1'")+ action = str(config.get("action", "")).strip().lower()+ if action not in {"train", "infer", "preencode"}:+ raise ConfigurationError("action must be one of: train, infer, preencode")+ if not str(config.get("name", "")).strip():+ raise ConfigurationError("name must be a non-empty run identifier")+ for section in ("model", "data", "runtime"):+ if not isinstance(config.get(section), Mapping):+ raise ConfigurationError(f"{section} must be a mapping")+++def load_config(path: str | Path, overrides: Sequence[str] = ()) -> ResolvedConfig:+ """Load a config without executing constructors or arbitrary code."""++ source_path = Path(path).resolve()+ raw = source_path.read_bytes()+ try:+ if source_path.suffix.lower() == ".json":+ loaded = json.loads(raw, object_pairs_hook=_construct_json_mapping)+ else:+ loaded = yaml.load(raw, Loader=_StrictSafeLoader)+ except (json.JSONDecodeError, UnicodeDecodeError, yaml.YAMLError, ConfigurationError) as exc:+ raise ConfigurationError(f"cannot parse {source_path}: {exc}") from exc+ if not isinstance(loaded, dict):+ raise ConfigurationError(f"{source_path} must contain a YAML mapping")+ values = _expand_env(loaded)+ for override in overrides:+ _apply_override(values, override)+ _validate_top_level(values)+ _validate_json_value(values)++ # Import lazily to keep configuration loading usable without torch.+ from .routes import validate_route++ validate_route(values)+ resolved = canonical_json(values)+ return ResolvedConfig(+ path=source_path,+ source_digest=hashlib.blake2s(raw).hexdigest(),+ resolved_digest=hashlib.blake2s(resolved).hexdigest(),+ values=values,+ )diff --git a/src/solarwm/config/routes.py b/src/solarwm/config/routes.pynew file mode 100644index 0000000..aca4f8a--- /dev/null+++ b/src/solarwm/config/routes.py@@ -0,0 +1,113 @@+"""Canonical supported model, stage, and objective routes."""++from __future__ import annotations++from collections.abc import Mapping+from dataclasses import dataclass+from typing import Any++from solarwm.errors import ConfigurationError++_ALIASES = {+ "minimax_h3": "minimax_h3",+ "minimax-h3": "minimax_h3",+}+++@dataclass(frozen=True, order=True)+class Route:+ family: str+ stage: str+ causal_mode: str+ objective: str+ variant: str = ""++ @property+ def key(self) -> str:+ suffix = f":{self.variant}" if self.variant else ""+ return f"{self.family}:{self.stage}:{self.causal_mode}:{self.objective}{suffix}"+++_SUPPORTED = frozenset(+ {+ Route("minimax_h3", "stage0p5", "bidirectional", "flow_matching"),+ # Few-step flow-map distillation of a trained omni (Stage0.5) run, bidirectional.+ Route("minimax_h3", "stage0p5", "bidirectional", "anyflow_forward_map", "v1_5"),+ # Structure-guided DMD2 few-step distillation of an omni run from a released four-step LoRA.+ Route("minimax_h3", "stage0p5", "bidirectional", "dmd", "v2"),+ Route("minimax_h3", "stage1", "teacher_forcing", "anyflow_forward_map", "v1_5"),+ Route("minimax_h3", "stage2", "self_gradient_forcing", "flow_matching"),+ }+)+++def _mapping(config: Mapping[str, Any], key: str) -> Mapping[str, Any]:+ value = config.get(key, {})+ if not isinstance(value, Mapping):+ raise ConfigurationError(f"{key} must be a mapping")+ return value+++def resolve_route(config: Mapping[str, Any]) -> Route:+ model = _mapping(config, "model")+ train = _mapping(config, "train")+ family_raw = str(model.get("family", "")).strip().lower()+ family = _ALIASES.get(family_raw)+ if family is None:+ raise ConfigurationError(f"unsupported model.family {family_raw!r}")+ stage = str(train.get("stage", model.get("stage", ""))).strip().lower()+ causal = str(train.get("causal_mode", "")).strip().lower()+ objective = str(train.get("objective", "")).strip().lower()+ variant = str(train.get("objective_variant", "")).strip().lower()+ if objective == "anyflow_forward_map" and not variant:+ variant = "v1_5"+ if family == "ltx25_video" and objective == "flow_matching":+ objective = "native_rectified_flow"+ return Route(family, stage, causal, objective, variant)+++def validate_route(config: Mapping[str, Any]) -> Route:+ action = str(config.get("action", "")).lower()+ if action == "preencode":+ model = _mapping(config, "model")+ family_raw = str(model.get("family", "")).strip().lower()+ if family_raw not in _ALIASES:+ raise ConfigurationError(f"unsupported model.family {family_raw!r}")+ # Preencoding does not have a causal training route.+ return Route(_ALIASES[family_raw], "preencode", "none", "none")++ route = resolve_route(config)+ if route not in _SUPPORTED:+ allowed = sorted(item.key for item in _SUPPORTED if item.family == route.family)+ raise ConfigurationError(+ f"unsupported route {route.key!r}; allowed for {route.family}: {allowed}"+ )++ data = _mapping(config, "data")+ frames = int(data.get("pixel_frames", 0))+ if route.family in {"wan22_ti2v_5b", "wan22_i2v_a14b"}:+ allowed_frames = {81, 153} if route.stage == "stage0p5" else {81}+ if frames not in allowed_frames:+ raise ConfigurationError(+ f"{route.family} {route.stage} requires pixel_frames in "+ f"{sorted(allowed_frames)}, got {frames}"+ )+ elif route.family == "ltx25_video" and frames != 153:+ raise ConfigurationError("LTX-2.5 Stage0.5 requires 153 frames")+ elif route.family == "minimax_h3" and frames != 158:+ raise ConfigurationError("MiniMax-H3 Stage0.5 requires 158 frames")++ transform = str(model_value(config, "camera_translation_transform", "linear")).lower()+ if transform not in {"linear", "logd4"}:+ raise ConfigurationError("camera_translation_transform must be linear or logd4")+ if route.family == "minimax_h3" and transform != "logd4":+ raise ConfigurationError("MiniMax-H3 requires logd4")+ return route+++def model_value(config: Mapping[str, Any], key: str, default: Any = None) -> Any:+ return _mapping(config, "model").get(key, default)+++def supported_routes() -> tuple[Route, ...]:+ return tuple(sorted(_SUPPORTED))diff --git a/src/solarwm/data/__init__.py b/src/solarwm/data/__init__.pynew file mode 100644index 0000000..ff17fec--- /dev/null+++ b/src/solarwm/data/__init__.py@@ -0,0 +1,33 @@+"""Canonical indexes, deterministic sampling, cameras, and shard transports."""++from .archive import RawSample, RawSampleReader, TarShardReader+from .index import (+ IndexInventory,+ IndexRow,+ ensure_disjoint,+ inventory,+ read_index,+ resolve_index_path,+ select_index_rows,+)+from .sampling import CanonicalSampler, ReaderIdentity, SamplePlan, SamplingConfig+from .transport import LocalResolver, resolver_from_config++__all__ = [+ "CanonicalSampler",+ "IndexInventory",+ "IndexRow",+ "LocalResolver",+ "RawSample",+ "RawSampleReader",+ "ReaderIdentity",+ "SamplePlan",+ "SamplingConfig",+ "TarShardReader",+ "ensure_disjoint",+ "inventory",+ "read_index",+ "resolve_index_path",+ "resolver_from_config",+ "select_index_rows",+]diff --git a/src/solarwm/data/archive.py b/src/solarwm/data/archive.pynew file mode 100644index 0000000..7792a98--- /dev/null+++ b/src/solarwm/data/archive.py@@ -0,0 +1,150 @@+"""Random access to indexed members in uncompressed WebDataset tar shards."""++from __future__ import annotations++import json+import tarfile+import threading+from collections import OrderedDict+from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from pathlib import Path+from typing import Any++from solarwm.errors import DataContractError++from .index import IndexRow, validate_relative_key+from .sampling import SamplePlan+from .transport import ShardResolver+++class TarShardReader:+ """Small per-worker LRU of tar handles; never extracts paths to disk."""++ def __init__(self, resolver: ShardResolver, *, max_open: int = 4) -> None:+ if max_open < 1:+ raise DataContractError("tar max_open must be positive")+ self.resolver = resolver+ self.max_open = max_open+ self._handles: OrderedDict[Path, tarfile.TarFile] = OrderedDict()+ self._lock = threading.RLock()++ def close(self) -> None:+ with self._lock:+ while self._handles:+ _, handle = self._handles.popitem(last=False)+ handle.close()++ def __enter__(self) -> TarShardReader:+ return self++ def __exit__(self, *_: object) -> None:+ self.close()++ def _handle(self, row: IndexRow) -> tarfile.TarFile:+ path = self.resolver.resolve(row).resolve()+ handle = self._handles.pop(path, None)+ if handle is None:+ try:+ # Published shards are uncompressed tars. Strict mode prevents a+ # transport-specific decompressor from hiding format drift.+ handle = tarfile.open(path, "r:") # noqa: SIM115 - retained in the LRU+ except (OSError, tarfile.TarError) as exc:+ raise DataContractError(f"cannot open WDS shard {path}: {exc}") from exc+ self._handles[path] = handle+ while len(self._handles) > self.max_open:+ _, old = self._handles.popitem(last=False)+ old.close()+ return handle++ def read(self, row: IndexRow, member: Any) -> bytes:+ member_name = validate_relative_key(member, field="tar member")+ with self._lock:+ handle = self._handle(row)+ try:+ entry = handle.getmember(member_name)+ extracted = handle.extractfile(entry)+ except (KeyError, OSError, tarfile.TarError) as exc:+ raise DataContractError(+ f"missing member {member_name!r} for sample {row.sample_id!r}"+ ) from exc+ if extracted is None:+ raise DataContractError(+ f"member {member_name!r} for sample {row.sample_id!r} is not a file"+ )+ return extracted.read()+++@dataclass(frozen=True)+class RawSample:+ """Transport-neutral raw payload handed to a model-family codec."""++ plan: SamplePlan+ index_values: Mapping[str, Any]+ caption: str+ scene: str+ manifest: Mapping[str, Any]+ members: Mapping[str, bytes]+++class RawSampleReader:+ """Materialize bytes and metadata without performing model-specific encoding."""++ def __init__(+ self,+ rows: Sequence[IndexRow],+ shards: TarShardReader,+ *,+ member_fields: Sequence[str] = ("video_member", "camera_member"),+ ) -> None:+ self.rows = tuple(rows)+ self.shards = shards+ self.member_fields = tuple(member_fields)++ def materialize(self, plan: SamplePlan) -> RawSample:+ try:+ row = self.rows[plan.row_ordinal]+ except IndexError as exc:+ raise DataContractError(f"plan row ordinal is invalid: {plan.row_ordinal}") from exc+ if row.sample_id != plan.sample_id or row.key != plan.key or row.shard != plan.shard:+ raise DataContractError(f"sample plan identity drift for {plan.sample_id!r}")++ manifest = row.values.get("manifest")+ if manifest is None:+ name = row.values.get("manifest_member")+ if name is None:+ raise DataContractError(f"sample {row.sample_id!r} lacks manifest/manifest_member")+ try:+ manifest = json.loads(self.shards.read(row, name))+ except (json.JSONDecodeError, UnicodeDecodeError) as exc:+ raise DataContractError(+ f"sample {row.sample_id!r} has an invalid manifest member"+ ) from exc+ if not isinstance(manifest, Mapping):+ raise DataContractError(f"sample {row.sample_id!r} manifest is not an object")++ members: dict[str, bytes] = {}+ for field_name in self.member_fields:+ member_name = row.values.get(field_name)+ if member_name is None:+ raise DataContractError(f"sample {row.sample_id!r} lacks required {field_name}")+ members[field_name] = self.shards.read(row, member_name)++ metadata = manifest.get("metadata", {})+ prompt = manifest.get("prompt", {})+ scene = (+ metadata.get("scene", row.values.get("dataset", row.key))+ if isinstance(metadata, Mapping)+ else row.values.get("dataset", row.key)+ )+ caption = row.values.get("caption") or (+ prompt.get("text", "") if isinstance(prompt, Mapping) else ""+ )+ return RawSample(+ plan=plan,+ index_values=dict(row.values),+ caption=str(caption),+ scene=str(scene),+ manifest=dict(manifest),+ members=members,+ )diff --git a/src/solarwm/data/camera.py b/src/solarwm/data/camera.pynew file mode 100644index 0000000..a1c8019--- /dev/null+++ b/src/solarwm/data/camera.py@@ -0,0 +1,157 @@+"""Explicit camera conventions shared by raw, latent, train, and inference paths."""++from __future__ import annotations++import io+from dataclasses import dataclass+from typing import Literal++import numpy as np++from solarwm.errors import DataContractError++CameraStorage = Literal["absolute_c2w", "absolute_w2c", "relative_w2c"]+TranslationTransform = Literal["linear", "logd4"]+++class CameraGuardError(DataContractError):+ """A valid camera trajectory rejected by configured magnitude guards."""+++def camera_audit_prefix_frames(num_frames: int, fps: float, max_seconds: float = 10.0) -> int:+ """Apply the first-ten-second magnitude-guard window."""++ count = max(0, int(num_frames))+ if count == 0:+ return 0+ rate = float(fps)+ seconds = float(max_seconds)+ if not np.isfinite(rate) or rate <= 0 or not np.isfinite(seconds) or seconds <= 0:+ return count+ return min(count, max(1, int(np.ceil(rate * seconds))))+++def load_camera_npz(value: bytes, array_key: str) -> tuple[np.ndarray, CameraStorage]:+ """Load one explicitly declared camera array without changing its dtype."""++ key_contract: dict[str, CameraStorage] = {+ "c2w": "absolute_c2w",+ "w2c": "absolute_w2c",+ "vipe_c2w": "absolute_c2w",+ "vipe_w2c": "absolute_w2c",+ "relative_w2c": "relative_w2c",+ }+ if array_key not in key_contract:+ raise DataContractError(f"unsupported camera array key {array_key!r}")+ try:+ with np.load(io.BytesIO(value), allow_pickle=False) as archive:+ if array_key.startswith("vipe_") and "vipe_status" in archive.files:+ status = str(np.asarray(archive["vipe_status"]).reshape(-1)[0]).lower()+ if status != "ok":+ raise DataContractError(f"VIPE camera status is {status!r}, expected 'ok'")+ if array_key not in archive.files:+ raise DataContractError(f"camera archive lacks declared array {array_key!r}")+ matrices = np.asarray(archive[array_key])+ except (OSError, ValueError) as exc:+ raise DataContractError("camera member is not a valid non-pickled NPZ") from exc+ return _validate_matrices(matrices, name=array_key), key_contract[array_key]+++def _validate_matrices(value: np.ndarray, *, name: str) -> np.ndarray:+ matrices = np.asarray(value)+ if matrices.ndim != 3 or matrices.shape[1:] != (4, 4):+ raise DataContractError(f"{name} must have shape [T,4,4], got {matrices.shape}")+ if not np.isfinite(matrices).all():+ raise DataContractError(f"{name} contains non-finite values")+ return matrices+++def invert_se3(matrices: np.ndarray) -> np.ndarray:+ matrices = _validate_matrices(matrices, name="SE3")+ result = np.zeros_like(matrices)+ rotation = matrices[:, :3, :3]+ translation = matrices[:, :3, 3]+ rotation_t = np.swapaxes(rotation, -1, -2)+ result[:, :3, :3] = rotation_t+ result[:, :3, 3] = -np.einsum("tij,tj->ti", rotation_t, translation)+ result[:, 3, 3] = 1+ return result+++def absolute_c2w(value: np.ndarray, storage: CameraStorage) -> np.ndarray:+ matrices = _validate_matrices(value, name="camera")+ if storage == "absolute_c2w":+ return matrices+ if storage == "absolute_w2c":+ return invert_se3(matrices)+ raise DataContractError("relative_w2c cannot be converted back to authoritative absolute C2W")+++def relative_w2c(value: np.ndarray, storage: CameraStorage) -> np.ndarray:+ matrices = _validate_matrices(value, name="camera")+ if storage == "relative_w2c":+ relative = matrices.copy()+ else:+ c2w = absolute_c2w(matrices, storage)+ relative = np.matmul(invert_se3(c2w), c2w[0])+ identity = np.eye(4, dtype=relative.dtype)+ if not np.allclose(relative[0], identity, rtol=1e-5, atol=1e-5):+ raise DataContractError("first relative W2C row is not identity")+ return relative+++def relative_c2w(value: np.ndarray, storage: CameraStorage) -> np.ndarray:+ return invert_se3(relative_w2c(value, storage))+++def transform_translation(+ relative: np.ndarray,+ transform: TranslationTransform,+) -> np.ndarray:+ """Apply model-conditioning compression without mutating source camera data."""++ matrices = _validate_matrices(relative, name="relative camera")+ if transform == "linear":+ return matrices+ if transform != "logd4":+ raise DataContractError(f"unknown camera translation transform {transform!r}")+ result = matrices.copy()+ working_dtype = np.float64 if matrices.dtype == np.float64 else np.float32+ translation = matrices[:, :3, 3].astype(working_dtype, copy=False)+ norms = np.linalg.norm(translation, axis=-1)+ scale = np.zeros_like(norms)+ nonzero = norms != 0+ scale[nonzero] = np.log1p(norms[nonzero]) / (4.0 * norms[nonzero])+ result[:, :3, 3] = (translation * scale[:, None]).astype(result.dtype, copy=False)+ return result+++@dataclass(frozen=True)+class CameraGuards:+ max_rel_translation: float | None = 20.0+ max_camera_abs: float | None = 20.0++ def apply(self, relative: np.ndarray, *, audit_rows: int | None = None) -> np.ndarray:+ matrices = _validate_matrices(relative, name="relative camera")+ count = matrices.shape[0] if audit_rows is None else int(audit_rows)+ if not 1 <= count <= matrices.shape[0]:+ raise DataContractError("audit_rows is outside the camera trajectory")+ prefix = matrices[:count]+ if self.max_rel_translation is not None:+ magnitude = np.linalg.norm(prefix[:, :3, 3], axis=-1)+ if float(magnitude.max(initial=0.0)) > self.max_rel_translation:+ raise CameraGuardError("relative translation exceeds configured guard")+ result = matrices.copy()+ if self.max_camera_abs is not None and (+ float(np.abs(result[:count]).max(initial=0.0)) > self.max_camera_abs+ ):+ raise CameraGuardError("camera matrix exceeds configured absolute guard")+ return result+++def causal_pixel_indices(pixel_frames: int, temporal_stride: int) -> np.ndarray:+ if pixel_frames < 1 or temporal_stride < 1:+ raise DataContractError("pixel_frames and temporal_stride must be positive")+ if pixel_frames == 1:+ return np.asarray([0], dtype=np.int64)+ return np.asarray([0, *range(1, pixel_frames, temporal_stride)], dtype=np.int64)diff --git a/src/solarwm/data/index.py b/src/solarwm/data/index.pynew file mode 100644index 0000000..784cb92--- /dev/null+++ b/src/solarwm/data/index.py@@ -0,0 +1,313 @@+"""Ordered, transport-neutral WebDataset index contracts."""++from __future__ import annotations++import base64+import binascii+import gzip+import hashlib+import json+import random+import re+from collections.abc import Iterable, Iterator, Mapping+from dataclasses import dataclass+from pathlib import Path, PurePosixPath+from typing import Any, TextIO++from solarwm.errors import DataContractError+++def _open_text(path: Path) -> TextIO:+ if path.suffix == ".gz":+ return gzip.open(path, "rt", encoding="utf-8", newline="")+ return path.open("r", encoding="utf-8", newline="")+++_URI_SCHEME = re.compile(r"[A-Za-z][A-Za-z0-9+.-]*:")+++def validate_relative_key(value: Any, *, field: str = "shard") -> str:+ """Validate a portable path stored in a published index.++ Runtime roots may be absolute or object-store URIs. Index keys may not be;+ keeping that distinction is what makes one index portable between them.+ """++ if not isinstance(value, str) or not value or "\\" in value:+ raise DataContractError(f"{field} must be a non-empty POSIX path: {value!r}")+ if value.startswith("/") or _URI_SCHEME.match(value):+ raise DataContractError(f"{field} must be relative, got {value!r}")+ # PurePosixPath normalizes repeated separators and dot segments. Inspect+ # the authoritative bytes first so non-canonical spellings cannot alias.+ if any(part in {"", ".", ".."} for part in value.split("/")):+ raise DataContractError(f"{field} contains a non-portable segment: {value!r}")+ path = PurePosixPath(value)+ return path.as_posix()+++def resolve_index_path(data: Mapping[str, Any], field: str) -> Path:+ """Resolve a portable index key independently from shard transport.++ Recipe controls are normally staged locally even when shards stream from+ object storage. ``index_root`` states that control-plane location. A local+ all-in-one tree may omit it and use its payload root for both purposes.+ """++ key = validate_relative_key(data.get(field), field=f"data.{field}")+ raw_index_root = str(data.get("index_root") or "").strip()+ transport = data.get("transport", {})+ if not isinstance(transport, Mapping):+ raise DataContractError("data.transport must be a mapping")+ kind = str(transport.get("kind") or "").strip().lower()+ if raw_index_root:+ root = Path(raw_index_root).expanduser()+ if not root.is_absolute():+ raise DataContractError("data.index_root must be an absolute local path")+ return root / key+ if kind == "local":+ root = Path(str(transport.get("root") or "")).expanduser()+ if not root.is_absolute():+ raise DataContractError("local data.transport.root must be absolute")+ return root / key+ raise DataContractError(+ "staged recipe controls need an absolute local data.index_root"+ )+++@dataclass(frozen=True)+class IndexRow:+ """A lossless normalized view over one canonical JSONL row."""++ ordinal: int+ sample_id: str+ key: str+ shard: str+ epoch_repeats: int+ values: Mapping[str, Any]++ @classmethod+ def from_mapping(cls, ordinal: int, values: Mapping[str, Any]) -> IndexRow:+ if not isinstance(values, Mapping):+ raise DataContractError(f"index row {ordinal} is not an object")+ identities: dict[str, str] = {}+ for field in ("sample_id", "key"):+ value = values.get(field)+ if not isinstance(value, str) or not value:+ raise DataContractError(f"index row {ordinal} {field} must be a non-empty string")+ identities[field] = value+ sample_id = identities["sample_id"]+ key = identities["key"]+ shard = validate_relative_key(values.get("shard"))+ raw_repeats = values.get("epoch_repeats", 1)+ if type(raw_repeats) is not int:+ raise DataContractError(f"index row {ordinal} epoch_repeats must be an integer")+ if raw_repeats < 1:+ raise DataContractError(f"index row {ordinal} epoch_repeats must be >= 1")+ return cls(+ ordinal=ordinal,+ sample_id=sample_id,+ key=key,+ shard=shard,+ epoch_repeats=raw_repeats,+ values=dict(values),+ )+++@dataclass(frozen=True)+class IndexInventory:+ rows: int+ virtual_occurrences: int+ ordered_sample_id_digest: str+ ordered_row_digest: str+ decompressed_digest: str+++@dataclass(frozen=True)+class ShardIdentity:+ """Canonical transport metadata declared by one index row."""++ generation: str | None+ size: int+ md5_b64: str | None+ digest: str | None+++def shard_identity(row: IndexRow) -> ShardIdentity | None:+ """Parse and validate optional transport metadata on an index row.++ A size is required once any identity field is present. Digests remain+ optional publication metadata; runtime transport does not require them.+ Generation may be omitted for a local tree, while bucket resolvers impose+ it as an additional requirement.+ """++ values = row.values+ names = (+ "shard_generation",+ "shard_size",+ "shard_md5_b64",+ "shard_digest",+ )+ if not any(values.get(name) is not None and values.get(name) != "" for name in names):+ return None++ raw_size = values.get("shard_size")+ if raw_size is None or raw_size == "":+ raise DataContractError(+ f"sample {row.sample_id!r} has a partial shard identity: shard_size is required"+ )+ if isinstance(raw_size, bool):+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_size")+ if isinstance(raw_size, float):+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_size")+ try:+ size = int(raw_size)+ except (TypeError, ValueError) as exc:+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_size") from exc+ if size <= 0:+ raise DataContractError(+ f"sample {row.sample_id!r} has a partial shard identity: shard_size is required"+ )++ generation_value = values.get("shard_generation")+ generation_text = (+ str(generation_value).strip()+ if generation_value is not None and generation_value != ""+ else ""+ )+ generation = generation_text or None++ raw_md5 = values.get("shard_md5_b64")+ md5_b64 = str(raw_md5).strip() if raw_md5 is not None and raw_md5 != "" else None+ if md5_b64 is not None:+ try:+ decoded_md5 = base64.b64decode(md5_b64, validate=True)+ except (ValueError, binascii.Error) as exc:+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_md5_b64") from exc+ if len(decoded_md5) != hashlib.md5().digest_size:+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_md5_b64")+ md5_b64 = base64.b64encode(decoded_md5).decode("ascii")++ raw_digest = values.get("shard_digest")+ digest = (+ str(raw_digest).strip().lower() if raw_digest is not None and raw_digest != "" else None+ )+ if digest is not None:+ try:+ decoded_digest = bytes.fromhex(digest)+ except ValueError as exc:+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_digest") from exc+ if len(digest) != 64 or len(decoded_digest) != hashlib.blake2s().digest_size:+ raise DataContractError(f"sample {row.sample_id!r} has invalid shard_digest")++ return ShardIdentity(+ generation=generation,+ size=size,+ md5_b64=md5_b64,+ digest=digest,+ )+++def iter_index(path: str | Path) -> Iterator[IndexRow]:+ source = Path(path)+ with _open_text(source) as handle:+ ordinal = 0+ for line_number, line in enumerate(handle, 1):+ if not line.strip():+ continue+ try:+ values = json.loads(line)+ except json.JSONDecodeError as exc:+ raise DataContractError(f"invalid JSON at {source}:{line_number}: {exc}") from exc+ yield IndexRow.from_mapping(ordinal, values)+ ordinal += 1+++def read_index(path: str | Path) -> tuple[IndexRow, ...]:+ rows = tuple(iter_index(path))+ if not rows:+ raise DataContractError(f"empty index: {Path(path)}")+ seen: set[str] = set()+ duplicates: list[str] = []+ for row in rows:+ if row.sample_id in seen:+ duplicates.append(row.sample_id)+ seen.add(row.sample_id)+ if duplicates:+ preview = sorted(set(duplicates))[:8]+ raise DataContractError(f"duplicate sample_id values: {preview}")+ _validate_shard_identities(rows)+ return rows+++def select_index_rows(+ rows: tuple[IndexRow, ...],+ *,+ sample_count: int,+ seed: int,+) -> tuple[IndexRow, ...]:+ """Choose one reproducible validation subset from a recipe test index."""++ if isinstance(sample_count, bool) or not isinstance(sample_count, int) or sample_count < 1:+ raise DataContractError("validation sample_count must be a positive integer")+ if isinstance(seed, bool) or not isinstance(seed, int) or seed < 0:+ raise DataContractError("validation selection seed must be a non-negative integer")+ if len(rows) < sample_count:+ raise DataContractError(+ "recipe test index has fewer rows than validation.sample_count: "+ f"rows={len(rows)} sample_count={sample_count}"+ )+ indices = random.Random(seed).sample(range(len(rows)), sample_count)+ return tuple(rows[index] for index in indices)+++def _validate_shard_identities(rows: Iterable[IndexRow]) -> None:+ identities: dict[str, ShardIdentity | None] = {}+ for row in rows:+ identity = shard_identity(row)+ if row.shard in identities and identities[row.shard] != identity:+ raise DataContractError(f"shard identity drift inside index: {row.shard}")+ identities[row.shard] = identity+++def inventory(path: str | Path, rows: Iterable[IndexRow] | None = None) -> IndexInventory:+ source = Path(path)+ materialized = tuple(rows) if rows is not None else read_index(source)+ ordered = hashlib.blake2s()+ ordered_rows = hashlib.blake2s()+ for row in materialized:+ ordered.update(row.sample_id.encode("utf-8"))+ ordered.update(b"\n")+ ordered_rows.update(+ (+ json.dumps(+ row.values,+ allow_nan=False,+ ensure_ascii=False,+ separators=(",", ":"),+ sort_keys=True,+ )+ + "\n"+ ).encode()+ )+ decompressed = hashlib.blake2s()+ with _open_text(source) as handle:+ for line in handle:+ decompressed.update(line.encode("utf-8"))+ return IndexInventory(+ rows=len(materialized),+ virtual_occurrences=sum(row.epoch_repeats for row in materialized),+ ordered_sample_id_digest=ordered.hexdigest(),+ ordered_row_digest=ordered_rows.hexdigest(),+ decompressed_digest=decompressed.hexdigest(),+ )+++def ensure_disjoint(train: Iterable[IndexRow], test: Iterable[IndexRow]) -> None:+ train_ids = {row.sample_id for row in train}+ test_ids = {row.sample_id for row in test}+ overlap = train_ids & test_ids+ if overlap:+ raise DataContractError(+ f"train/test sample_id overlap ({len(overlap)}): {sorted(overlap)[:8]}"+ )diff --git a/src/solarwm/data/sampling.py b/src/solarwm/data/sampling.pynew file mode 100644index 0000000..af280f3--- /dev/null+++ b/src/solarwm/data/sampling.py@@ -0,0 +1,366 @@+"""Deterministic occurrence ownership, shuffle, and random starts.++The sampler is deliberately transport-free. A local directory and an object+store therefore produce the same plans for the same index and topology.+"""++from __future__ import annotations++import hashlib+import random+from collections.abc import Iterable, Iterator, Sequence+from dataclasses import dataclass+from pathlib import PurePosixPath+from typing import Literal++import numpy as np++from solarwm.errors import DataContractError++from .index import IndexRow++PartitionMode = Literal["global_occurrence", "node_shard"]+ShardPartitionScope = Literal["complete", "collection_relative"]+++@dataclass(frozen=True)+class SamplingConfig:+ seed: int+ pixel_frames: int+ random_start: bool = True+ fixed_start_from_index: bool = False+ clip_seconds: float | None = None+ output_fps: float | None = None+ shuffle_buffer: int = 4096+ partition_mode: PartitionMode = "global_occurrence"+ shard_partition_scope: ShardPartitionScope = "complete"++ def __post_init__(self) -> None:+ if self.pixel_frames < 1:+ raise DataContractError("pixel_frames must be positive")+ if self.shuffle_buffer < 1:+ raise DataContractError("shuffle_buffer must be positive")+ if self.shard_partition_scope not in {"complete", "collection_relative"}:+ raise DataContractError("unknown shard_partition_scope")+ if self.random_start and self.fixed_start_from_index:+ raise DataContractError(+ "random_start and fixed_start_from_index are mutually exclusive"+ )+ if (self.clip_seconds is None) != (self.output_fps is None):+ raise DataContractError("clip_seconds and output_fps must be set together")+ if self.clip_seconds is not None:+ expected = round(self.clip_seconds * float(self.output_fps)) + 1+ if expected != self.pixel_frames:+ raise DataContractError(+ f"clip_seconds/output_fps imply {expected} frames, expected {self.pixel_frames}"+ )+++@dataclass(frozen=True)+class ReaderIdentity:+ """One logical DP DataLoader worker, never a raw SP process."""++ rank: int+ world_size: int+ worker_id: int = 0+ num_workers: int = 1+ node_id: int = 0+ node_count: int = 1+ local_rank: int = 0+ local_world_size: int = 1++ def __post_init__(self) -> None:+ if not 0 <= self.rank < self.world_size:+ raise DataContractError("reader rank is outside world_size")+ if not 0 <= self.worker_id < self.num_workers:+ raise DataContractError("worker_id is outside num_workers")+ if not 0 <= self.node_id < self.node_count:+ raise DataContractError("node_id is outside node_count")+ if not 0 <= self.local_rank < self.local_world_size:+ raise DataContractError("local_rank is outside local_world_size")++ @classmethod+ def from_topology(+ cls,+ topology: object,+ *,+ worker_id: int = 0,+ num_workers: int = 1,+ ) -> ReaderIdentity:+ """Map raw ranks to logical DP readers so all SP peers are identical."""++ required = (+ "dp_rank",+ "dp_world_size",+ "node_id",+ "node_count",+ "local_dp_rank",+ "local_dp_world_size",+ )+ try:+ values = {name: int(getattr(topology, name)) for name in required}+ except (AttributeError, TypeError, ValueError) as exc:+ raise DataContractError("invalid distributed topology") from exc+ return cls(+ rank=values["dp_rank"],+ world_size=values["dp_world_size"],+ worker_id=worker_id,+ num_workers=num_workers,+ node_id=values["node_id"],+ node_count=values["node_count"],+ local_rank=values["local_dp_rank"],+ local_world_size=values["local_dp_world_size"],+ )++ @property+ def global_reader(self) -> int:+ return self.rank * self.num_workers + self.worker_id++ @property+ def total_readers(self) -> int:+ return self.world_size * self.num_workers++ @property+ def local_reader(self) -> int:+ return self.local_rank * self.num_workers + self.worker_id++ @property+ def local_readers(self) -> int:+ return self.local_world_size * self.num_workers+++@dataclass(frozen=True)+class Occurrence:+ row: IndexRow+ repeat_ordinal: int+++@dataclass(frozen=True)+class SamplePlan:+ sample_id: str+ key: str+ shard: str+ row_ordinal: int+ repeat_ordinal: int+ epoch: int+ start_frame: int+ source_frame_indices: tuple[int, ...]+ reader_rank: int+ worker_id: int+++def frame_offsets(config: SamplingConfig, source_fps: float) -> np.ndarray:+ if config.clip_seconds is None:+ return np.arange(config.pixel_frames, dtype=np.int64)+ fps = source_fps if source_fps > 0 else float(config.output_fps)+ offsets = np.rint(+ np.arange(config.pixel_frames, dtype=np.float64) * (fps / float(config.output_fps))+ ).astype(np.int64)+ return np.maximum.accumulate(offsets)+++class CanonicalSampler:+ """Produce deterministic healthy-path sample plans.++ Equality is scoped to the same topology and worker count. Topology changes+ intentionally repartition the canonical occurrence stream.+ """++ def __init__(+ self,+ rows: Sequence[IndexRow],+ config: SamplingConfig,+ identity: ReaderIdentity,+ ) -> None:+ if not rows:+ raise DataContractError("sampler needs at least one index row")+ self.rows = tuple(rows)+ self.config = config+ self.identity = identity+ self._rng = random.Random(config.seed + identity.global_reader * 100003)++ def iter_epoch(self, epoch: int) -> Iterator[SamplePlan]:+ if epoch < 0:+ raise DataContractError("epoch must be non-negative")+ if self.config.partition_mode == "node_shard":+ yield from self._iter_node_shard(epoch)+ else:+ yield from self._iter_global(epoch)++ def _plan(self, occurrence: Occurrence, epoch: int) -> SamplePlan:+ values = occurrence.row.values+ try:+ num_frames = int(values.get("num_frames") or values["manifest"]["video"]["num_frames"])+ except (KeyError, TypeError, ValueError) as exc:+ raise DataContractError(f"sample {occurrence.row.sample_id} lacks num_frames") from exc+ try:+ source_fps = float(+ values.get("fps") or values.get("manifest", {}).get("video", {}).get("fps") or 0.0+ )+ except (TypeError, ValueError) as exc:+ raise DataContractError(f"sample {occurrence.row.sample_id} has invalid fps") from exc+ offsets = frame_offsets(self.config, source_fps)+ max_offset = int(offsets[-1])+ if num_frames <= max_offset:+ raise DataContractError(+ f"sample {occurrence.row.sample_id} has {num_frames} frames, needs "+ f"an offset through {max_offset}"+ )+ if self.config.random_start:+ start = self._rng.randint(0, num_frames - max_offset - 1)+ elif self.config.fixed_start_from_index:+ try:+ start = int(values["start_frame"])+ except (KeyError, TypeError, ValueError) as exc:+ raise DataContractError(+ f"sample {occurrence.row.sample_id} lacks a valid fixed start_frame"+ ) from exc+ else:+ # A disabled random start selects the first valid frame.+ start = 0+ if start < 0 or start + max_offset >= num_frames:+ raise DataContractError(+ f"sample {occurrence.row.sample_id} fixed start {start} is outside "+ f"{num_frames} source frames"+ )+ indices = tuple(int(item) for item in start + offsets)+ return SamplePlan(+ sample_id=occurrence.row.sample_id,+ key=occurrence.row.key,+ shard=occurrence.row.shard,+ row_ordinal=occurrence.row.ordinal,+ repeat_ordinal=occurrence.repeat_ordinal,+ epoch=epoch,+ start_frame=start,+ source_frame_indices=indices,+ reader_rank=self.identity.rank,+ worker_id=self.identity.worker_id,+ )++ def _iter_global(self, epoch: int) -> Iterator[SamplePlan]:+ selected: list[Occurrence] = []+ virtual_index = 0+ for row in self.rows:+ for repeat in range(row.epoch_repeats):+ if virtual_index % self.identity.total_readers == self.identity.global_reader:+ selected.append(Occurrence(row, repeat))+ virtual_index += 1+ if not selected:+ raise DataContractError("this reader owns no virtual occurrences")++ # The production path streams through this finite buffer instead of+ # loading and globally shuffling the complete index.+ buffer: list[Occurrence] = []+ for occurrence in selected:+ if self.config.shuffle_buffer == 1:+ # Bypass ``randrange(1)`` to avoid consuming an RNG draw.+ # Even a one-element draw can advance Python's RNG state and+ # would therefore change the subsequent random start.+ yield self._plan(occurrence, epoch)+ else:+ buffer.append(occurrence)+ if len(buffer) >= self.config.shuffle_buffer:+ index = self._rng.randrange(len(buffer))+ yield self._plan(buffer.pop(index), epoch)+ while buffer:+ index = self._rng.randrange(len(buffer))+ yield self._plan(buffer.pop(index), epoch)++ def _iter_node_shard(self, epoch: int) -> Iterator[SamplePlan]:+ selected_groups = self.node_shard_groups(epoch)++ reader_cursor = 0+ emitted = 0+ for _, rows in selected_groups:+ occurrences: list[Occurrence] = []+ for row in rows:+ for repeat in range(row.epoch_repeats):+ if reader_cursor % self.identity.local_readers == self.identity.local_reader:+ occurrences.append(Occurrence(row, repeat))+ reader_cursor += 1+ self._rng.shuffle(occurrences)+ for occurrence in occurrences:+ emitted += 1+ yield self._plan(occurrence, epoch)+ if not emitted:+ raise DataContractError("this node-shard reader emitted no occurrences")++ def node_shard_groups(self, epoch: int) -> tuple[tuple[str, tuple[IndexRow, ...]], ...]:+ """Return the deterministic shard order shared by every reader on a node."""++ if self.config.partition_mode != "node_shard":+ raise DataContractError("node shard groups require partition_mode=node_shard")+ if epoch < 0:+ raise DataContractError("epoch must be non-negative")+ groups: list[tuple[str, list[IndexRow]]] = []+ current_shard = ""+ current: list[IndexRow] = []+ for row in self.rows:+ if current and row.shard != current_shard:+ groups.append((current_shard, current))+ current = []+ current_shard = row.shard+ current.append(row)+ if current:+ groups.append((current_shard, current))++ group_rng = random.Random(+ self.config.seed + self.identity.node_id * 1000003 + epoch * 1000000007+ )+ selected_groups: list[tuple[str, tuple[IndexRow, ...]]] = []+ group_buffer: list[tuple[str, list[IndexRow]]] = []+ for shard, rows in groups:+ partition_key = _shard_partition_key(shard, self.config.shard_partition_scope)+ assignment = (+ int.from_bytes(+ _stable_partition_digest(+ f"{self.config.seed}|{epoch}|{partition_key}".encode()+ )[:8],+ "big",+ )+ % self.identity.node_count+ )+ if assignment != self.identity.node_id:+ continue+ group_buffer.append((shard, rows))+ if len(group_buffer) >= self.config.shuffle_buffer:+ selected_shard, selected_rows = group_buffer.pop(+ group_rng.randrange(len(group_buffer))+ )+ selected_groups.append((selected_shard, tuple(selected_rows)))+ while group_buffer:+ selected_shard, selected_rows = group_buffer.pop(group_rng.randrange(len(group_buffer)))+ selected_groups.append((selected_shard, tuple(selected_rows)))+ return tuple(selected_groups)+++def _shard_partition_key(shard: str, scope: ShardPartitionScope) -> str:+ if scope == "complete":+ return shard+ parts = PurePosixPath(shard).parts+ shard_directory = max(+ (index for index, part in enumerate(parts) if part == "shards"),+ default=-1,+ )+ if shard_directory > 0:+ return "/".join(parts[shard_directory - 1 :])+ return shard+++def _stable_partition_digest(value: bytes) -> bytes:+ return hashlib.sha256(value).digest()+++def plan_fingerprint(plans: Iterable[SamplePlan]) -> str:+ digest = hashlib.blake2s()+ digest.update(b"solarwm.sample-plan.v1\n")+ for plan in plans:+ digest.update(+ (+ f"{plan.sample_id}\t{plan.key}\t{plan.shard}\t{plan.row_ordinal}\t"+ f"{plan.repeat_ordinal}\t{plan.epoch}\t{plan.start_frame}\t"+ f"{','.join(map(str, plan.source_frame_indices))}\t"+ f"{plan.reader_rank}\t{plan.worker_id}\n"+ ).encode()+ )+ return digest.hexdigest()diff --git a/src/solarwm/data/transport.py b/src/solarwm/data/transport.pynew file mode 100644index 0000000..42ceff8--- /dev/null+++ b/src/solarwm/data/transport.py@@ -0,0 +1,65 @@+"""Physical shard resolution below the canonical data plan.++The transport receives an :class:`~solarwm.data.index.IndexRow` only after+sample ownership, shuffle order, and frame starts have been decided, so where a+shard is read from cannot perturb the logical RNG stream.+"""++from __future__ import annotations++from dataclasses import dataclass+from pathlib import Path+from typing import Protocol++from solarwm.errors import DataContractError++from .index import IndexRow, ShardIdentity, shard_identity, validate_relative_key+++def join_root(root: str, relative_key: str) -> str:+ """Join a runtime root and portable index key without changing semantics."""++ key = validate_relative_key(relative_key)+ return str(Path(root).expanduser() / key)+++class ShardResolver(Protocol):+ """Resolve one immutable index row to a readable local file."""++ def resolve(self, row: IndexRow) -> Path: ...+++@dataclass(frozen=True)+class LocalResolver:+ """Resolve relative shard keys beneath a mounted directory."""++ root: Path++ def resolve(self, row: IndexRow) -> Path:+ target = (self.root / row.shard).resolve()+ root = self.root.resolve()+ if target != root and root not in target.parents:+ raise DataContractError(f"shard escapes local root: {row.shard}")+ if not target.is_file():+ raise DataContractError(f"local shard is missing: {target}")+ identity = shard_identity(row)+ if identity is not None:+ _verify_size(target, identity, label="local shard")+ return target+++def _verify_size(path: Path, identity: ShardIdentity, *, label: str) -> None:+ try:+ size = path.stat().st_size+ except OSError as exc:+ raise DataContractError(f"cannot stat {label} {path}: {exc}") from exc+ if size != identity.size:+ raise DataContractError(f"{label} size mismatch for {path}: {size} != {identity.size}")+++def resolver_from_config(root: str, *, cache_dir: str | None = None, max_gib: float = 256.0) -> ShardResolver:+ """The shard resolver for a run: shards are read from a local (or mounted) directory."""++ if cache_dir:+ raise DataContractError("data.cache_dir is not used: shards are read from a local directory")+ return LocalResolver(Path(root).expanduser())diff --git a/src/solarwm/errors.py b/src/solarwm/errors.pynew file mode 100644index 0000000..2e2767d--- /dev/null+++ b/src/solarwm/errors.py@@ -0,0 +1,21 @@+"""Framework-specific exceptions with actionable failure boundaries."""+++class SolarWMError(RuntimeError):+ """Base class for expected framework failures."""+++class ConfigurationError(SolarWMError):+ """The resolved run configuration violates a public contract."""+++class DataContractError(SolarWMError):+ """An index, sample, camera, or tensor violates the data contract."""+++class BackendContractError(SolarWMError):+ """A model backend cannot satisfy the requested route."""+++class CheckpointError(SolarWMError):+ """A checkpoint transaction or compatibility gate is invalid."""diff --git a/src/solarwm/inference/__init__.py b/src/solarwm/inference/__init__.pynew file mode 100644index 0000000..6e704e1--- /dev/null+++ b/src/solarwm/inference/__init__.py@@ -0,0 +1,31 @@+"""The generation engine shared verbatim by CLI inference and validation."""++from .comparison import (+ ComparisonValidationRecord,+ encode_compare_mp4,+ publish_comparison_complete,+ publish_comparison_partition,+)+from .engine import (+ GeneratedFile,+ GeneratedSample,+ InferenceAdapter,+ InferenceCase,+ InferenceEngine,+ InferenceSummary,+ run_validation,+)++__all__ = [+ "ComparisonValidationRecord",+ "GeneratedFile",+ "GeneratedSample",+ "InferenceAdapter",+ "InferenceCase",+ "InferenceEngine",+ "InferenceSummary",+ "encode_compare_mp4",+ "publish_comparison_complete",+ "publish_comparison_partition",+ "run_validation",+]diff --git a/src/solarwm/inference/comparison.py b/src/solarwm/inference/comparison.pynew file mode 100644index 0000000..2d5a1c3--- /dev/null+++ b/src/solarwm/inference/comparison.py@@ -0,0 +1,321 @@+"""Comparison-video visualization and publication contract."""++from __future__ import annotations++import hashlib+import json+import os+import re+import shutil+import subprocess+import tempfile+from collections.abc import Mapping+from dataclasses import dataclass+from pathlib import Path+from typing import Any, Literal++from solarwm.errors import BackendContractError+++@dataclass(frozen=True)+class ComparisonValidationRecord:+ slot: int+ compare_path: Path+ manifest_path: Path+++def _sample_slug(value: object, *, max_length: int = 160) -> str:+ text = str(value).strip()+ slug = re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._")+ if not slug:+ slug = hashlib.blake2s(text.encode()).hexdigest()[:16]+ if len(slug) <= max_length:+ return slug+ digest = hashlib.blake2s(text.encode()).hexdigest()[:16]+ return f"{slug[: max_length - len(digest) - 2]}__{digest}"+++def encode_compare_mp4(+ reference: Any,+ generated: Any,+ *,+ fps: float,+ layout: Literal["btchw", "bcthw"],+ value_range: Literal["zero_one", "minus_one_one"],+) -> bytes:+ """Encode the left-GT/right-GEN H.264 visualization."""++ import torch++ def canonical(value: Any, name: str) -> Any:+ if value.ndim != 5 or int(value.shape[0]) != 1:+ raise BackendContractError(+ f"validation compare {name} has unexpected shape {tuple(value.shape)}"+ )+ result = value if layout == "btchw" else value.permute(0, 2, 1, 3, 4)+ if int(result.shape[2]) != 3:+ raise BackendContractError(+ f"validation compare {name} is not RGB: {tuple(result.shape)}"+ )+ # Backends keep source pixels on CPU while decoded generations normally+ # remain on the local CUDA device. The comparison is encoded by+ # ffmpeg on the host, so normalize both sides on CPU before joining.+ result = result.detach().to(device="cpu", dtype=torch.float32)+ if value_range == "minus_one_one":+ result = result.mul(0.5).add(0.5)+ return torch.nan_to_num(result, nan=0.0, posinf=1.0, neginf=0.0).clamp_(0, 1)++ gt = canonical(reference, "GT")+ gen = canonical(generated, "GEN")+ frames = min(int(gt.shape[1]), int(gen.shape[1]))+ if frames < 1 or tuple(gt.shape[2:]) != tuple(gen.shape[2:]):+ raise BackendContractError(+ f"validation compare GT/GEN shapes differ: gt={tuple(gt.shape)} gen={tuple(gen.shape)}"+ )+ joined = torch.cat((gt[:, :frames], gen[:, :frames]), dim=-1)+ array = joined[0].permute(0, 2, 3, 1).mul(255.0).to(torch.uint8).cpu().numpy()+ height, width = int(array.shape[1]), int(array.shape[2])+ executable = shutil.which("ffmpeg")+ if executable is None:+ raise BackendContractError("validation compare requires ffmpeg")+ with tempfile.TemporaryDirectory(prefix="solarwm-validation-compare-") as directory:+ target = Path(directory) / "compare.mp4"+ command = [+ executable,+ "-hide_banner",+ "-loglevel",+ "error",+ "-f",+ "rawvideo",+ "-pix_fmt",+ "rgb24",+ "-s:v",+ f"{width}x{height}",+ "-r",+ f"{float(fps):g}",+ "-i",+ "pipe:0",+ "-an",+ "-c:v",+ "libx264",+ "-pix_fmt",+ "yuv420p",+ "-movflags",+ "+faststart",+ "-y",+ str(target),+ ]+ result = subprocess.run(command, input=array.tobytes(), capture_output=True, check=False)+ if result.returncode or not target.is_file():+ detail = result.stderr.decode("utf-8", errors="replace")[-2000:]+ raise BackendContractError(f"ffmpeg failed to encode validation compare: {detail}")+ return target.read_bytes()+++def _comparison_manifest(+ manifest: Mapping[str, Any],+ *,+ step: int,+ pass_name: str,+ compare_path: str,+ cases: int,+ dp_world_size: int,+ sp_size: int,+ logical_world_size_per_round: int,+) -> dict[str, Any]:+ case = manifest["case"]+ metadata = case.get("metadata", {})+ generation_pass = metadata.get("generation_pass", {})+ slot = int(case["slot"])+ dp_rank = slot % int(dp_world_size)+ round_index = slot // int(logical_world_size_per_round)+ shape = manifest.get("shape", ())+ output_pixel_frames = int(shape[1]) if len(shape) >= 2 else None+ if metadata.get("output_pixel_frames") is not None:+ output_pixel_frames = int(metadata["output_pixel_frames"])+ rollout_latent_frames = generation_pass.get(+ "output_rollout_latent_frames",+ generation_pass.get("rollout_latent_frames", metadata.get("rollout_latent_frames")),+ )+ comparison = {+ "rank": dp_rank * int(sp_size),+ "global_rank": dp_rank * int(sp_size),+ "dp_rank": dp_rank,+ "sp_rank": 0,+ "sp_size": int(sp_size),+ "logical_validation_rank": slot,+ "logical_rank_within_round": slot % int(logical_world_size_per_round),+ "logical_validation_world_size": int(cases),+ "logical_world_size_per_round": int(logical_world_size_per_round),+ "validation_round_index": round_index,+ "validation_num_rounds": int(cases) // int(logical_world_size_per_round),+ "source_pixel_frames": metadata.get("source_pixel_frames"),+ "source_num_frames": metadata.get("source_num_frames"),+ "validation_noise_seed": int(case["noise_seed"]),+ "step": int(step),+ "video_id": metadata.get("key", case["sample_id"]),+ "dataset": metadata.get("dataset"),+ "scene": metadata.get("scene"),+ "start_frame": int(case["start_frame"]),+ "T_pix": output_pixel_frames,+ "T_lat": rollout_latent_frames,+ "train_T_lat": metadata.get("train_latent_frames"),+ "rollout_T_lat": rollout_latent_frames,+ "collective_rollout_T_lat": generation_pass.get("rollout_latent_frames"),+ "variable_rollout_by_source": bool(+ generation_pass.get("variable_rollout_by_source", False)+ ),+ "variable_rollout_start_adjusted": bool(+ metadata.get("variable_rollout_start_adjusted", False)+ ),+ "generation_mode": generation_pass.get("mode", metadata.get("generation_mode")),+ "active_pass_name": pass_name,+ "inference_mode": generation_pass.get("solver", metadata.get("inference_mode")),+ "camera_translation_transform": metadata.get("camera_translation_transform"),+ "sample_solver": generation_pass.get("solver", metadata.get("sample_solver")),+ "num_inference_steps": generation_pass.get(+ "num_inference_steps", metadata.get("num_inference_steps")+ ),+ "weights_source": generation_pass.get("weights", metadata.get("weights_source")),+ "requested_video_id": metadata.get("requested_key"),+ "requested_start_frame": metadata.get("requested_start_frame"),+ "validation_input_status": metadata.get("validation_input_status", "requested"),+ "artifact_valid": metadata.get("artifact_valid", True) is not False,+ "compare_mp4": compare_path,+ "video_stats": manifest.get("metrics", {}),+ "generated_latents_finite": manifest.get("metrics", {}).get("finite_fraction") == 1.0,+ "error": metadata.get("recoverable_input_error"),+ "solarwm_manifest": manifest,+ }+ denoising_step_list = manifest.get("provenance", {}).get("denoising_step_list")+ if denoising_step_list is not None:+ comparison["denoising_step_list"] = [int(value) for value in denoising_step_list]+ return comparison+++def publish_comparison_partition(+ source: Path,+ destination: Path,+ *,+ step: int,+ pass_name: str,+ cases: int,+ dp_world_size: int,+ sp_size: int,+ run_root: Path,+ logical_world_size_per_round: int | None = None,+) -> tuple[ComparisonValidationRecord, ...]:+ """Merge one node/rank partition into the common comparison directory."""++ cases_per_round = (+ int(cases) if logical_world_size_per_round is None else int(logical_world_size_per_round)+ )+ if int(cases) < 1 or cases_per_round < 1 or int(cases) % cases_per_round:+ raise BackendContractError("comparison validation cases must form complete logical rounds")++ compare_dir = destination / "compare"+ manifest_dir = destination / "manifests"+ compare_dir.mkdir(parents=True, exist_ok=True)+ manifest_dir.mkdir(parents=True, exist_ok=True)+ records = []+ for sample_dir in sorted(source.glob("slot-*")):+ manifest = json.loads((sample_dir / "manifest.json").read_text(encoding="utf-8"))+ case = manifest["case"]+ metadata = case.get("metadata", {})+ slot = int(case["slot"])+ key = _sample_slug(metadata.get("key") or case["sample_id"])+ start = int(case["start_frame"])+ dp_rank = slot % int(dp_world_size)+ round_index = slot // cases_per_round+ filename = (+ f"rank{slot:03d}_round{round_index:02d}_dp{dp_rank:03d}_{key}_start{start:06d}.mp4"+ )+ source_compare = sample_dir / "compare.mp4"+ if not source_compare.is_file():+ raise BackendContractError(f"validation slot {slot} lacks compare.mp4")+ compare_path = compare_dir / filename+ try:+ os.link(source_compare, compare_path)+ except FileExistsError as exc:+ raise BackendContractError(+ f"comparison validation compare already exists: {compare_path}"+ ) from exc+ relative = compare_path.relative_to(run_root).as_posix()+ comparison_manifest = _comparison_manifest(+ manifest,+ step=step,+ pass_name=pass_name,+ compare_path=relative,+ cases=cases,+ dp_world_size=dp_world_size,+ sp_size=sp_size,+ logical_world_size_per_round=cases_per_round,+ )+ manifest_path = manifest_dir / f"rank_{slot:03d}.json"+ payload = (json.dumps(comparison_manifest, ensure_ascii=False, indent=2) + "\n").encode()+ descriptor = os.open(+ manifest_path,+ os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_WRONLY,+ 0o444,+ )+ try:+ with os.fdopen(descriptor, "wb", closefd=True) as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ except Exception:+ compare_path.unlink(missing_ok=True)+ manifest_path.unlink(missing_ok=True)+ raise+ records.append(ComparisonValidationRecord(slot, compare_path, manifest_path))+ if not records:+ raise BackendContractError(f"comparison validation source has no slots: {source}")+ return tuple(records)+++def publish_comparison_complete(+ destination: Path,+ *,+ step: int,+ pass_name: str,+ local_slots: int,+ global_slots: int,+) -> None:+ payload = (+ json.dumps(+ {+ "schema": "solarwm.validation-comparison.v1",+ "step": int(step),+ "pass": pass_name,+ "local_slots": int(local_slots),+ "global_slots": int(global_slots),+ "compare_layout": "ground_truth_left_generated_right",+ },+ allow_nan=False,+ ensure_ascii=False,+ separators=(",", ":"),+ sort_keys=True,+ )+ + "\n"+ ).encode()+ target = destination / "COMPLETE.json"+ if target.exists():+ return+ temporary = target.with_name(+ f".{target.name}.{hashlib.blake2s(payload).hexdigest()[:16]}.partial"+ )+ temporary.write_bytes(payload)+ try:+ os.link(temporary, target)+ except FileExistsError:+ pass+ finally:+ temporary.unlink(missing_ok=True)+++__all__ = [+ "ComparisonValidationRecord",+ "encode_compare_mp4",+ "publish_comparison_complete",+ "publish_comparison_partition",+]diff --git a/src/solarwm/inference/engine.py b/src/solarwm/inference/engine.pynew file mode 100644index 0000000..4146dfe--- /dev/null+++ b/src/solarwm/inference/engine.py@@ -0,0 +1,295 @@+"""Atomic, fixed-plan generation used by both inference and validation."""++from __future__ import annotations++import hashlib+import stat+import uuid+from collections.abc import Callable, Mapping, Sequence+from dataclasses import asdict, dataclass, field+from pathlib import Path, PurePosixPath+from typing import Any, Protocol++from solarwm.errors import BackendContractError+from solarwm.runtime.create_only import (+ link_file_create_only,+ publish_directory_no_replace,+ write_file_create_only,+)+from solarwm.runtime.serialization import canonical_json_bytes++_RESERVED_SAMPLE_PATHS = frozenset({"manifest.json"})+++def _canonical_artifact_path(name: object) -> PurePosixPath:+ if not isinstance(name, str):+ raise BackendContractError(f"invalid generated artifact {name!r}")+ raw_parts = name.split("/")+ if (+ not name+ or "\\" in name+ or "\x00" in name+ or any(part in {"", ".", ".."} for part in raw_parts)+ ):+ raise BackendContractError(f"invalid generated artifact {name!r}")+ path = PurePosixPath(name)+ if path.is_absolute() or path.as_posix() != name:+ raise BackendContractError(f"invalid generated artifact {name!r}")+ if path.parts[0] in _RESERVED_SAMPLE_PATHS:+ raise BackendContractError(f"generated artifact path is reserved: {name!r}")+ return path+++@dataclass(frozen=True)+class InferenceCase:+ slot: int+ sample_id: str+ prompt: str+ start_frame: int+ noise_seed: int+ camera_fingerprint: str+ metadata: Mapping[str, Any] = field(default_factory=dict)++ def __post_init__(self) -> None:+ if self.slot < 0 or self.start_frame < 0 or self.noise_seed < 0:+ raise BackendContractError("inference slot/start/noise values must be non-negative")+ if not self.sample_id or not self.camera_fingerprint:+ raise BackendContractError("inference case lacks sample/camera identity")+++@dataclass(frozen=True)+class GeneratedFile:+ """A generated artifact already materialized on the output filesystem.++ Large videos use this path-backed form so the inference engine never has to+ hold the complete encoded artifact in memory. The producer must place the+ file on the same filesystem as ``runtime.output_dir``; the engine publishes+ it with a create-only hard link and then removes the private source name.+ """++ path: Path+ delete_after_publish: bool = True++ def __post_init__(self) -> None:+ path = Path(self.path).expanduser()+ try:+ mode = path.lstat().st_mode+ except OSError as exc:+ raise BackendContractError(f"generated file is not readable: {path}: {exc}") from exc+ if not stat.S_ISREG(mode) or path.is_symlink():+ raise BackendContractError(f"generated file must be a regular non-symlink file: {path}")+ object.__setattr__(self, "path", path.resolve())+++@dataclass(frozen=True)+class GeneratedSample:+ artifacts: Mapping[str, bytes | GeneratedFile]+ shape: tuple[int, ...]+ dtype: str+ metrics: Mapping[str, float] = field(default_factory=dict)+ provenance: Mapping[str, Any] = field(default_factory=dict)++ def __post_init__(self) -> None:+ if not self.artifacts or not self.shape or not self.dtype:+ raise BackendContractError("generated sample lacks artifacts/shape/dtype")+ paths: list[PurePosixPath] = []+ for name, value in self.artifacts.items():+ path = _canonical_artifact_path(name)+ if not isinstance(value, (bytes, GeneratedFile)):+ raise BackendContractError(f"invalid generated artifact {name!r}")+ if any(+ path == other or path in other.parents or other in path.parents for other in paths+ ):+ raise BackendContractError(f"generated artifact paths collide: {name!r}")+ paths.append(path)+++class InferenceAdapter(Protocol):+ family: str++ def generate(self, case: InferenceCase, *, weights_id: str) -> GeneratedSample: ...+++@dataclass(frozen=True)+class InferenceSummary:+ output_dir: Path+ weights_id: str+ cases: int+ ordered_manifest_digest: str+++def _write_file(path: Path, value: bytes) -> None:+ write_file_create_only(+ path,+ value,+ error_type=BackendContractError,+ label="inference output file",+ )+++def _file_identity(path: Path) -> tuple[int, str]:+ digest = hashlib.blake2s()+ size = 0+ try:+ with path.open("rb") as handle:+ for chunk in iter(lambda: handle.read(1024 * 1024), b""):+ size += len(chunk)+ digest.update(chunk)+ except OSError as exc:+ raise BackendContractError(f"cannot read generated file {path}: {exc}") from exc+ return size, digest.hexdigest()+++class InferenceEngine:+ """One implementation for standalone generation and training validation."""++ def __init__(self, adapter: InferenceAdapter) -> None:+ self.adapter = adapter++ def run(+ self,+ cases: Sequence[InferenceCase],+ *,+ weights_id: str,+ output_dir: str | Path,+ collective_error: Callable[[str | None, str], None] | None = None,+ collective_case_waves: int | None = None,+ ) -> InferenceSummary:+ if not weights_id:+ raise BackendContractError("inference requires a stable weights_id")+ if not cases and collective_case_waves is None:+ raise BackendContractError("inference requires cases")+ slots = [case.slot for case in cases]+ if len(slots) != len(set(slots)):+ raise BackendContractError("inference case slots are duplicated")+ waves = len(cases) if collective_case_waves is None else int(collective_case_waves)+ if waves < 1:+ raise BackendContractError("inference collective case waves must be positive")+ if waves < len(cases):+ raise BackendContractError(+ "inference collective case waves are shorter than local cases"+ )+ target = Path(output_dir).resolve()+ staging: Path | None = None+ setup_error: str | None = None+ try:+ if target.exists():+ raise BackendContractError(f"inference output already exists: {target}")+ target.parent.mkdir(parents=True, exist_ok=True)+ staging = target.with_name(f".{target.name}.{uuid.uuid4().hex}.partial")+ staging.mkdir()+ except Exception as exc:+ setup_error = f"{type(exc).__name__}: {exc}"+ if collective_error is not None:+ collective_error(setup_error, "output setup")+ if setup_error is not None or staging is None:+ raise BackendContractError(f"inference output setup failed: {setup_error}")+ manifests: list[dict[str, Any]] = []+ for case_index in range(waves):+ case_error: str | None = None+ manifest: dict[str, Any] | None = None+ if case_index < len(cases):+ case = cases[case_index]+ generated: GeneratedSample | None = None+ try:+ generated = self.adapter.generate(case, weights_id=weights_id)+ sample_dir = staging / f"slot-{case.slot:06d}"+ sample_dir.mkdir()+ artifact_records: list[dict[str, Any]] = []+ for name, value in sorted(generated.artifacts.items()):+ artifact = sample_dir / name+ if isinstance(value, GeneratedFile):+ size, digest = _file_identity(value.path)+ link_file_create_only(+ value.path,+ artifact,+ error_type=BackendContractError,+ label="inference generated file",+ )+ else:+ _write_file(artifact, value)+ size = len(value)+ digest = hashlib.blake2s(value).hexdigest()+ artifact_records.append(+ {+ "path": f"slot-{case.slot:06d}/{name}",+ "bytes": size,+ "digest": digest,+ }+ )+ manifest = {+ "schema": "solarwm.inference-sample.v1",+ "family": self.adapter.family,+ "weights_id": weights_id,+ "case": asdict(case),+ "shape": list(generated.shape),+ "dtype": generated.dtype,+ "metrics": dict(generated.metrics),+ "provenance": dict(generated.provenance),+ "artifacts": artifact_records,+ }+ _write_file(+ sample_dir / "manifest.json",+ canonical_json_bytes(manifest),+ )+ except Exception as exc:+ case_error = f"{type(exc).__name__}: {exc}"+ finally:+ if generated is not None:+ for value in generated.artifacts.values():+ if isinstance(value, GeneratedFile) and value.delete_after_publish:+ value.path.unlink(missing_ok=True)+ if collective_error is not None:+ collective_error(case_error, f"case wave {case_index}")+ if case_error is not None:+ raise BackendContractError(f"inference case wave {case_index} failed: {case_error}")+ if case_index < len(cases):+ if manifest is None:+ raise BackendContractError(+ f"inference case wave {case_index} returned no manifest"+ )+ manifests.append(manifest)++ ordered_digest = ""+ commit_error: str | None = None+ try:+ ordered_bytes = b"".join(canonical_json_bytes(item) for item in manifests)+ ordered_digest = hashlib.blake2s(ordered_bytes).hexdigest()+ _write_file(staging / "ordered-manifest.jsonl", ordered_bytes)+ _write_file(+ staging / "COMPLETE.json",+ canonical_json_bytes(+ {+ "schema": "solarwm.inference-complete.v1",+ "family": self.adapter.family,+ "weights_id": weights_id,+ "cases": len(cases),+ "ordered_manifest_digest": ordered_digest,+ }+ ),+ )+ publish_directory_no_replace(+ staging,+ target,+ error_type=BackendContractError,+ label="inference output",+ )+ except Exception as exc:+ commit_error = f"{type(exc).__name__}: {exc}"+ if collective_error is not None:+ collective_error(commit_error, "partition commit")+ if commit_error is not None:+ raise BackendContractError(f"inference partition commit failed: {commit_error}")+ return InferenceSummary(target, weights_id, len(cases), ordered_digest)+++def run_validation(+ adapter: InferenceAdapter,+ cases: Sequence[InferenceCase],+ *,+ weights_id: str,+ output_dir: str | Path,+) -> InferenceSummary:+ """Training validation is deliberately only a named call to inference."""++ return InferenceEngine(adapter).run(cases, weights_id=weights_id, output_dir=output_dir)diff --git a/src/solarwm/inference/validation_plan.py b/src/solarwm/inference/validation_plan.pynew file mode 100644index 0000000..e2fa9ed--- /dev/null+++ b/src/solarwm/inference/validation_plan.py@@ -0,0 +1,167 @@+"""Immutable validation-case plans shared across training validations."""++from __future__ import annotations++import hashlib+import json+from collections.abc import Mapping, Sequence+from dataclasses import asdict+from pathlib import Path+from typing import Any++from solarwm.errors import BackendContractError+from solarwm.runtime.create_only import write_file_create_only+from solarwm.runtime.serialization import canonical_json_bytes++from .engine import InferenceCase++_SCHEMA = "solarwm.validation-plan.v1"+++def validation_plan_key(backend: str, config: Mapping[str, Any]) -> str:+ """Bind a frozen plan to every config field that can affect case identity."""++ data = config.get("data")+ if isinstance(data, Mapping):+ data = dict(data)+ payload = {+ "schema": "solarwm.validation-plan-key.v1",+ "backend": str(backend),+ "model": config.get("model"),+ "data": data,+ "distributed": config.get("distributed"),+ "validation": config.get("validation"),+ }+ return hashlib.sha256(canonical_json_bytes(payload)).hexdigest()+++def _validated_cases(raw: object, *, expected_count: int) -> tuple[InferenceCase, ...]:+ if not isinstance(raw, list) or len(raw) != expected_count:+ raise BackendContractError(+ "frozen validation plan case count differs: "+ f"expected={expected_count} actual={len(raw) if isinstance(raw, list) else 'invalid'}"+ )+ cases: list[InferenceCase] = []+ for item in raw:+ if not isinstance(item, Mapping) or set(item) != {+ "slot",+ "sample_id",+ "prompt",+ "start_frame",+ "noise_seed",+ "camera_fingerprint",+ "metadata",+ }:+ raise BackendContractError("frozen validation plan contains an invalid case")+ metadata = item["metadata"]+ if not isinstance(metadata, Mapping):+ raise BackendContractError("frozen validation case metadata must be a mapping")+ cases.append(+ InferenceCase(+ slot=int(item["slot"]),+ sample_id=str(item["sample_id"]),+ prompt=str(item["prompt"]),+ start_frame=int(item["start_frame"]),+ noise_seed=int(item["noise_seed"]),+ camera_fingerprint=str(item["camera_fingerprint"]),+ metadata=dict(metadata),+ )+ )+ if [case.slot for case in cases] != list(range(expected_count)):+ raise BackendContractError("frozen validation plan slots are not contiguous and ordered")+ return tuple(cases)+++def validation_plan_payload(+ *,+ backend: str,+ plan_key: str,+ cases: Sequence[InferenceCase],+) -> bytes:+ ordered = tuple(cases)+ if not backend or len(plan_key) != 64:+ raise BackendContractError("frozen validation plan lacks a backend/config identity")+ if [case.slot for case in ordered] != list(range(len(ordered))):+ raise BackendContractError(+ "validation cases must be contiguous and ordered before freezing"+ )+ body = {+ "schema": _SCHEMA,+ "backend": backend,+ "plan_key": plan_key,+ "case_count": len(ordered),+ "cases": [asdict(case) for case in ordered],+ }+ body_bytes = canonical_json_bytes(body)+ return canonical_json_bytes(+ {+ **body,+ "payload_digest": hashlib.sha256(body_bytes).hexdigest(),+ }+ )+++def load_validation_plan(+ path: str | Path,+ *,+ backend: str,+ plan_key: str,+ expected_count: int,+) -> tuple[InferenceCase, ...] | None:+ source = Path(path)+ if not source.exists():+ return None+ if not source.is_file() or source.is_symlink():+ raise BackendContractError(f"frozen validation plan is not a regular file: {source}")+ try:+ value = json.loads(source.read_bytes())+ except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:+ raise BackendContractError(f"cannot read frozen validation plan {source}: {exc}") from exc+ if not isinstance(value, Mapping) or value.get("schema") != _SCHEMA:+ raise BackendContractError("frozen validation plan schema differs")+ if value.get("backend") != backend or value.get("plan_key") != plan_key:+ raise BackendContractError("frozen validation plan config identity differs")+ if int(value.get("case_count", -1)) != expected_count:+ raise BackendContractError("frozen validation plan declared case count differs")+ payload_digest = value.get("payload_digest")+ body = {key: item for key, item in value.items() if key != "payload_digest"}+ actual_digest = hashlib.sha256(canonical_json_bytes(body)).hexdigest()+ if payload_digest != actual_digest:+ raise BackendContractError("frozen validation plan payload digest differs")+ return _validated_cases(value.get("cases"), expected_count=expected_count)+++def publish_validation_plan(+ path: str | Path,+ *,+ backend: str,+ plan_key: str,+ cases: Sequence[InferenceCase],+) -> str:+ """Create the immutable plan, or accept an identical race winner."""++ target = Path(path)+ payload = validation_plan_payload(backend=backend, plan_key=plan_key, cases=cases)+ try:+ write_file_create_only(+ target,+ payload,+ error_type=BackendContractError,+ label="frozen validation plan",+ )+ except BackendContractError:+ try:+ existing = target.read_bytes()+ except OSError:+ raise+ if existing != payload:+ raise+ return hashlib.sha256(payload).hexdigest()+++__all__ = [+ "load_validation_plan",+ "publish_validation_plan",+ "validation_plan_key",+ "validation_plan_payload",+]diff --git a/src/solarwm/preencode/__init__.py b/src/solarwm/preencode/__init__.pynew file mode 100644index 0000000..52d8a4e--- /dev/null+++ b/src/solarwm/preencode/__init__.py@@ -0,0 +1,21 @@+"""Versioned model-codec contracts and deterministic preencoded shards."""++from .contracts import (+ EncodedPayload,+ EncoderContract,+ OnlineCodec,+ TensorSpec,+ validate_encoded_tensors,+)+from .shards import ShardReceipt, write_index, write_shard++__all__ = [+ "EncodedPayload",+ "EncoderContract",+ "OnlineCodec",+ "ShardReceipt",+ "TensorSpec",+ "validate_encoded_tensors",+ "write_index",+ "write_shard",+]diff --git a/src/solarwm/preencode/contracts.py b/src/solarwm/preencode/contracts.pynew file mode 100644index 0000000..c65419a--- /dev/null+++ b/src/solarwm/preencode/contracts.py@@ -0,0 +1,159 @@+"""The same codec contract is used by offline production and online training."""++from __future__ import annotations++import hashlib+from collections.abc import Mapping+from dataclasses import asdict, dataclass, field+from typing import Any, Protocol++from solarwm.config.loader import canonical_json+from solarwm.errors import DataContractError+++@dataclass(frozen=True)+class TensorSpec:+ name: str+ shape: tuple[int | None, ...]+ dtype: str++ def __post_init__(self) -> None:+ if not self.name or not self.shape or not self.dtype:+ raise DataContractError("tensor spec requires name, shape, and dtype")+ if any(value is not None and value < 0 for value in self.shape):+ raise DataContractError(f"tensor spec {self.name!r} has a negative dimension")+++@dataclass(frozen=True)+class EncoderContract:+ schema: str+ family: str+ format_version: str+ pixel_frames: int+ latent_frames: int+ height: int+ width: int+ camera_convention: str+ tensors: tuple[TensorSpec, ...]+ extras: Mapping[str, Any] = field(default_factory=dict)++ def __post_init__(self) -> None:+ if self.schema != "solarwm.encoder.v1":+ raise DataContractError("encoder schema must be solarwm.encoder.v1")+ if not self.family or not self.format_version or not self.camera_convention:+ raise DataContractError("encoder family/format/camera convention are required")+ if min(self.pixel_frames, self.latent_frames, self.height, self.width) < 1:+ raise DataContractError("encoder dimensions must be positive")+ names = [spec.name for spec in self.tensors]+ if not names or len(names) != len(set(names)):+ raise DataContractError("encoder tensor names must be non-empty and unique")++ def as_dict(self) -> dict[str, Any]:+ return {+ **asdict(self),+ "tensors": [asdict(spec) for spec in self.tensors],+ }++ @property+ def digest(self) -> str:+ return hashlib.blake2s(canonical_json(self.as_dict())).hexdigest()+++class OnlineCodec(Protocol):+ """Model-owned raw-to-tensor implementation shared with offline encoding."""++ @property+ def contract(self) -> EncoderContract: ...++ def encode(+ self,+ *,+ sample_id: str,+ pixels: Any,+ caption: str,+ camera: Any,+ seed: int,+ ) -> Mapping[str, Any]: ...+++def _dtype_name(value: Any) -> str:+ dtype = getattr(value, "dtype", None)+ if dtype is None:+ raise DataContractError(f"tensor {type(value).__name__} has no dtype")+ rendered = str(dtype).lower()+ for prefix in ("torch.", "numpy."):+ if rendered.startswith(prefix):+ rendered = rendered[len(prefix) :]+ return rendered+++def validate_encoded_tensors(values: Mapping[str, Any], contract: EncoderContract) -> None:+ expected = {spec.name: spec for spec in contract.tensors}+ if set(values) != set(expected):+ raise DataContractError(+ f"encoded tensor fields differ: got {sorted(values)}, expected {sorted(expected)}"+ )+ for name, spec in expected.items():+ value = values[name]+ try:+ shape = tuple(int(dimension) for dimension in value.shape)+ except (AttributeError, TypeError, ValueError) as exc:+ raise DataContractError(f"encoded field {name!r} has no valid shape") from exc+ if len(shape) != len(spec.shape) or any(+ expected_dimension is not None and actual != expected_dimension+ for actual, expected_dimension in zip(shape, spec.shape, strict=True)+ ):+ raise DataContractError(+ f"encoded field {name!r} shape {shape} != contract {spec.shape}"+ )+ dtype = _dtype_name(value)+ if dtype != spec.dtype.lower():+ raise DataContractError(f"encoded field {name!r} dtype {dtype!r} != {spec.dtype!r}")+ tensor_isfinite = getattr(value, "isfinite", None)+ if callable(tensor_isfinite):+ finite = tensor_isfinite()+ else:+ try:+ import numpy as np++ finite = np.isfinite(value)+ except (TypeError, ValueError) as exc:+ raise DataContractError(+ f"encoded field {name!r} cannot be checked for finiteness"+ ) from exc+ finite_all = finite.all()+ if hasattr(finite_all, "item"):+ finite_all = finite_all.item()+ if not bool(finite_all):+ raise DataContractError(f"encoded field {name!r} contains non-finite values")+++@dataclass(frozen=True)+class EncodedPayload:+ """Serialized sample ready for deterministic tar packing."""++ sample_id: str+ key: str+ source_sample_id: str+ start_frame: int+ source_frame_indices: tuple[int, ...]+ encoder_contract_digest: str+ members: Mapping[str, bytes]+ metadata: Mapping[str, Any] = field(default_factory=dict)++ def __post_init__(self) -> None:+ if not self.sample_id or not self.key or not self.source_sample_id:+ raise DataContractError("encoded payload lacks sample/key/source identity")+ if self.start_frame < 0 or not self.source_frame_indices:+ raise DataContractError("encoded payload has an invalid frame window")+ if self.source_frame_indices[0] != self.start_frame:+ raise DataContractError("encoded payload start differs from first source frame")+ if len(self.encoder_contract_digest) != 64:+ raise DataContractError(+ "encoded payload has an invalid encoder contract content digest"+ )+ if not self.members:+ raise DataContractError("encoded payload has no serialized members")+ for suffix, value in self.members.items():+ if not suffix or "/" in suffix or suffix in {".", ".."} or not isinstance(value, bytes):+ raise DataContractError(f"invalid encoded member suffix {suffix!r}")diff --git a/src/solarwm/preencode/shards.py b/src/solarwm/preencode/shards.pynew file mode 100644index 0000000..ae5c6f7--- /dev/null+++ b/src/solarwm/preencode/shards.py@@ -0,0 +1,155 @@+"""Deterministic uncompressed tar and relative-index production."""++from __future__ import annotations++import base64+import hashlib+import io+import os+import tarfile+import uuid+from collections.abc import Mapping, Sequence+from dataclasses import dataclass+from pathlib import Path+from typing import Any++from solarwm.config.loader import canonical_json+from solarwm.data.index import validate_relative_key+from solarwm.errors import DataContractError++from .contracts import EncodedPayload+++@dataclass(frozen=True)+class ShardReceipt:+ relative_path: str+ samples: int+ size: int+ digest: str+ md5_b64: str+ rows: tuple[Mapping[str, Any], ...]+++def _digest(path: Path) -> tuple[str, str]:+ digest = hashlib.blake2s()+ md5 = hashlib.md5(usedforsecurity=False)+ with path.open("rb") as handle:+ while chunk := handle.read(16 * 1024 * 1024):+ digest.update(chunk)+ md5.update(chunk)+ return digest.hexdigest(), base64.b64encode(md5.digest()).decode()+++def _tar_member(archive: tarfile.TarFile, name: str, value: bytes) -> None:+ info = tarfile.TarInfo(name)+ info.size = len(value)+ info.mtime = 0+ info.mode = 0o644+ info.uid = 0+ info.gid = 0+ info.uname = ""+ info.gname = ""+ archive.addfile(info, io.BytesIO(value))+++def write_shard(+ root: str | Path,+ relative_path: str,+ samples: Sequence[EncodedPayload],+) -> ShardReceipt:+ """Write one shard without overwriting any existing output."""++ relative = validate_relative_key(relative_path)+ if not relative.endswith(".tar"):+ raise DataContractError("preencoded shards must use the .tar suffix")+ if not samples:+ raise DataContractError("cannot write an empty preencoded shard")+ ids = [sample.sample_id for sample in samples]+ if len(ids) != len(set(ids)):+ raise DataContractError("preencoded shard contains duplicate sample IDs")++ target = Path(root).resolve() / relative+ if target.exists():+ raise DataContractError(f"preencoded shard already exists: {target}")+ target.parent.mkdir(parents=True, exist_ok=True)+ temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.partial")+ rows: list[dict[str, Any]] = []+ try:+ with tarfile.open(temporary, "w:", format=tarfile.USTAR_FORMAT) as archive:+ for sample in samples:+ prefix = f"samples/{hashlib.blake2s(sample.sample_id.encode()).hexdigest()[:24]}"+ members: dict[str, str] = {}+ for suffix, value in sorted(sample.members.items()):+ member = f"{prefix}/{suffix}"+ _tar_member(archive, member, value)+ members[suffix] = member+ provenance = {+ "schema": "solarwm.preencoded-sample.v1",+ "sample_id": sample.sample_id,+ "key": sample.key,+ "source_sample_id": sample.source_sample_id,+ "start_frame": sample.start_frame,+ "source_frame_indices": list(sample.source_frame_indices),+ "encoder_contract_digest": sample.encoder_contract_digest,+ "members": members,+ "metadata": dict(sample.metadata),+ }+ provenance_member = f"{prefix}/provenance.json"+ _tar_member(archive, provenance_member, canonical_json(provenance))+ rows.append(+ {+ "sample_id": sample.sample_id,+ "key": sample.key,+ "source_sample_id": sample.source_sample_id,+ "shard": relative,+ "epoch_repeats": 1,+ "start_frame": sample.start_frame,+ "source_frame_indices": list(sample.source_frame_indices),+ "encoder_contract_digest": sample.encoder_contract_digest,+ "members": members,+ "provenance_member": provenance_member,+ "metadata": dict(sample.metadata),+ }+ )+ with temporary.open("rb") as handle:+ os.fsync(handle.fileno())+ os.replace(temporary, target)+ finally:+ temporary.unlink(missing_ok=True)+ digest, md5_b64 = _digest(target)+ size = target.stat().st_size+ for row in rows:+ row["shard_size"] = size+ row["shard_digest"] = digest+ row["shard_md5_b64"] = md5_b64+ return ShardReceipt(relative, len(samples), size, digest, md5_b64, tuple(rows))+++def write_index(path: str | Path, rows: Sequence[Mapping[str, Any]]) -> str:+ """Atomically write ordered JSONL and return its content digest."""++ if not rows:+ raise DataContractError("cannot write an empty preencoded index")+ sample_ids: set[str] = set()+ payload = bytearray()+ for row in rows:+ sample_id = str(row.get("sample_id") or "")+ if not sample_id or sample_id in sample_ids:+ raise DataContractError(f"missing or duplicate sample_id {sample_id!r}")+ sample_ids.add(sample_id)+ validate_relative_key(str(row.get("shard") or ""))+ payload.extend(canonical_json(dict(row)))+ target = Path(path)+ if target.exists():+ raise DataContractError(f"preencoded index already exists: {target}")+ target.parent.mkdir(parents=True, exist_ok=True)+ temporary = target.with_name(f".{target.name}.{uuid.uuid4().hex}.partial")+ try:+ with temporary.open("wb") as handle:+ handle.write(payload)+ handle.flush()+ os.fsync(handle.fileno())+ os.replace(temporary, target)+ finally:+ temporary.unlink(missing_ok=True)+ return hashlib.blake2s(payload).hexdigest()diff --git a/src/solarwm/runtime/__init__.py b/src/solarwm/runtime/__init__.pynew file mode 100644index 0000000..94acb60--- /dev/null+++ b/src/solarwm/runtime/__init__.py@@ -0,0 +1,10 @@+"""Runtime topology, environment, and deterministic seed utilities."""++from .randomness import RNGIdentity, rng_identity+from .topology import Topology++__all__ = [+ "RNGIdentity",+ "Topology",+ "rng_identity",+]diff --git a/src/solarwm/runtime/create_only.py b/src/solarwm/runtime/create_only.pynew file mode 100644index 0000000..ef43f81--- /dev/null+++ b/src/solarwm/runtime/create_only.py@@ -0,0 +1,179 @@+"""Atomic create-only directory publication for local staging trees."""++from __future__ import annotations++import ctypes+import errno+import fcntl+import os+import stat+import uuid+from pathlib import Path+++def _raise(error_type: type[Exception], message: str) -> None:+ raise error_type(message)+++def _rename_no_replace(+ source: Path,+ destination: Path,+ *,+ error_type: type[Exception],+ label: str,+) -> None:+ """Call Linux renameat2 with RENAME_NOREPLACE and no unsafe fallback."""++ libc = ctypes.CDLL(None, use_errno=True)+ renameat2 = getattr(libc, "renameat2", None)+ if renameat2 is None:+ _raise(+ error_type,+ f"{label} requires Linux renameat2(RENAME_NOREPLACE)",+ )+ renameat2.argtypes = (+ ctypes.c_int,+ ctypes.c_char_p,+ ctypes.c_int,+ ctypes.c_char_p,+ ctypes.c_uint,+ )+ renameat2.restype = ctypes.c_int+ result = renameat2(+ -100, # AT_FDCWD+ os.fsencode(source),+ -100,+ os.fsencode(destination),+ 1, # RENAME_NOREPLACE+ )+ if result == 0:+ return+ error = ctypes.get_errno()+ if error in {errno.EEXIST, errno.ENOTEMPTY}:+ _raise(error_type, f"{label} target appeared during publication: {destination}")+ _raise(error_type, f"{label} create-only publication failed: {os.strerror(error)}")+++def publish_directory_no_replace(+ source: str | Path,+ destination: str | Path,+ *,+ error_type: type[Exception],+ label: str,+) -> None:+ """Publish one sibling staging directory without replacing any target.++ The parent advisory lock serializes cooperating writers. Linux+ ``RENAME_NOREPLACE`` is still mandatory so an uncooperative writer racing+ after the check cannot be overwritten.+ """++ raw_staging = Path(source).expanduser()+ raw_target = Path(destination).expanduser()+ staging = raw_staging.parent.resolve() / raw_staging.name+ target = raw_target.parent.resolve() / raw_target.name+ if not staging.is_dir() or staging.is_symlink():+ _raise(error_type, f"{label} staging directory is missing or is a symlink")+ if staging.parent != target.parent:+ _raise(error_type, f"{label} staging and target must be siblings")+ lock_path = target.parent / ".solarwm-directory-publication.lock"+ with lock_path.open("a+b") as lock:+ fcntl.flock(lock.fileno(), fcntl.LOCK_EX)+ try:+ _rename_no_replace(+ staging,+ target,+ error_type=error_type,+ label=label,+ )+ directory_fd = os.open(target.parent, os.O_RDONLY)+ try:+ os.fsync(directory_fd)+ finally:+ os.close(directory_fd)+ finally:+ fcntl.flock(lock.fileno(), fcntl.LOCK_UN)+++def write_file_create_only(+ path: str | Path,+ payload: bytes,+ *,+ error_type: type[Exception],+ label: str,+ mode: int = 0o644,+) -> None:+ """Atomically create one immutable file without replacing a race winner.++ The payload is fsynced under a private sibling name, then hard-linked to+ the public name. ``link(2)`` is the commit point and fails when the target+ already exists; unlike ``os.replace`` it can never overwrite a prior artifact.+ """++ destination = Path(path).expanduser()+ destination.parent.mkdir(parents=True, exist_ok=True)+ temporary = destination.with_name(f".{destination.name}.{uuid.uuid4().hex}.partial")+ descriptor: int | None = None+ try:+ descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, mode)+ view = memoryview(payload)+ while view:+ written = os.write(descriptor, view)+ if written <= 0:+ _raise(error_type, f"{label} made no progress while writing {temporary}")+ view = view[written:]+ os.fsync(descriptor)+ os.close(descriptor)+ descriptor = None+ try:+ os.link(temporary, destination)+ except FileExistsError:+ _raise(error_type, f"{label} already exists: {destination}")+ temporary.unlink()+ directory_fd = os.open(destination.parent, os.O_RDONLY)+ try:+ os.fsync(directory_fd)+ finally:+ os.close(directory_fd)+ finally:+ if descriptor is not None:+ os.close(descriptor)+ temporary.unlink(missing_ok=True)+++def link_file_create_only(+ source: str | Path,+ destination: str | Path,+ *,+ error_type: type[Exception],+ label: str,+) -> None:+ """Atomically publish an existing immutable file without copying its bytes."""++ source_path = Path(source).expanduser()+ target = Path(destination).expanduser()+ try:+ source_stat = source_path.stat(follow_symlinks=False)+ except OSError as exc:+ _raise(error_type, f"{label} source is not readable: {source_path}: {exc}")+ if not stat.S_ISREG(source_stat.st_mode) or source_path.is_symlink():+ _raise(error_type, f"{label} source must be a regular non-symlink file: {source_path}")+ target.parent.mkdir(parents=True, exist_ok=True)+ try:+ os.link(source_path, target, follow_symlinks=False)+ except FileExistsError:+ _raise(error_type, f"{label} already exists: {target}")+ except OSError as exc:+ _raise(error_type, f"{label} create-only link failed: {exc}")+ directory_fd = os.open(target.parent, os.O_RDONLY)+ try:+ os.fsync(directory_fd)+ finally:+ os.close(directory_fd)+++__all__ = [+ "link_file_create_only",+ "publish_directory_no_replace",+ "write_file_create_only",+]diff --git a/src/solarwm/runtime/distributed.py b/src/solarwm/runtime/distributed.pynew file mode 100644index 0000000..f638fa6--- /dev/null+++ b/src/solarwm/runtime/distributed.py@@ -0,0 +1,254 @@+"""Small fail-closed checks around model-family distributed process groups."""++from __future__ import annotations++import hashlib+import time+from collections.abc import Callable, Mapping, Sequence+from datetime import timedelta+from typing import Any, TypeVar++from solarwm.config.loader import canonical_json+from solarwm.errors import BackendContractError++T = TypeVar("T")+++class DataReadiness:+ """Finish remote batch reads on every rank before entering GPU collectives.++ All raw ranks construct and call this at the same reader boundary. The+ separate CPU group keeps storage latency out of the NCCL watchdog window;+ it does not select samples, advance RNGs, or move batch tensors.+ """++ def __init__(self, *, enabled: bool) -> None:+ self.group: Any = None+ if not enabled:+ return+ import torch+ import torch.distributed as dist++ if not dist.is_initialized() or dist.get_world_size() == 1:+ return+ self.torch = torch+ self.dist = dist+ self.group = dist.new_group(backend="gloo", timeout=timedelta(hours=2))++ def read(self, call: Callable[[], T]) -> T:+ if self.group is None:+ return call()+ started = time.monotonic()+ result: T | None = None+ local_error: Exception | None = None+ try:+ result = call()+ except Exception as exc:+ local_error = exc+ rank = self.dist.get_rank()+ world = self.dist.get_world_size()+ status = self.torch.tensor(+ [rank if local_error is not None else world], dtype=self.torch.int64, device="cpu"+ )+ self.dist.all_reduce(status, op=self.dist.ReduceOp.MIN, group=self.group)+ failed_rank = int(status.item())+ if failed_rank != world:+ messages = [+ f"{type(local_error).__name__}: {local_error}" if rank == failed_rank else ""+ ]+ self.dist.broadcast_object_list(messages, src=failed_rank, group=self.group)+ raise BackendContractError(+ f"training data read failed on rank {failed_rank}: {messages[0]}"+ ) from local_error+ elapsed = time.monotonic() - started+ if elapsed >= 30:+ print(+ f"[data-readiness][rank{rank}] all batches ready after {elapsed:.1f}s", flush=True+ )+ return result # type: ignore[return-value]+++def _collective_coordinates(+ dist: Any | None,+ *,+ rank: int | None,+ world_size: int | None,+) -> tuple[int, int, bool]:+ """Resolve one raw-world collective without importing Torch.++ Callers pass ``torch.distributed`` in heavy runtimes and a small fake in+ CPU tests. An explicitly multi-rank call may never silently degrade to a+ local check when the process group is unavailable.+ """++ initialized = bool(dist is not None and dist.is_initialized())+ resolved_rank = (+ int(dist.get_rank()) if rank is None and initialized else int(0 if rank is None else rank)+ )+ resolved_world = (+ int(dist.get_world_size())+ if world_size is None and initialized+ else int(1 if world_size is None else world_size)+ )+ if resolved_world < 1 or not 0 <= resolved_rank < resolved_world:+ raise BackendContractError(+ "collective failure propagation received an invalid rank/world_size"+ )+ if resolved_world > 1 and not initialized:+ raise BackendContractError(+ "multi-rank failure propagation requires initialized torch.distributed"+ )+ if initialized:+ actual_rank = int(dist.get_rank())+ actual_world = int(dist.get_world_size())+ if (resolved_rank, resolved_world) != (actual_rank, actual_world):+ raise BackendContractError(+ "collective failure propagation rank/world_size differs from the process group"+ )+ return resolved_rank, resolved_world, initialized+++def propagate_collective_error(+ local_error: str | None,+ *,+ dist: Any | None,+ label: str,+ rank: int | None = None,+ world_size: int | None = None,+ error_type: type[Exception] = BackendContractError,+) -> None:+ """Raise the same rank-ordered failure on every raw-world peer.++ Only strings cross the process-group boundary. Exception instances are+ deliberately never pickled or broadcast. Every peer must call this at+ the same phase boundary.+ """++ resolved_rank, resolved_world, initialized = _collective_coordinates(+ dist,+ rank=rank,+ world_size=world_size,+ )+ message = str(local_error or "")+ local = (resolved_rank, message)+ gathered: list[Any] = [local]+ if initialized and resolved_world > 1:+ gathered = [None] * resolved_world+ dist.all_gather_object(gathered, local)++ normalized: list[tuple[int, str]] = []+ for item in gathered:+ if not isinstance(item, (tuple, list)) or len(item) != 2 or isinstance(item[0], bool):+ raise error_type(f"{label} failure exchange returned a malformed rank record")+ try:+ peer_rank = int(item[0])+ except (TypeError, ValueError) as exc:+ raise error_type(f"{label} failure exchange returned a malformed rank value") from exc+ peer_error = str(item[1] or "")+ if not 0 <= peer_rank < resolved_world:+ raise error_type(f"{label} failure exchange returned an out-of-range rank")+ normalized.append((peer_rank, peer_error))+ if len(normalized) != resolved_world or {item[0] for item in normalized} != set(+ range(resolved_world)+ ):+ raise error_type(f"{label} failure exchange did not cover every raw rank")+ failures = [(peer_rank, error) for peer_rank, error in sorted(normalized) if error]+ if failures:+ detail = " | ".join(f"rank {peer_rank}: {error}" for peer_rank, error in failures)+ raise error_type(f"{label} failed collectively: {detail}")+++def collective_call(+ call: Callable[[], T],+ *,+ dist: Any | None,+ label: str,+ rank: int | None = None,+ world_size: int | None = None,+ error_type: type[Exception] = BackendContractError,+) -> T:+ """Run rank-local work, then exchange errors before any later collective."""++ result: T | None = None+ local_exception: Exception | None = None+ local_error = ""+ try:+ result = call()+ except Exception as exc:+ local_exception = exc+ local_error = f"{type(exc).__name__}: {exc}"+ try:+ propagate_collective_error(+ local_error,+ dist=dist,+ label=label,+ rank=rank,+ world_size=world_size,+ error_type=error_type,+ )+ except Exception as exc:+ if local_exception is not None:+ raise exc from local_exception+ raise+ return result # type: ignore[return-value]+++def collective_rank_zero_call(+ call: Callable[[], T],+ *,+ dist: Any | None,+ label: str,+ rank: int | None = None,+ world_size: int | None = None,+ error_type: type[Exception] = BackendContractError,+) -> T:+ """Run work on raw rank zero and broadcast only its successful value."""++ resolved_rank, resolved_world, initialized = _collective_coordinates(+ dist,+ rank=rank,+ world_size=world_size,+ )+ result = collective_call(+ call if resolved_rank == 0 else lambda: None, # type: ignore[arg-type]+ dist=dist,+ label=label,+ rank=resolved_rank,+ world_size=resolved_world,+ error_type=error_type,+ )+ values: list[Any] = [result if resolved_rank == 0 else None]+ if initialized and resolved_world > 1:+ dist.broadcast_object_list(values, src=0)+ return values[0] # type: ignore[return-value]+++def identity_fingerprint(value: Mapping[str, Any]) -> str:+ return hashlib.blake2s(canonical_json(value)).hexdigest()+++def assert_peer_fingerprints(values: Sequence[str]) -> str:+ if not values or any(not value for value in values):+ raise BackendContractError("SP peer fingerprints are missing")+ if len(set(values)) != 1:+ raise BackendContractError(f"SP peers loaded different sample/RNG identities: {values}")+ return values[0]+++def gather_and_assert_sp_identity(+ value: Mapping[str, Any], *, sp_size: int, group: Any = None+) -> str:+ """Collect identity hashes in the already-created SP group and compare."""++ local = identity_fingerprint(value)+ if sp_size == 1:+ return local+ try:+ import torch.distributed as dist+ except ImportError as exc:+ raise BackendContractError("torch is required for sequence parallelism") from exc+ if not dist.is_available() or not dist.is_initialized():+ raise BackendContractError("SP identity check requires initialized torch.distributed")+ peers: list[str | None] = [None] * sp_size+ dist.all_gather_object(peers, local, group=group)+ return assert_peer_fingerprints([str(value or "") for value in peers])diff --git a/src/solarwm/runtime/images.py b/src/solarwm/runtime/images.pynew file mode 100644index 0000000..63b5867--- /dev/null+++ b/src/solarwm/runtime/images.py@@ -0,0 +1,39 @@+"""Runtime environment inspection."""++from __future__ import annotations++import importlib.metadata+import sys+from typing import Any+++def probe_python_runtime() -> dict[str, Any]:+ """Return a serializable in-container probe without importing a backend."""++ result: dict[str, Any] = {+ "python": sys.executable,+ "python_version": sys.version.split()[0],+ }+ try:+ import torch+ except ImportError:+ result.update({"torch": None, "cuda": None, "cuda_available": False})+ else:+ result.update(+ {+ "torch": torch.__version__,+ "cuda": torch.version.cuda,+ "cuda_available": torch.cuda.is_available(),+ "nccl": list(torch.cuda.nccl.version())+ if torch.cuda.is_available() and torch.distributed.is_nccl_available()+ else None,+ }+ )+ packages: dict[str, str] = {}+ for name in ("diffusers", "transformers", "flash-attn", "peft", "safetensors"):+ try:+ packages[name] = importlib.metadata.version(name)+ except importlib.metadata.PackageNotFoundError:+ continue+ result["packages"] = packages+ return resultdiff --git a/src/solarwm/runtime/output_layout.py b/src/solarwm/runtime/output_layout.pynew file mode 100644index 0000000..a72a85e--- /dev/null+++ b/src/solarwm/runtime/output_layout.py@@ -0,0 +1,211 @@+"""Canonical output paths for training, validation, and standalone inference."""++from __future__ import annotations++import os+import re+import shutil+from collections.abc import Mapping+from dataclasses import dataclass+from pathlib import Path, PurePosixPath+from typing import Any++from solarwm.errors import BackendContractError++CAMERA_DATASET_TRIPLET_LAYOUT = "dataset_triplet_v1"+CAMERA_TRANSACTION_LAYOUT = "transaction_v1"+_PORTABLE_COMPONENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*")+++@dataclass(frozen=True)+class CameraInferenceOutputLayout:+ """Resolved standalone camera-inference publication and provenance roots."""++ publish_root: Path+ run_root: Path+ run_id: str+ layout: str+++def portable_output_component(value: object, *, field: str) -> str:+ """Validate one collision-free, cross-platform output path component."""++ component = str(value or "")+ if not _PORTABLE_COMPONENT.fullmatch(component) or component in {".", ".."}:+ raise BackendContractError(+ f"{field} must be a portable ASCII path component, got {component!r}"+ )+ return component+++def camera_inference_output_layout(+ config: Mapping[str, Any],+) -> CameraInferenceOutputLayout | None:+ """Resolve the opt-out dataset-triplet layout for camera-length inference.++ All non-camera routes return ``None`` and therefore preserve their existing+ output semantics. Camera-length inference may explicitly select+ ``transaction_v1`` to retain the original generation transaction layout.+ """++ inference = config.get("inference", {})+ runtime = config.get("runtime", {})+ if not isinstance(inference, Mapping) or not isinstance(runtime, Mapping):+ raise BackendContractError("inference and runtime must be mappings")+ camera_length = (+ str(config.get("action", "")).strip().lower() == "infer"+ and str(inference.get("length", "fixed")).strip().lower() == "camera"+ )+ if not camera_length:+ return None+ layout = str(inference.get("output_layout", CAMERA_DATASET_TRIPLET_LAYOUT)).strip().lower()+ if layout not in {CAMERA_DATASET_TRIPLET_LAYOUT, CAMERA_TRANSACTION_LAYOUT}:+ raise BackendContractError(+ "camera-length inference.output_layout must be "+ f"{CAMERA_DATASET_TRIPLET_LAYOUT} or {CAMERA_TRANSACTION_LAYOUT}"+ )+ if layout == CAMERA_TRANSACTION_LAYOUT:+ return None+ raw_root = str(runtime.get("output_dir", "")).strip()+ publish_root = Path(raw_root).expanduser()+ if not raw_root or not publish_root.is_absolute():+ raise BackendContractError(+ "camera-length dataset publication requires an absolute runtime.output_dir"+ )+ run_id = portable_output_component(+ inference.get("run_id", config.get("name")),+ field="inference.run_id",+ )+ publish_root = publish_root.resolve()+ return CameraInferenceOutputLayout(+ publish_root=publish_root,+ run_root=publish_root / "runs" / run_id,+ run_id=run_id,+ layout=layout,+ )+++def invocation_output_dir(config: Mapping[str, Any]) -> Path:+ """Return the directory that owns one invocation's provenance files."""++ layout = camera_inference_output_layout(config)+ if layout is not None:+ return layout.run_root+ runtime = config.get("runtime", {})+ if not isinstance(runtime, Mapping):+ raise BackendContractError("runtime must be a mapping")+ raw_root = str(runtime.get("output_dir", "")).strip()+ if not raw_root:+ raise BackendContractError("runtime.output_dir is required")+ return Path(raw_root).expanduser()+++def _run_root(output_dir: str | Path) -> Path:+ root = Path(output_dir).expanduser()+ if not root.is_absolute():+ raise BackendContractError("training output directory must be absolute")+ return root.resolve()+++def checkpoint_model_dir(+ output_dir: str | Path,+ *,+ step: int,+ width: int,+) -> Path:+ """Return the stable top-level public checkpoint directory."""++ if isinstance(step, bool) or not isinstance(step, int) or step < 1:+ raise BackendContractError("checkpoint step must be a positive integer")+ if width not in {6, 8}:+ raise BackendContractError("checkpoint step width must be 6 or 8")+ return _run_root(output_dir) / f"checkpoint_model_{step:0{width}d}"+++def validation_pass_component(value: object) -> str:+ """Validate and return one validation-pass path component."""++ pass_name = str(value or "").strip()+ path = PurePosixPath(pass_name)+ if (+ not pass_name+ or pass_name in {".", ".."}+ or "/" in pass_name+ or "\\" in pass_name+ or path.parts != (pass_name,)+ ):+ raise BackendContractError("validation pass names must be portable path components")+ return pass_name+++def public_validation_dir(+ output_dir: str | Path,+ *,+ step: int,+ pass_name: str,+) -> Path:+ """Return the one public validation directory for a step/pass pair."""++ if isinstance(step, bool) or not isinstance(step, int) or step < 0:+ raise BackendContractError("validation step must be a non-negative integer")+ pass_name = validation_pass_component(pass_name)+ return _run_root(output_dir) / "validation" / f"step_{step:06d}_{pass_name}"+++def validation_staging_root(output_dir: str | Path) -> Path:+ """Private, non-publishable staging root for rank-local validation output."""++ return _run_root(output_dir) / "validation" / ".staging"+++def cleanup_validation_staging(path: str | Path, *, output_dir: str | Path) -> None:+ """Remove one successful staging subtree, never anything outside ``.staging``.++ Callers invoke this only after their distributed output and completion gates+ pass. Failed validations deliberately retain their staging tree for diagnosis.+ """++ staging_root = validation_staging_root(output_dir)+ target = Path(os.path.abspath(Path(path).expanduser()))+ if target == staging_root or staging_root not in target.parents:+ raise BackendContractError(+ f"refusing to clean validation path outside private staging: {target}"+ )++ current = target+ while current != staging_root:+ if current.is_symlink():+ raise BackendContractError(+ f"refusing to clean validation path containing a symlink: {target}"+ )+ current = current.parent+ if not target.exists():+ raise BackendContractError(f"validation staging path is missing: {target}")+ if not target.is_dir():+ raise BackendContractError(f"validation staging path is not a directory: {target}")+ shutil.rmtree(target)++ parent = target.parent+ while parent == staging_root or staging_root in parent.parents:+ try:+ parent.rmdir()+ except OSError:+ break+ if parent == staging_root:+ break+ parent = parent.parent+++__all__ = [+ "CAMERA_DATASET_TRIPLET_LAYOUT",+ "CAMERA_TRANSACTION_LAYOUT",+ "CameraInferenceOutputLayout",+ "camera_inference_output_layout",+ "checkpoint_model_dir",+ "cleanup_validation_staging",+ "invocation_output_dir",+ "portable_output_component",+ "public_validation_dir",+ "validation_pass_component",+ "validation_staging_root",+]diff --git a/src/solarwm/runtime/provenance.py b/src/solarwm/runtime/provenance.pynew file mode 100644index 0000000..029162b--- /dev/null+++ b/src/solarwm/runtime/provenance.py@@ -0,0 +1,142 @@+"""Launch provenance assembled before model allocation or data iteration."""++from __future__ import annotations++import hashlib+import re+import subprocess+from collections.abc import Mapping+from dataclasses import asdict, dataclass+from pathlib import Path+from typing import Any++from solarwm import __version__+from solarwm.config.loader import canonical_json+from solarwm.errors import ConfigurationError++_SECRET_KEYS = {"access_token", "api_key", "password", "secret", "token"}+_IMMUTABLE_IMAGE = re.compile(r"[^\s@]+@[a-z0-9][a-z0-9._+-]*:[0-9a-f]{32,}\Z")+++@dataclass(frozen=True)+class GitIdentity:+ available: bool+ root: str = ""+ commit: str = ""+ tree: str = ""+ clean: bool = False+ status_digest: str = ""++ def as_dict(self) -> dict[str, Any]:+ return asdict(self)+++def _git(root: Path, *arguments: str, check: bool = True) -> str:+ process = subprocess.run(+ ["git", "-C", str(root), *arguments],+ check=False,+ capture_output=True,+ text=True,+ )+ if check and process.returncode:+ raise ConfigurationError(+ f"git {' '.join(arguments)} failed for {root}: {process.stderr.strip()}"+ )+ return process.stdout+++def git_identity(path: str | Path) -> GitIdentity:+ source = Path(path).resolve()+ process = subprocess.run(+ ["git", "-C", str(source), "rev-parse", "--show-toplevel"],+ check=False,+ stdout=subprocess.PIPE,+ stderr=subprocess.DEVNULL,+ text=True,+ )+ if process.returncode:+ return GitIdentity(available=False)+ root = Path(process.stdout.strip()).resolve()+ commit = _git(root, "rev-parse", "HEAD").strip()+ tree = _git(root, "rev-parse", "HEAD^{tree}").strip()+ status = _git(root, "status", "--porcelain=v1", "--untracked-files=all")+ return GitIdentity(+ available=True,+ root=str(root),+ commit=commit,+ tree=tree,+ clean=not bool(status),+ status_digest=hashlib.blake2s(status.encode()).hexdigest(),+ )+++def reject_inline_secrets(value: Any, path: tuple[str, ...] = ()) -> None:+ """Prevent resolved-config artifacts from accidentally persisting credentials."""++ if isinstance(value, Mapping):+ for key, child in value.items():+ normalized = str(key).strip().lower()+ child_path = (*path, str(key))+ if normalized in _SECRET_KEYS and child is not None and child != "":+ raise ConfigurationError(+ f"inline credential at {'.'.join(child_path)} is forbidden; "+ "use a workload identity or *_file reference"+ )+ reject_inline_secrets(child, child_path)+ elif isinstance(value, list):+ for index, child in enumerate(value):+ reject_inline_secrets(child, (*path, str(index)))+++def build_launch_manifest(+ *,+ config: Mapping[str, Any],+ source_config: Path,+ source_digest: str,+ resolved_digest: str,+ route: str,+ repository: str | Path,+) -> dict[str, Any]:+ reject_inline_secrets(config)+ runtime = config.get("runtime", {})+ if not isinstance(runtime, Mapping):+ raise ConfigurationError("runtime must be a mapping")+ declared_image = str(runtime.get("image") or "")+ if bool(runtime.get("enforce_image", False)):+ if not declared_image:+ raise ConfigurationError("runtime.enforce_image=true requires runtime.image")+ if not _IMMUTABLE_IMAGE.fullmatch(declared_image):+ raise ConfigurationError("runtime.enforce_image=true requires a digest-pinned image")++ source = git_identity(repository)+ if bool(runtime.get("require_clean_source", False)) and (+ not source.available or not source.clean+ ):+ raise ConfigurationError("runtime.require_clean_source=true requires a clean Git checkout")++ identity = {+ "schema": "solarwm.launch.v2",+ "version": __version__,+ "action": str(config["action"]),+ "name": str(config["name"]),+ "route": route,+ "source_config": {+ "path": str(source_config),+ "digest": source_digest,+ },+ "resolved_config_digest": resolved_digest,+ "source": source.as_dict(),+ "runtime": {+ "declared_image": declared_image,+ "image_enforced": bool(runtime.get("enforce_image", False)),+ },+ "data": {+ "index": str(config.get("data", {}).get("index", "")),+ "test_index": str(config.get("data", {}).get("test_index", "")),+ "generation": str(config.get("data", {}).get("generation", "")),+ },+ "checkpoint": dict(config.get("checkpoint", {})),+ "metadata": dict(config.get("metadata", {})),+ }+ identity["launch_identity_digest"] = hashlib.blake2s(canonical_json(identity)).hexdigest()+ return identitydiff --git a/src/solarwm/runtime/randomness.py b/src/solarwm/runtime/randomness.pynew file mode 100644index 0000000..8839394--- /dev/null+++ b/src/solarwm/runtime/randomness.py@@ -0,0 +1,94 @@+"""Named RNG domains shared by data-parallel and sequence-parallel execution."""++from __future__ import annotations++import hashlib+import random+from dataclasses import asdict, dataclass+from typing import Any++import numpy as np++from solarwm.errors import ConfigurationError++from .topology import Topology++_MODEL_INIT_NAMESPACES = {+ "minimax_h3": 1_211_301_888,+}+++@dataclass(frozen=True)+class RNGIdentity:+ """Seeds recorded beside every training microbatch."""++ base_seed: int+ model_init_seed: int+ objective_seed: int+ dp_rank: int+ sp_rank: int++ def as_dict(self) -> dict[str, int]:+ return asdict(self)+++def model_init_seed(family: str, base_seed: int) -> int:+ try:+ namespace = _MODEL_INIT_NAMESPACES[family]+ except KeyError as exc:+ raise ConfigurationError(f"unknown RNG namespace for {family!r}") from exc+ return namespace + int(base_seed)+++def objective_seed(base_seed: int, dp_rank: int) -> int:+ """Objective seed; SP peers intentionally receive the same value."""++ if dp_rank < 0:+ raise ConfigurationError("dp_rank must be non-negative")+ return int(base_seed) * 100003 + int(dp_rank) * 1024+++def rng_identity(family: str, base_seed: int, topology: Topology) -> RNGIdentity:+ return RNGIdentity(+ base_seed=int(base_seed),+ model_init_seed=model_init_seed(family, base_seed),+ objective_seed=objective_seed(base_seed, topology.dp_rank),+ dp_rank=topology.dp_rank,+ sp_rank=topology.sp_rank,+ )+++def seed_process(seed: int, *, deterministic: bool = False) -> None:+ """Seed Python, NumPy, and torch when installed."""++ value = int(seed)+ random.seed(value)+ np.random.seed(value % (2**32))+ try:+ import torch+ except ImportError:+ return+ torch.manual_seed(value)+ if torch.cuda.is_available():+ torch.cuda.manual_seed_all(value)+ if deterministic:+ torch.backends.cudnn.deterministic = True+ torch.backends.cudnn.benchmark = False+++def stable_validation_seed(video_id: str, step: int, slot: int, base_seed: int) -> int:+ """Derive a stable Wan generation seed from public case identity."""++ payload = f"{video_id}|{int(step)}|{int(slot)}|{int(base_seed)}".encode()+ digest = hashlib.blake2b(payload, digest_size=8).digest()+ return int.from_bytes(digest, "little") & 0x7FFFFFFF+++def torch_generator(seed: int, *, device: Any) -> Any:+ """Construct a torch generator lazily so core inspection needs no torch."""++ try:+ import torch+ except ImportError as exc:+ raise ConfigurationError("torch is required to construct a model RNG") from exc+ return torch.Generator(device=device).manual_seed(int(seed))diff --git a/src/solarwm/runtime/safe_state.py b/src/solarwm/runtime/safe_state.pynew file mode 100644index 0000000..e8107f0--- /dev/null+++ b/src/solarwm/runtime/safe_state.py@@ -0,0 +1,203 @@+"""Safe, versioned serialization for process RNG state.++Checkpoint payloads must remain loadable with ``torch.load(weights_only=True)``.+NumPy's native ``RandomState`` tuple contains an ndarray whose pickle reducer is+not admitted by that loader, so checkpoint writers store only validated scalar+containers and reconstruct the ndarray after loading.+"""++from __future__ import annotations++import math+from collections.abc import Mapping, Sequence+from typing import Any++from solarwm.errors import BackendContractError++_PYTHON_SCHEMA = "solarwm.python-rng-state.v1"+_NUMPY_SCHEMA = "solarwm.numpy-rng-state.v1"+_MT19937_WORDS = 624+_UINT32_MAX = (1 << 32) - 1+++def _integer(value: Any, *, label: str, minimum: int, maximum: int) -> int:+ if isinstance(value, bool) or not isinstance(value, int):+ raise BackendContractError(f"{label} must be an integer")+ result = int(value)+ if not minimum <= result <= maximum:+ raise BackendContractError(f"{label} must be between {minimum} and {maximum}, got {result}")+ return result+++def _gaussian(value: Any, *, label: str) -> float | None:+ if value is None:+ return None+ if isinstance(value, bool) or not isinstance(value, (int, float)):+ raise BackendContractError(f"{label} must be a finite number or null")+ result = float(value)+ if not math.isfinite(result):+ raise BackendContractError(f"{label} must be finite")+ return result+++def encode_python_rng_state(state: object) -> dict[str, Any]:+ """Convert ``random.getstate()`` into a safe, explicit checkpoint value."""++ if not isinstance(state, tuple) or len(state) != 3:+ raise BackendContractError("Python RNG state must be a three-item tuple")+ version = _integer(state[0], label="Python RNG version", minimum=3, maximum=3)+ raw_words = state[1]+ if not isinstance(raw_words, tuple) or len(raw_words) != _MT19937_WORDS + 1:+ raise BackendContractError("Python MT19937 state must contain 625 integers")+ words = [+ _integer(+ value,+ label=f"Python RNG word {index}",+ minimum=0,+ maximum=_UINT32_MAX,+ )+ for index, value in enumerate(raw_words[:-1])+ ]+ position = _integer(+ raw_words[-1],+ label="Python RNG position",+ minimum=0,+ maximum=_MT19937_WORDS,+ )+ return {+ "schema": _PYTHON_SCHEMA,+ "version": version,+ "words": words,+ "position": position,+ "gaussian": _gaussian(state[2], label="Python RNG Gaussian cache"),+ }+++def decode_python_rng_state(value: object) -> tuple[Any, ...]:+ """Validate and reconstruct a state accepted by ``random.setstate``."""++ if not isinstance(value, Mapping) or value.get("schema") != _PYTHON_SCHEMA:+ raise BackendContractError("checkpoint Python RNG state has an unsupported schema")+ version = _integer(value.get("version"), label="Python RNG version", minimum=3, maximum=3)+ raw_words = value.get("words")+ if (+ not isinstance(raw_words, Sequence)+ or isinstance(raw_words, (str, bytes, bytearray))+ or len(raw_words) != _MT19937_WORDS+ ):+ raise BackendContractError("checkpoint Python RNG state must contain 624 words")+ words = tuple(+ _integer(+ item,+ label=f"Python RNG word {index}",+ minimum=0,+ maximum=_UINT32_MAX,+ )+ for index, item in enumerate(raw_words)+ )+ position = _integer(+ value.get("position"),+ label="Python RNG position",+ minimum=0,+ maximum=_MT19937_WORDS,+ )+ gaussian = _gaussian(value.get("gaussian"), label="Python RNG Gaussian cache")+ return (version, (*words, position), gaussian)+++def encode_numpy_rng_state(state: object) -> dict[str, Any]:+ """Convert ``numpy.random.get_state()`` without retaining an ndarray reducer."""++ if not isinstance(state, tuple) or len(state) != 5:+ raise BackendContractError("NumPy RNG state must be a five-item tuple")+ if state[0] != "MT19937":+ raise BackendContractError("only NumPy MT19937 global RNG state is supported")+ raw_words = state[1]+ if (+ not hasattr(raw_words, "tolist")+ or getattr(raw_words, "dtype", None) is None+ or getattr(raw_words, "shape", None) != (_MT19937_WORDS,)+ ):+ raise BackendContractError("NumPy MT19937 state must contain 624 uint32 words")+ words = [+ _integer(+ item,+ label=f"NumPy RNG word {index}",+ minimum=0,+ maximum=_UINT32_MAX,+ )+ for index, item in enumerate(raw_words.tolist())+ ]+ position = _integer(+ state[2],+ label="NumPy RNG position",+ minimum=0,+ maximum=_MT19937_WORDS,+ )+ has_gaussian = _integer(state[3], label="NumPy RNG Gaussian flag", minimum=0, maximum=1)+ return {+ "schema": _NUMPY_SCHEMA,+ "bit_generator": "MT19937",+ "words": words,+ "position": position,+ "has_gaussian": has_gaussian,+ "cached_gaussian": _gaussian(state[4], label="NumPy RNG Gaussian cache"),+ }+++def decode_numpy_rng_state(value: object) -> tuple[Any, ...]:+ """Validate and reconstruct a state accepted by ``numpy.random.set_state``."""++ if not isinstance(value, Mapping) or value.get("schema") != _NUMPY_SCHEMA:+ raise BackendContractError("checkpoint NumPy RNG state has an unsupported schema")+ if value.get("bit_generator") != "MT19937":+ raise BackendContractError("checkpoint NumPy RNG must use MT19937")+ raw_words = value.get("words")+ if (+ not isinstance(raw_words, Sequence)+ or isinstance(raw_words, (str, bytes, bytearray))+ or len(raw_words) != _MT19937_WORDS+ ):+ raise BackendContractError("checkpoint NumPy RNG state must contain 624 words")+ words = [+ _integer(+ item,+ label=f"NumPy RNG word {index}",+ minimum=0,+ maximum=_UINT32_MAX,+ )+ for index, item in enumerate(raw_words)+ ]+ position = _integer(+ value.get("position"),+ label="NumPy RNG position",+ minimum=0,+ maximum=_MT19937_WORDS,+ )+ has_gaussian = _integer(+ value.get("has_gaussian"),+ label="NumPy RNG Gaussian flag",+ minimum=0,+ maximum=1,+ )+ cached = _gaussian(value.get("cached_gaussian"), label="NumPy RNG Gaussian cache")+ if cached is None:+ raise BackendContractError("NumPy RNG Gaussian cache must be a finite number")++ import numpy as np++ return (+ "MT19937",+ np.asarray(words, dtype=np.uint32),+ position,+ has_gaussian,+ cached,+ )+++__all__ = [+ "decode_numpy_rng_state",+ "decode_python_rng_state",+ "encode_numpy_rng_state",+ "encode_python_rng_state",+]diff --git a/src/solarwm/runtime/serialization.py b/src/solarwm/runtime/serialization.pynew file mode 100644index 0000000..93b01d2--- /dev/null+++ b/src/solarwm/runtime/serialization.py@@ -0,0 +1,25 @@+"""Deterministic serialization helpers shared by runtime artifacts."""++from __future__ import annotations++import json+from typing import Any+++def canonical_json_bytes(value: Any, *, trailing_newline: bool = True) -> bytes:+ """Serialize JSON deterministically and reject non-standard numeric values."""++ suffix = "\n" if trailing_newline else ""+ return (+ json.dumps(+ value,+ allow_nan=False,+ ensure_ascii=False,+ separators=(",", ":"),+ sort_keys=True,+ )+ + suffix+ ).encode("utf-8")+++__all__ = ["canonical_json_bytes"]diff --git a/src/solarwm/runtime/topology.py b/src/solarwm/runtime/topology.pynew file mode 100644index 0000000..2833ac1--- /dev/null+++ b/src/solarwm/runtime/topology.py@@ -0,0 +1,79 @@+"""Logical data-parallel topology shared by data and model execution."""++from __future__ import annotations++import os+from collections.abc import Mapping+from dataclasses import dataclass++from solarwm.errors import ConfigurationError+++@dataclass(frozen=True)+class Topology:+ raw_world_size: int+ raw_rank: int+ local_world_size: int+ local_rank: int+ sp_size: int = 1++ @classmethod+ def from_environ(cls, sp_size: int, environ: Mapping[str, str] | None = None) -> Topology:+ values = os.environ if environ is None else environ+ required = ("WORLD_SIZE", "RANK", "LOCAL_WORLD_SIZE", "LOCAL_RANK")+ missing = [name for name in required if name not in values]+ if missing:+ raise ConfigurationError(f"torchrun environment is missing {missing}")+ try:+ return cls(+ raw_world_size=int(values["WORLD_SIZE"]),+ raw_rank=int(values["RANK"]),+ local_world_size=int(values["LOCAL_WORLD_SIZE"]),+ local_rank=int(values["LOCAL_RANK"]),+ sp_size=int(sp_size),+ )+ except ValueError as exc:+ raise ConfigurationError("torchrun rank variables must be integers") from exc++ def __post_init__(self) -> None:+ for name in ("raw_world_size", "local_world_size", "sp_size"):+ if getattr(self, name) < 1:+ raise ConfigurationError(f"{name} must be positive")+ if not 0 <= self.raw_rank < self.raw_world_size:+ raise ConfigurationError("raw_rank is outside raw_world_size")+ if not 0 <= self.local_rank < self.local_world_size:+ raise ConfigurationError("local_rank is outside local_world_size")+ if self.raw_world_size % self.sp_size:+ raise ConfigurationError("raw_world_size must be divisible by sp_size")+ if self.local_world_size % self.sp_size:+ raise ConfigurationError("local_world_size must be divisible by sp_size")++ @property+ def dp_world_size(self) -> int:+ return self.raw_world_size // self.sp_size++ @property+ def dp_rank(self) -> int:+ return self.raw_rank // self.sp_size++ @property+ def sp_rank(self) -> int:+ return self.raw_rank % self.sp_size++ @property+ def node_count(self) -> int:+ if self.raw_world_size % self.local_world_size:+ raise ConfigurationError("raw_world_size must be divisible by local_world_size")+ return self.raw_world_size // self.local_world_size++ @property+ def node_id(self) -> int:+ return self.raw_rank // self.local_world_size++ @property+ def local_dp_world_size(self) -> int:+ return self.local_world_size // self.sp_size++ @property+ def local_dp_rank(self) -> int:+ return self.local_rank // self.sp_sizediff --git a/src/solarwm/storage.py b/src/solarwm/storage.pynew file mode 100644index 0000000..989ab9d--- /dev/null+++ b/src/solarwm/storage.py@@ -0,0 +1,136 @@+"""Files that may live in object storage.++A path with a ``<scheme>://`` prefix is remote and is read and written through fsspec, which picks the+filesystem from the scheme; install the fsspec backend for your storage and configure its credentials the+way that backend reads them. Any other path is a local file. Each call retries a few times: a busy store+turns a request away now and then, and a member thrown away over that is encoding time spent for nothing.+"""++from __future__ import annotations++import datetime as _dt+import os+import posixpath+import time+from typing import Callable, TypeVar++T = TypeVar("T")+++def is_remote(path: str) -> bool:+ return "://" in str(path)+++def _fs(path: str):+ from fsspec.core import url_to_fs++ return url_to_fs(path)+++def _retry(call: Callable[[], T], tries: int = 4) -> T:+ for attempt in range(tries):+ try:+ return call()+ except FileNotFoundError:+ raise+ except Exception:+ if attempt + 1 == tries:+ raise+ time.sleep(2**attempt)+ raise AssertionError("unreachable")+++def get(source: str, destination: str) -> None:+ """Copy one remote object to a local file."""++ fs, path = _fs(source)+ os.makedirs(os.path.dirname(os.path.abspath(destination)), exist_ok=True)+ _retry(lambda: fs.get_file(path, destination))+++def get_matching(pattern: str, directory: str) -> list[str]:+ """Copy every object matching a glob (``.../*.json``) into a local directory; returns the local files."""++ fs, path = _fs(pattern)+ os.makedirs(directory, exist_ok=True)+ local = []+ for name in _retry(lambda: fs.glob(path)):+ target = os.path.join(directory, posixpath.basename(name))+ _retry(lambda name=name, target=target: fs.get_file(name, target))+ local.append(target)+ return local+++def put(source: str, destination: str) -> None:+ """Copy one local file to a remote object."""++ fs, path = _fs(destination)+ _retry(lambda: fs.put_file(source, path))+++def write_bytes(destination: str, data: bytes) -> None:+ fs, path = _fs(destination)+ _retry(lambda: fs.pipe_file(path, data))+++def move(source: str, destination: str) -> None:+ """A local file to a remote object; the local file is removed once the copy succeeded."""++ put(source, destination)+ os.remove(source)+++def _mtime(info: dict) -> float | None:+ for key in ("mtime", "LastModified", "last_modified", "updated", "modified", "created", "timeCreated"):+ value = info.get(key)+ if value is None:+ continue+ if isinstance(value, (int, float)):+ return float(value)+ if isinstance(value, _dt.datetime):+ return (value if value.tzinfo else value.replace(tzinfo=_dt.timezone.utc)).timestamp()+ try:+ return _dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp()+ except ValueError:+ continue+ return None+++def stat(path: str) -> tuple[int, float | None] | None:+ """(size, modification time) of one object, or None when it is not there."""++ fs, key = _fs(path)+ try:+ info = _retry(lambda: fs.info(key))+ except FileNotFoundError:+ return None+ if info.get("type") == "directory":+ return None+ return int(info.get("size") or 0), _mtime(info)+++def listing(prefix: str) -> dict[str, float | None] | None:+ """Base name -> modification time of the objects directly under a prefix; an empty dict when the prefix+ holds nothing, None when the listing failed (it then says nothing about what is there)."""++ fs, key = _fs(prefix.rstrip("/"))+ try:+ entries = _retry(lambda: fs.ls(key, detail=True, refresh=True) if _accepts_refresh(fs) else fs.ls(key, detail=True))+ except FileNotFoundError:+ return {}+ except Exception:+ return None+ return {+ posixpath.basename(entry["name"].rstrip("/")): _mtime(entry)+ for entry in entries+ if entry.get("type") != "directory"+ }+++def _accepts_refresh(fs) -> bool:+ import inspect++ try:+ return "refresh" in inspect.signature(fs.ls).parameters+ except (TypeError, ValueError):+ return Falsediff --git a/src/solarwm/training/__init__.py b/src/solarwm/training/__init__.pynew file mode 100644index 0000000..d507262--- /dev/null+++ b/src/solarwm/training/__init__.py@@ -0,0 +1,21 @@+"""Backend-neutral training lifecycle with strict numeric gates."""++from .engine import (+ BatchIdentity,+ GradientStatus,+ JsonlEventSink,+ MicrobatchResult,+ StepPolicy,+ TrainingEngine,+ TrainingRuntime,+)++__all__ = [+ "BatchIdentity",+ "GradientStatus",+ "JsonlEventSink",+ "MicrobatchResult",+ "StepPolicy",+ "TrainingEngine",+ "TrainingRuntime",+]diff --git a/src/solarwm/training/anyflow.py b/src/solarwm/training/anyflow.pynew file mode 100644index 0000000..c8d5fac--- /dev/null+++ b/src/solarwm/training/anyflow.py@@ -0,0 +1,316 @@+"""Backbone-independent torch primitives for AnyFlow v1.5."""++from __future__ import annotations++from collections.abc import Callable, Sequence+from typing import NamedTuple++import torch++SAMPLE_TYPE_DIFFUSION = 0+SAMPLE_TYPE_CONSISTENCY = 1+SAMPLE_TYPE_FLOW_MAP = 2+GatherResult = torch.Tensor | Sequence[torch.Tensor]+GatherFn = Callable[[torch.Tensor], GatherResult]+++class AnyFlowTimePairs(NamedTuple):+ """Per-sample normalized AnyFlow time pairs and deterministic type masks."""++ t: torch.Tensor+ r: torch.Tensor+ is_diffusion: torch.Tensor+ is_consistency: torch.Tensor+ sample_type: torch.Tensor+++def apply_timestep_shift(normalized: torch.Tensor, shift: float) -> torch.Tensor:+ """Apply the FlowMatch/AnyFlow rational shift to normalized times."""+ if not torch.is_tensor(normalized):+ normalized = torch.as_tensor(normalized, dtype=torch.float32)+ if not torch.is_floating_point(normalized):+ normalized = normalized.to(torch.float32)+ if not isinstance(shift, (int, float)) or not torch.isfinite(torch.tensor(float(shift))):+ raise ValueError(f"shift must be finite, got {shift!r}")+ if shift <= 0:+ raise ValueError(f"shift must be positive, got {shift}")+ if shift == 1.0:+ return normalized+ return shift * normalized / (1.0 + (shift - 1.0) * normalized)+++def sample_anyflow_time_pairs(+ batch_size: int,+ *,+ logical_dp_rank: int = 0,+ logical_dp_world_size: int = 1,+ diffusion_ratio: float = 0.5,+ consistency_ratio: float = 0.25,+ generator: torch.Generator | None = None,+ device: torch.device | str = "cpu",+ variant: str = "v1_5",+) -> AnyFlowTimePairs:+ """Sample normalized ``(t, r)`` and assign sample types globally over DP.++ Exactly two FP32 uniforms are consumed per local sample. Type assignment+ follows the official trainer's logical-global flat-index policy: the first+ rounded ``diffusion_ratio`` fraction has ``r=t``, the following rounded+ ``consistency_ratio`` fraction has ``r=0``, and the remainder has strict+ ``0 < r < t``. Enforcing the strict inequality uses ``nextafter`` and does+ not consume additional random numbers.++ All logical DP ranks are assumed to have the same local ``batch_size``.+ Sequence-parallel replicas must call this with the same logical DP rank,+ generator state, and arguments (or broadcast the returned tensors).+ """+ if not isinstance(batch_size, int) or batch_size <= 0:+ raise ValueError(f"batch_size must be a positive integer, got {batch_size!r}")+ if not isinstance(logical_dp_world_size, int) or logical_dp_world_size <= 0:+ raise ValueError(+ f"logical_dp_world_size must be a positive integer, got {logical_dp_world_size!r}"+ )+ if not isinstance(logical_dp_rank, int) or not 0 <= logical_dp_rank < logical_dp_world_size:+ raise ValueError(+ f"logical_dp_rank must be in [0, {logical_dp_world_size}), got {logical_dp_rank!r}"+ )+ if not 0.0 <= diffusion_ratio <= 1.0:+ raise ValueError(f"diffusion_ratio must be in [0, 1], got {diffusion_ratio}")+ if not 0.0 <= consistency_ratio <= 1.0:+ raise ValueError(f"consistency_ratio must be in [0, 1], got {consistency_ratio}")+ if diffusion_ratio + consistency_ratio > 1.0:+ raise ValueError(+ "diffusion_ratio + consistency_ratio must not exceed 1, got "+ f"{diffusion_ratio + consistency_ratio}"+ )++ if variant != "v1_5":+ raise ValueError("only AnyFlow v1.5 is supported")+ # Keep both trainers' RNG call order: first all u1 samples, then all u2+ # samples. This still consumes exactly two FP32 uniforms per item.+ u1 = torch.rand(batch_size, dtype=torch.float32, device=device, generator=generator)+ u2 = torch.rand(batch_size, dtype=torch.float32, device=device, generator=generator)+ if variant == "v1_5":+ t = u1+ r = u2 * t+ else:+ t = torch.maximum(u1, u2)+ r = torch.minimum(u1, u2)++ global_batch_size = batch_size * logical_dp_world_size+ global_indices = (+ torch.arange(batch_size, dtype=torch.long, device=device) + logical_dp_rank * batch_size+ )+ num_diffusion = round(diffusion_ratio * global_batch_size)+ num_consistency = round(consistency_ratio * global_batch_size)+ is_diffusion = global_indices < num_diffusion+ is_consistency = (global_indices >= num_diffusion) & (+ global_indices < num_diffusion + num_consistency+ )+ is_flow_map = ~(is_diffusion | is_consistency)++ r = torch.where(is_diffusion, t, r)+ r = torch.where(is_consistency, torch.zeros_like(r), r)++ # torch.rand can produce exactly zero, and two generated FP32 values can be+ # equal. Repair only general flow-map samples, without extra RNG draws.+ zero = torch.zeros((), dtype=r.dtype, device=r.device)+ one = torch.ones((), dtype=r.dtype, device=r.device)+ smallest_positive = torch.nextafter(zero, one)+ strict_r = torch.where(r > 0, r, smallest_positive)+ strict_t = torch.where(t > strict_r, t, torch.nextafter(strict_r, one))+ r = torch.where(is_flow_map, strict_r, r)+ t = torch.where(is_flow_map, strict_t, t)++ sample_type = torch.full((batch_size,), SAMPLE_TYPE_FLOW_MAP, dtype=torch.int8, device=device)+ sample_type = torch.where(+ is_diffusion,+ torch.full_like(sample_type, SAMPLE_TYPE_DIFFUSION),+ sample_type,+ )+ sample_type = torch.where(+ is_consistency,+ torch.full_like(sample_type, SAMPLE_TYPE_CONSISTENCY),+ sample_type,+ )+ return AnyFlowTimePairs(t, r, is_diffusion, is_consistency, sample_type)+++def bounded_difference_timesteps(+ t: torch.Tensor,+ r: torch.Tensor,+ *,+ epsilon: float = 5.0,+ num_train_timesteps: int = 1000,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Return the FAR v1.5 finite-difference window bounded by ``[r, N]``."""+ if epsilon <= 0:+ raise ValueError(f"epsilon must be positive, got {epsilon}")+ if not isinstance(num_train_timesteps, int) or num_train_timesteps <= 0:+ raise ValueError(+ f"num_train_timesteps must be a positive integer, got {num_train_timesteps!r}"+ )+ t_tensor = torch.as_tensor(t)+ r_tensor = torch.as_tensor(r, device=t_tensor.device, dtype=t_tensor.dtype)+ try:+ t_tensor, r_tensor = torch.broadcast_tensors(t_tensor, r_tensor)+ except RuntimeError as exc:+ raise ValueError(+ f"t and r are not broadcastable: {tuple(t_tensor.shape)} vs {tuple(r_tensor.shape)}"+ ) from exc+ t_plus = (t_tensor + epsilon).clamp_max(num_train_timesteps)+ t_minus = torch.maximum(t_tensor - epsilon, r_tensor)+ return t_plus, t_minus++[email protected]_grad()+def central_difference_derivative(+ u_plus: torch.Tensor,+ u_minus: torch.Tensor,+ t_plus: torch.Tensor,+ t_minus: torch.Tensor,+) -> torch.Tensor:+ """Compute FAR v1.5's derivative using the actual bounded window width."""+ if u_plus.shape != u_minus.shape:+ raise ValueError(+ "u_plus and u_minus must have identical shapes, got "+ f"{tuple(u_plus.shape)} and {tuple(u_minus.shape)}"+ )+ plus = torch.as_tensor(t_plus, device=u_plus.device, dtype=torch.float32)+ minus = torch.as_tensor(t_minus, device=u_plus.device, dtype=torch.float32)+ try:+ plus, minus = torch.broadcast_tensors(plus, minus)+ except RuntimeError as exc:+ raise ValueError(+ f"t_plus and t_minus are not broadcastable: {tuple(plus.shape)} vs {tuple(minus.shape)}"+ ) from exc+ denominator = plus - minus+ denominator = _append_trailing_singletons(denominator.clamp_min(1e-6), u_plus.ndim)+ return ((u_plus.detach().float() - u_minus.detach().float()) / denominator).detach()++[email protected]_grad()+def gaussian_timestep_weights(+ shifted_raw_timesteps: torch.Tensor,+ *,+ shift: float,+ num_train_timesteps: int = 1000,+) -> torch.Tensor:+ """Match FAR v1.5's mean-normalized Gaussian (``bsmntw``) grid."""+ if not torch.is_tensor(shifted_raw_timesteps):+ shifted_raw_timesteps = torch.as_tensor(shifted_raw_timesteps)+ if not isinstance(num_train_timesteps, int) or num_train_timesteps <= 0:+ raise ValueError(+ f"num_train_timesteps must be a positive integer, got {num_train_timesteps!r}"+ )+ device = shifted_raw_timesteps.device+ work_dtype = torch.float64 if shifted_raw_timesteps.dtype == torch.float64 else torch.float32+ normalized_grid = torch.linspace(+ 1.0,+ 0.0,+ num_train_timesteps + 1,+ dtype=work_dtype,+ device=device,+ )[:-1]+ shifted_grid = apply_timestep_shift(normalized_grid, shift)+ raw_grid = shifted_grid * num_train_timesteps+ unnormalized = torch.exp(+ -2.0 * ((raw_grid - num_train_timesteps / 2.0) / num_train_timesteps).square()+ )+ unnormalized = unnormalized - unnormalized.min()+ weight_sum = unnormalized.sum()+ if not bool(torch.isfinite(weight_sum)) or not bool(weight_sum > 0):+ raise ValueError("gaussian training-grid weights have invalid normalization")+ grid_weights = unnormalized * (num_train_timesteps / weight_sum)++ flat = shifted_raw_timesteps.to(dtype=work_dtype).reshape(-1)+ nearest = torch.argmin((raw_grid[:, None] - flat[None, :]).abs(), dim=0)+ return grid_weights[nearest].reshape(shifted_raw_timesteps.shape)+++def adaptive_rescale_non_diffusion_losses(+ per_sample_loss: torch.Tensor,+ is_diffusion: torch.Tensor,+ *,+ gather_fn: GatherFn | None = None,+ process_group: object | None = None,+ eps: float = 1e-5,+) -> torch.Tensor:+ """AnyFlow adaptive rescaling with empty-class-safe logical-DP gather.++ Diffusion losses stay unchanged. Each non-diffusion loss is scaled toward+ the finite logical-global mean diffusion loss, matching the reference+ ``mean_diffusion / (loss.detach() + eps)`` rule. If the logical batch has+ no finite diffusion reference (or the local batch has no non-diffusion+ samples), this is an identity operation instead of producing NaN.++ ``process_group`` must be the logical DP group, excluding SP replicas.+ Tests and non-distributed callers may inject ``gather_fn(tensor)`` instead.+ """+ if gather_fn is not None and process_group is not None:+ raise ValueError("provide at most one of gather_fn and process_group")+ if per_sample_loss.ndim != 1 or is_diffusion.ndim != 1:+ raise ValueError("per_sample_loss and is_diffusion must both be 1-D")+ if per_sample_loss.shape != is_diffusion.shape:+ raise ValueError(+ "per_sample_loss and is_diffusion shapes differ: "+ f"{tuple(per_sample_loss.shape)} vs {tuple(is_diffusion.shape)}"+ )+ if eps <= 0:+ raise ValueError(f"eps must be positive, got {eps}")++ is_diffusion = is_diffusion.to(device=per_sample_loss.device, dtype=torch.bool)+ global_loss = _gather_flattened(+ per_sample_loss, gather_fn=gather_fn, process_group=process_group+ )+ global_is_diffusion = _gather_flattened(+ is_diffusion, gather_fn=gather_fn, process_group=process_group+ ).to(dtype=torch.bool)+ if global_loss.numel() != global_is_diffusion.numel():+ raise ValueError(+ "gathered loss and sample-type mask lengths differ: "+ f"{global_loss.numel()} vs {global_is_diffusion.numel()}"+ )++ with torch.no_grad():+ candidates = global_loss[global_is_diffusion]+ candidates = candidates[torch.isfinite(candidates)]+ if candidates.numel() == 0 or bool(is_diffusion.all()):+ return per_sample_loss+ reference = candidates.mean().to(device=per_sample_loss.device, dtype=per_sample_loss.dtype)+ denominator = per_sample_loss.detach() + eps+ scale = reference / denominator+ scale = torch.where(torch.isfinite(scale), scale, torch.ones_like(scale))+ return torch.where(is_diffusion, per_sample_loss, per_sample_loss * scale)+++def _append_trailing_singletons(value: torch.Tensor, ndim: int) -> torch.Tensor:+ if value.ndim > ndim:+ raise ValueError(f"cannot broadcast shape {tuple(value.shape)} to ndim={ndim}")+ return value.reshape(*value.shape, *((1,) * (ndim - value.ndim)))+++def _gather_flattened(+ tensor: torch.Tensor,+ *,+ gather_fn: GatherFn | None,+ process_group: object | None,+) -> torch.Tensor:+ detached = tensor.detach()+ if gather_fn is not None:+ gathered = gather_fn(detached)+ if torch.is_tensor(gathered):+ return gathered.to(device=tensor.device).reshape(-1)+ if not isinstance(gathered, Sequence) or len(gathered) == 0:+ raise ValueError("gather_fn must return a tensor or a non-empty tensor sequence")+ return torch.cat([part.to(device=tensor.device).reshape(-1) for part in gathered], dim=0)++ if process_group is not None:+ if not torch.distributed.is_available() or not torch.distributed.is_initialized():+ raise RuntimeError(+ "process_group was provided but torch.distributed is not initialized"+ )+ world_size = torch.distributed.get_world_size(group=process_group)+ parts = [torch.empty_like(detached) for _ in range(world_size)]+ torch.distributed.all_gather(parts, detached, group=process_group)+ return torch.cat([part.reshape(-1) for part in parts], dim=0)+ return detached.reshape(-1)diff --git a/src/solarwm/training/ema.py b/src/solarwm/training/ema.pynew file mode 100644index 0000000..c3f94b4--- /dev/null+++ b/src/solarwm/training/ema.py@@ -0,0 +1,116 @@+"""Collective-free rank-local EMA for FSDP-exposed parameter shards."""++from __future__ import annotations++from collections.abc import Iterator, Mapping+from contextlib import contextmanager+from typing import Any++from solarwm.errors import BackendContractError+++def ema_decay_for_step(*, target_decay: float, global_step: int, warmup_steps: int = 0) -> float:+ decay = float(target_decay)+ step = int(global_step)+ warmup = int(warmup_steps)+ if not 0 <= decay <= 1 or step < 0 or warmup < 0:+ raise BackendContractError("invalid EMA decay/step/warmup")+ return 0.0 if step < warmup else decay+++class ShardedEMA:+ """Track only locally exposed shards; save-time gathering belongs to the backend."""++ def __init__(+ self,+ module: Any,+ *,+ decay: float,+ device: Any,+ dtype: Any,+ trainable_only: bool = False,+ ) -> None:+ try:+ import torch+ except ImportError as exc:+ raise BackendContractError("torch is required for EMA") from exc+ if not 0 <= float(decay) <= 1:+ raise BackendContractError("EMA decay must be in [0,1]")+ if not isinstance(dtype, torch.dtype) or not dtype.is_floating_point:+ raise BackendContractError("EMA dtype must be floating point")+ self.decay = float(decay)+ self.device = torch.device(device)+ self.dtype = dtype+ self.trainable_only = bool(trainable_only)+ self.num_updates = 0+ self.shadow = {+ name: parameter.detach().to(device=self.device, dtype=dtype).clone()+ for name, parameter in self._named(module)+ }+ if not self.shadow:+ raise BackendContractError("EMA selected no parameters")++ def _named(self, module: Any) -> list[tuple[str, Any]]:+ root = getattr(module, "module", module)+ return [+ (name, parameter)+ for name, parameter in root.named_parameters()+ if not self.trainable_only or parameter.requires_grad+ ]++ def _validate(self, module: Any) -> list[tuple[str, Any]]:+ named = self._named(module)+ if {name for name, _ in named} != set(self.shadow):+ raise BackendContractError("EMA parameter names differ from the live model")+ for name, parameter in named:+ shadow = self.shadow[name]+ if shadow.shape != parameter.shape or shadow.dtype != self.dtype:+ raise BackendContractError(f"EMA shard layout differs for {name}")+ return named++ def update(self, module: Any, *, decay: float | None = None) -> None:+ import torch++ value = self.decay if decay is None else float(decay)+ if not 0 <= value <= 1:+ raise BackendContractError("EMA update decay must be in [0,1]")+ with torch.no_grad():+ for name, parameter in self._validate(module):+ live = parameter.detach().to(device=self.device, dtype=self.dtype)+ self.shadow[name].mul_(value).add_(live, alpha=1 - value)+ self.num_updates += 1++ def state_dict(self) -> dict[str, Any]:+ return {name: value.clone() for name, value in self.shadow.items()}++ def load_state_dict(self, values: Mapping[str, Any], *, num_updates: int) -> None:+ if set(values) != set(self.shadow):+ raise BackendContractError("EMA checkpoint names differ from the live layout")+ restored = {}+ for name, current in self.shadow.items():+ value = values[name]+ if value.shape != current.shape or not value.is_floating_point():+ raise BackendContractError(f"EMA checkpoint layout differs for {name}")+ restored[name] = value.detach().to(device=self.device, dtype=self.dtype, copy=True)+ if int(num_updates) < 0:+ raise BackendContractError("EMA update count must be non-negative")+ self.shadow = restored+ self.num_updates = int(num_updates)++ @contextmanager+ def swapped_into(self, module: Any) -> Iterator[None]:+ import torch++ named = self._validate(module)+ live = {name: parameter.detach().clone() for name, parameter in named}+ try:+ with torch.no_grad():+ for name, parameter in named:+ parameter.copy_(+ self.shadow[name].to(device=parameter.device, dtype=parameter.dtype)+ )+ yield+ finally:+ with torch.no_grad():+ for name, parameter in self._named(module):+ parameter.copy_(live[name])diff --git a/src/solarwm/training/engine.py b/src/solarwm/training/engine.pynew file mode 100644index 0000000..f6d97f3--- /dev/null+++ b/src/solarwm/training/engine.py@@ -0,0 +1,252 @@+"""A small, auditable optimizer-step state machine shared by backends."""++from __future__ import annotations++import gc+import json+import math+import os+from collections.abc import Callable, Mapping, Sequence+from contextlib import contextmanager+from dataclasses import asdict, dataclass+from pathlib import Path+from typing import Any, Protocol++from solarwm.errors import BackendContractError+++@dataclass(frozen=True)+class BatchIdentity:+ sample_ids: tuple[str, ...]+ start_frames: tuple[int, ...]+ noise_seeds: tuple[int, ...]+ checkpoint_id: str+ plan_fingerprint: str++ def __post_init__(self) -> None:+ size = len(self.sample_ids)+ if size < 1:+ raise BackendContractError("microbatch identity has no samples")+ if len(self.start_frames) != size or len(self.noise_seeds) != size:+ raise BackendContractError("microbatch identity fields have different lengths")+ if not self.checkpoint_id or not self.plan_fingerprint:+ raise BackendContractError("microbatch identity lacks checkpoint/plan identity")+++@dataclass(frozen=True)+class MicrobatchResult:+ identity: BatchIdentity+ losses: Mapping[str, float]++ def __post_init__(self) -> None:+ if not self.losses:+ raise BackendContractError("microbatch produced no loss fields")+ for name, value in self.losses.items():+ if not name or not isinstance(value, (int, float)) or not math.isfinite(value):+ raise BackendContractError(f"non-finite or invalid loss {name!r}: {value!r}")+++@dataclass(frozen=True)+class GradientStatus:+ finite: bool+ norm: float++ def __post_init__(self) -> None:+ if self.finite and not math.isfinite(self.norm):+ raise BackendContractError("finite gradients reported a non-finite norm")+++@dataclass(frozen=True)+class StepPolicy:+ max_steps: int+ grad_accum: int = 1+ save_every: int = 0+ validate_every: int = 0+ validation_steps: tuple[int, ...] = ()+ save_steps: tuple[int, ...] = ()++ def __post_init__(self) -> None:+ if self.max_steps < 1 or self.grad_accum < 1:+ raise BackendContractError("max_steps and grad_accum must be positive")+ if self.save_every < 0 or self.validate_every < 0:+ raise BackendContractError("save/validate intervals must be non-negative")+ if any(step < 1 for step in (*self.validation_steps, *self.save_steps)):+ raise BackendContractError("validation_steps must be positive")++ def should_save(self, step: int) -> bool:+ return step in self.save_steps or bool(self.save_every and step % self.save_every == 0)++ def should_validate(self, step: int) -> bool:+ return step in self.validation_steps or bool(+ self.validate_every and step % self.validate_every == 0+ )+++class TrainingRuntime(Protocol):+ """Heavy backend hook surface; every method has one lifecycle meaning."""++ @property+ def global_step(self) -> int: ...++ def zero_grad(self) -> None: ...++ def train_microbatch(self, micro_index: int, grad_accum: int) -> MicrobatchResult: ...++ def assert_sp_peer_identity(self, identity: BatchIdentity) -> None: ...++ def prepare_optimizer_step(self) -> GradientStatus: ...++ def optimizer_step(self) -> None: ...++ def scheduler_step(self) -> None: ...++ def ema_update(self, step: int) -> None: ...++ def set_global_step(self, step: int) -> None: ...++ def save_checkpoint(self, step: int) -> str: ...++ def validate(self, step: int) -> Mapping[str, Any]: ...+++EventSink = Callable[[Mapping[str, Any]], None]+++@contextmanager+def suspend_automatic_cycle_collection():+ restore_gc = gc.isenabled()+ if restore_gc:+ gc.disable()+ try:+ yield+ finally:+ if restore_gc:+ gc.enable()+++class JsonlEventSink:+ """Append canonical finite JSON records to one rank-owned log."""++ def __init__(self, path: str | Path) -> None:+ self.path = Path(path)+ self.path.parent.mkdir(parents=True, exist_ok=True)++ def __call__(self, event: Mapping[str, Any]) -> None:+ try:+ payload = (+ json.dumps(+ event,+ allow_nan=False,+ ensure_ascii=False,+ separators=(",", ":"),+ sort_keys=True,+ )+ + "\n"+ ).encode()+ except (TypeError, ValueError) as exc:+ raise BackendContractError(f"invalid training event: {exc}") from exc+ descriptor = os.open(self.path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o644)+ try:+ os.write(descriptor, payload)+ finally:+ os.close(descriptor)+++class TrainingEngine:+ """Run finite updates; unsafe steps never reach optimizer, EMA, save, or validation."""++ def __init__(+ self,+ runtime: TrainingRuntime,+ policy: StepPolicy,+ *,+ event_sink: EventSink | None = None,+ ) -> None:+ self.runtime = runtime+ self.policy = policy+ self.event_sink = event_sink or (lambda _: None)++ def run(self) -> int:+ with suspend_automatic_cycle_collection():+ while self.runtime.global_step < self.policy.max_steps:+ self._step()+ return self.runtime.global_step++ def _step(self) -> None:+ next_step = self.runtime.global_step + 1+ self.runtime.zero_grad()+ microbatches: list[MicrobatchResult] = []+ try:+ for micro_index in range(self.policy.grad_accum):+ result = self.runtime.train_microbatch(micro_index, self.policy.grad_accum)+ self.runtime.assert_sp_peer_identity(result.identity)+ microbatches.append(result)+ except Exception:+ self.runtime.zero_grad()+ raise++ gradient = self.runtime.prepare_optimizer_step()+ if not gradient.finite or not math.isfinite(gradient.norm):+ self.runtime.zero_grad()+ raise BackendContractError(+ f"non-finite gradients before optimizer step {next_step}: {gradient.norm}"+ )++ self.runtime.optimizer_step()+ self.runtime.scheduler_step()+ self.runtime.set_global_step(next_step)+ self.runtime.ema_update(next_step)+ self.runtime.zero_grad()++ losses = _mean_losses(microbatches)+ event: dict[str, Any] = {+ "schema": "solarwm.training-step.v1",+ "event": "optimizer_step",+ "step": next_step,+ "grad_accum": self.policy.grad_accum,+ "gradient_norm": gradient.norm,+ "losses": losses,+ "microbatches": [+ {"identity": asdict(result.identity), "losses": dict(result.losses)}+ for result in microbatches+ ],+ }+ self.event_sink(event)++ crossed_boundary = False+ if self.policy.should_save(next_step):+ checkpoint_id = self.runtime.save_checkpoint(next_step)+ if not checkpoint_id:+ raise BackendContractError("checkpoint hook returned an empty identity")+ self.event_sink(+ {+ "schema": "solarwm.training-event.v1",+ "event": "checkpoint",+ "step": next_step,+ "checkpoint_id": checkpoint_id,+ }+ )+ crossed_boundary = True+ if self.policy.should_validate(next_step):+ report = self.runtime.validate(next_step)+ self.event_sink(+ {+ "schema": "solarwm.training-event.v1",+ "event": "validation",+ "step": next_step,+ "report": dict(report),+ }+ )+ crossed_boundary = True+ if crossed_boundary:+ gc.collect()+++def _mean_losses(results: Sequence[MicrobatchResult]) -> dict[str, float]:+ names = set(results[0].losses)+ if any(set(result.losses) != names for result in results):+ raise BackendContractError("loss fields changed inside one optimizer step")+ return {+ name: sum(float(result.losses[name]) for result in results) / len(results)+ for name in sorted(names)+ }diff --git a/src/solarwm/training/schedule.py b/src/solarwm/training/schedule.pynew file mode 100644index 0000000..bae85f8--- /dev/null+++ b/src/solarwm/training/schedule.py@@ -0,0 +1,51 @@+"""Pure learning-rate schedule math plus an optional torch adapter."""++from __future__ import annotations++import math+from typing import Any++from solarwm.errors import BackendContractError+++def warmup_cosine_factor(+ step: int, *, warmup_steps: int, total_steps: int, min_lr_ratio: float = 0.1+) -> float:+ # A bounded training run may intentionally stop before the warmup inherited+ # from the full training recipe completes. In that case+ # ``total_steps < warmup_steps`` is well-defined: every executed step uses+ # the same warmup prefix as the full run, and the cosine phase is never+ # reached.+ if step < 0 or warmup_steps < 0 or total_steps < 1:+ raise BackendContractError("invalid warmup/cosine step bounds")+ if not 0 <= min_lr_ratio <= 1:+ raise BackendContractError("min_lr_ratio must be in [0,1]")+ if step < warmup_steps:+ return float(step) / max(1, warmup_steps)+ progress = (step - warmup_steps) / max(1, total_steps - warmup_steps)+ progress = min(1.0, progress)+ cosine = 0.5 * (1 + math.cos(math.pi * progress))+ return cosine * (1 - min_lr_ratio) + min_lr_ratio+++def make_warmup_cosine(+ optimizer: Any,+ *,+ warmup_steps: int,+ total_steps: int,+ min_lr_ratio: float = 0.1,+) -> Any:+ try:+ import torch+ except ImportError as exc:+ raise BackendContractError("torch is required to construct a scheduler") from exc++ def factor(step: int) -> float:+ return warmup_cosine_factor(+ step,+ warmup_steps=warmup_steps,+ total_steps=total_steps,+ min_lr_ratio=min_lr_ratio,+ )++ return torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=factor)diff --git a/src/solarwm/training/sgf.py b/src/solarwm/training/sgf.pynew file mode 100644index 0000000..de8605e--- /dev/null+++ b/src/solarwm/training/sgf.py@@ -0,0 +1,93 @@+"""Shared self-gradient-forcing schedule and detached score surrogate."""++from __future__ import annotations++import torch+from torch import Tensor+++def should_update_student(global_step: int, critic_updates_per_student: int) -> bool:+ """Match the reference schedule: warm critic first, then update every N steps."""+ ratio = int(critic_updates_per_student)+ if ratio < 1:+ raise ValueError("train.critic_updates_per_student must be >= 1")+ step = int(global_step)+ return step > 0 and step % ratio == 0+++def _expanded_mask(mask: Tensor | None, value: Tensor) -> Tensor | None:+ if mask is None:+ return None+ try:+ return torch.broadcast_to(mask.to(device=value.device, dtype=torch.bool), value.shape)+ except RuntimeError as exc:+ raise ValueError(+ f"mask shape {tuple(mask.shape)} is not broadcastable to {tuple(value.shape)}"+ ) from exc+++def project_out_residual(update: Tensor, residual: Tensor) -> Tensor:+ """PDMD (Projected Distribution Matching Distillation, 2609.35768, Eq. 4): ``update`` with its component along+ ``residual`` removed, per sample (dim 0). The residual is the critic endpoint minus the student endpoint - an+ unbiased estimate of the critic's error - so the part of the DMD update parallel to it is mostly that error. A+ sample whose residual vanishes keeps its update unchanged."""++ d = update.float()+ r = residual.float()+ dims = tuple(range(1, d.ndim))+ rr = r.square().sum(dim=dims, keepdim=True)+ dr = (d * r).sum(dim=dims, keepdim=True)+ coef = torch.where(rr > 0, dr / rr.clamp_min(torch.finfo(torch.float32).tiny), torch.zeros_like(rr))+ return d - coef * r+++def compute_sgf_kl_gradient(+ *,+ fake_x0: Tensor,+ real_x0: Tensor,+ student_output: Tensor,+ mask: Tensor | None = None,+ normalize: bool = True,+) -> Tensor:+ """Compute the reference DMD gradient, with stable masked normalization."""+ fake = fake_x0.float()+ real = real_x0.float()+ output = student_output.float()+ grad = fake - real+ expanded = _expanded_mask(mask, grad)+ if expanded is not None:+ # Mask before every reduction/non-linearity. A conditioned frame can+ # legitimately contain non-finite score output at t=0; NaN * 0 would+ # otherwise poison the normalizer and silently zero every valid frame.+ grad = torch.where(expanded, grad, torch.zeros_like(grad))+ if normalize:+ distance = (output - real).abs()+ reduce_dims = tuple(range(1, distance.ndim))+ if expanded is None:+ normalizer = distance.mean(dim=reduce_dims, keepdim=True)+ else:+ distance = torch.where(expanded, distance, torch.zeros_like(distance))+ weights = expanded.float()+ normalizer = distance.sum(dim=reduce_dims, keepdim=True)+ normalizer = normalizer / weights.sum(dim=reduce_dims, keepdim=True).clamp_min(1.0)+ grad = grad / normalizer.clamp_min(1e-8)+ return torch.nan_to_num(grad)+++def sgf_student_loss(+ student_output: Tensor,+ kl_gradient: Tensor,+ *,+ mask: Tensor | None = None,+) -> Tensor:+ """Surrogate whose derivative with respect to student output is the KL gradient."""+ output = student_output.float()+ target = (output - kl_gradient.detach()).detach()+ diff = output - target+ expanded = _expanded_mask(mask, diff)+ if expanded is None:+ return 0.5 * diff.square().mean()+ # Mask before square so a non-finite conditioned frame has zero gradient+ # instead of producing NaN through a later multiplication by zero.+ diff = torch.where(expanded, diff, torch.zeros_like(diff))+ return 0.5 * diff.square().sum() / expanded.float().sum().clamp_min(1.0)diff --git a/src/solarwm/version.py b/src/solarwm/version.pynew file mode 100644index 0000000..02dc1c1--- /dev/null+++ b/src/solarwm/version.py@@ -0,0 +1,3 @@+"""Single source of truth for the package version."""++__version__ = "0.1.0"diff --git a/tools/hf_samples.py b/tools/hf_samples.pynew file mode 100644index 0000000..db858b9--- /dev/null+++ b/tools/hf_samples.py@@ -0,0 +1,89 @@+"""Turn the released training shards into the sample list tools/omni_preencode.py reads.++ python tools/hf_samples.py --data ./SolarRenderer-Data --out ./samples [--sources magpie,carla,unrealzoo]+ [--per-source N]++Reads the WebDataset shards of the Hugging Face dataset (train/<source>/*.tar), writes the files each+clip needs under <out>/<source>/<clip>/ and one sample per clip to <out>/samples.jsonl. A clip's observations follow+its ``training_input``: the structure laid over every frame, the first frame on latent frame 0, and the prompt is the+one written for that combination. --per-source keeps the first N clips of each source (a quick test). A rerun skips files that are already written.+"""++from __future__ import annotations++import argparse+import glob+import json+import os+import tarfile++INPUTS = { # training_input -> (structure, first frame)+ "structure + first frame + text": (True, True),+ "structure + text": (True, False),+ "first frame + text": (False, True),+ "text only": (False, False),+}+TRAIN = {"target": "render.mp4", "structure": "condition.mp4", "first": "first_frame.png"}+++def clips(shard: str, names: dict[str, str]):+ """(key, {role: bytes}, meta) for every clip of a shard, keeping only the members named in ``names``."""+ by: dict[str, dict] = {}+ with tarfile.open(shard) as tar:+ for m in tar:+ for role, suffix in [*names.items(), ("meta", "json")]:+ if m.isfile() and m.name.endswith("." + suffix):+ by.setdefault(m.name[: -len(suffix) - 1], {})[role] = tar.extractfile(m).read()+ for key, files in sorted(by.items()):+ yield key, files, json.loads(files.pop("meta"))+++def write(path: str, data: bytes) -> str:+ if not os.path.exists(path):+ with open(path + ".part", "wb") as fh:+ fh.write(data)+ os.replace(path + ".part", path)+ return os.path.abspath(path)+++def main():+ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)+ ap.add_argument("--data", required=True, help="local copy of Enderfga/SolarRenderer-Data")+ ap.add_argument("--out", required=True)+ ap.add_argument("--sources", default="magpie,carla,unrealzoo")+ ap.add_argument("--per-source", type=int, default=0)+ a = ap.parse_args()+ os.makedirs(a.out, exist_ok=True)+ n = 0+ with open(os.path.join(a.out, "samples.jsonl"), "w") as out:+ for source in a.sources.split(","):+ pattern = f"{a.data}/train/{source}/*.tar"+ shards = sorted(glob.glob(pattern))+ if not shards:+ raise SystemExit(f"no shards at {pattern}; download them first (see docs/training.md)")+ done = 0+ for shard in shards:+ for key, files, meta in clips(shard, TRAIN):+ if a.per_source and done >= a.per_source:+ break+ use_structure, use_first = INPUTS[meta["training_input"]]+ d = os.path.join(a.out, source, key)+ os.makedirs(d, exist_ok=True)+ target = write(f"{d}/render.mp4", files["target"])+ obs = []+ if use_structure:+ obs.append({"media": write(f"{d}/structure.mp4", files["structure"]), "frames": "all"})+ if use_first:+ obs.append({"media": write(f"{d}/first_frame.png", files["first"]), "frames": [0]})+ out.write(json.dumps({"id": f"{source}/{key}", "target": target, "text": meta["prompt"],+ "observations": obs, "source": source}, ensure_ascii=False) + "\n")+ done += 1+ if a.per_source and done >= a.per_source:+ break+ print(f"{source}: {done} samples", flush=True)+ n += done+ print(f"{n} samples -> {os.path.join(a.out, 'samples.jsonl')}")+++if __name__ == "__main__":+ main()diff --git a/tools/omni_preencode.py b/tools/omni_preencode.pynew file mode 100644index 0000000..97c2841--- /dev/null+++ b/tools/omni_preencode.py@@ -0,0 +1,1043 @@+"""Preencode unified omni samples with the released MiniMax-H3 ``ref2va`` front end.++Every sample of the input JSONL is::++ {"id": "...", "target": "clip.mp4", "text": "...",+ "observations": [{"media": "x.mp4|x.png", "frames": "all" | [0, 5], "degree": 0.999}, ...],+ "known_mask": "mask.mp4" | null, "source": "dataset-name"}++``frames`` says how an observation relates to the clip: ``"all"`` or a list of target latent frames+for an aligned one, ``null`` for a free one. No field names a task or a condition type.++The prompt embeddings, their token tags and the observation latents come from the released modular+blocks (setup -> Qwen3-VL presentation -> reference VAE encode), so a sample is conditioned exactly+as the released checkpoint expects. The target goes through the same reference encoder, on its own,+to get latents in the same convention. One worker per GPU: ``--shard i --shards n``.++Media and samples may be local paths or storage URLs (``<scheme>://...``, read and written through+solarwm.storage); with URLs, node-local scratch only ever holds what a worker is busy with.+"""++from __future__ import annotations++import argparse+import contextlib+import hashlib+import json+import math+import os+import subprocess+import sys+import threading+import time+import traceback++import numpy as np+import torch++from solarwm import storage++ROOT = os.environ.get("H3_REF_ROOT", "MiniMax-H3")+# MiniMax-H3's one frame rate. Every video handed to it is put on this grid first and then cut to+# the frames asked for, so a latent's place in time is a count of these frames from the clip's+# start - not a fraction of however long the clip happened to be.+FPS = 24.0+IMAGE_EXT = (".png", ".jpg", ".jpeg", ".webp", ".bmp")+# The shortest video H3's reference blocks take (read at 2 fps, sampled frames merged in pairs).+STILL_FRAMES = 13+SCRATCH = os.environ.get("OMNI_SCRATCH", "/tmp/omni_pre")+++class WriteRefused(RuntimeError):+ """The storage would not take a member this worker had already encoded."""+++class FetchFailed(RuntimeError):+ """A row's media could not be fetched - most often because it is not there yet."""+++def copy(source: str, destination: str) -> None:+ """Copy one file to or from storage (solarwm.storage retries a busy store a few times).++ Which way it went decides whose failure it is: a write the storage refused says nothing about+ the row, a fetch that failed says the row's media is missing - and that row is set aside."""++ try:+ if storage.is_remote(destination):+ storage.put(source, destination)+ else:+ storage.get(source, destination)+ except Exception as exc:+ error = WriteRefused if storage.is_remote(destination) else FetchFailed+ raise error(f"storage copy {source} -> {destination}: {str(exc)[-300:]}") from exc+++def sweep(cache: str, keep_seconds: float = 900.0) -> None:+ """Drop cached media no worker has touched lately; a sibling may still be using a fresh one."""++ now = time.time()+ for name in os.listdir(cache):+ with contextlib.suppress(OSError):+ if now - os.path.getatime(f"{cache}/{name}") > keep_seconds:+ os.remove(f"{cache}/{name}")+++class Sink:+ """Where encoded samples go: a remote object-storage prefix, or a local directory."""++ def __init__(self, out: str, shard: int) -> None:+ self.out, self.remote = out.rstrip("/"), "://" in out+ # Scratch is per set, not just per shard. A node encodes one set after another, and an+ # index named only by its shard number would be found by the next set, appended to, and+ # published under that set's prefix - carrying the previous set's rows into it.+ self.stage = f"{SCRATCH}/out/{self.out.rsplit('/', 1)[-1]}"+ self.index_name = f"index.{shard:05d}.jsonl"+ self.index_path = f"{self.stage}/{self.index_name}"+ os.makedirs(f"{self.stage}/samples", exist_ok=True)+ if not self.remote:+ os.makedirs(f"{self.out}/samples", exist_ok=True)+ self.index_path = f"{self.out}/{self.index_name}"+ elif not os.path.exists(self.index_path):+ # A restarted shard picks up its own index and skips what it already wrote.+ with contextlib.suppress(Exception): # a first run has no index yet+ copy(f"{self.out}/{self.index_name}", self.index_path)++ def done(self) -> set[str]:+ if not os.path.exists(self.index_path):+ return set()+ with open(self.index_path) as lines:+ return {json.loads(line)["id"] for line in lines if line.strip()}++ def member(self, name: str) -> str:+ """Local path to write a member to; ``publish`` moves it where it belongs."""++ return f"{self.stage if self.remote else self.out}/{name}"++ def publish(self, name: str) -> None:+ if self.remote:+ # Copied, then dropped here - not moved. A move that is tried again after it failed+ # has nothing left to move, because the first try took the file whether or not it+ # reported success, and the worker then throws away a member the storage already holds.+ copy(f"{self.stage}/{name}", f"{self.out}/{name}")+ with contextlib.suppress(OSError):+ os.remove(f"{self.stage}/{name}")++ def publish_index(self) -> None:+ if self.remote:+ copy(self.index_path, f"{self.out}/{self.index_name}")+++def fetch_fresh(path: str, destination: str) -> str:+ """The current version of a sample list, one copy per node.++ A list is rewritten under its own name, so a copy is kept per version - its size and time in+ the storage - and a new version is fetched as soon as there is one. The biggest lists run to+ gigabytes; every worker of every node fetching its own each pass would read terabytes+ for nothing. The first worker to get the lock fetches, the others wait and share its copy.+ ``destination`` is kept for the callers' sake; the copy lives in the node's list cache.+ """++ import fcntl++ if "://" not in path:+ return path+ found = storage.stat(path)+ version = f"{found[0]}_{int(found[1] or 0)}" if found else "unknown"+ cache = os.environ.get("OMNI_LIST_CACHE", "/tmp/omni_lists")+ os.makedirs(cache, exist_ok=True)+ local = f"{cache}/{os.path.basename(path)}.{version}"+ if os.path.exists(local):+ return local+ with open(f"{local}.lock", "w") as lock:+ fcntl.flock(lock, fcntl.LOCK_EX)+ if not os.path.exists(local):+ copy(path, f"{local}.part")+ os.replace(f"{local}.part", local)+ # Older versions of the same list are no longer anyone's.+ for name in os.listdir(cache):+ if name.startswith(os.path.basename(path) + ".") and not name.startswith(os.path.basename(local)):+ try:+ os.remove(f"{cache}/{name}")+ except OSError:+ pass+ return local+++def localize(path: str) -> str:+ """A remote path is fetched to local scratch once; anything else is used as is."""++ if "://" not in path:+ return path+ import hashlib++ local = f"{SCRATCH}/in/{hashlib.sha1(path.encode()).hexdigest()}{os.path.splitext(path)[1]}"+ if not os.path.exists(local):+ os.makedirs(f"{SCRATCH}/in", exist_ok=True)+ part = f"{local}.{os.getpid()}.part" # the workers of a node share this cache+ copy(path, part)+ os.replace(part, local)+ return local+++def open_media(path: str):+ """A PyAV container that tolerates undecodable metadata; newer PyAV releases dropped the option."""++ import av++ try:+ return av.open(path, metadata_errors="ignore")+ except TypeError:+ return av.open(path)+++def read_video(path: str, seconds: float = 0.0) -> tuple[np.ndarray, float]:+ """Decode a clip - only its first ``seconds`` when given.++ H3 keeps 6.6 s of any clip, and OmniHuman alone holds 4K clips a thousand frames long: decoding+ all of one to use its start kept one worker busy for most of a pass while its seven siblings,+ finished, held idle cards waiting for it.+ """++ import av++ path = localize(path)+ container = open_media(path)+ stream = container.streams.video[0]+ stream.thread_type = "AUTO" # frame threads: otherwise a worker decodes on a single core+ fps = float(stream.average_rate)+ limit = int(math.ceil(seconds * fps)) + 2 if seconds else 0+ frames = []+ for frame in container.decode(stream):+ frames.append(frame.to_ndarray(format="rgb24"))+ if limit and len(frames) >= limit:+ break+ container.close()+ return np.stack(frames), fps+++def prepared(frames: np.ndarray, fps: float, num_frames: int) -> np.ndarray:+ """A clip exactly as H3 would make it - on 24 fps, cut to ``num_frames``, LANCZOS-resized to the+ canvas its own aspect resolves to - with the resize spread over threads.++ H3 does this itself, one frame after another on one core, while the card waits: a quarter of+ every card's time went to it. A clip handed over already at 24 fps and at that size goes+ through H3 untouched, so doing the same steps here, in parallel, changes no pixel.+ """++ from concurrent.futures import ThreadPoolExecutor++ from diffusers.modular_pipelines.minimax_h3.modular_pipeline import resolve_canvas_size+ from PIL import Image++ frames = on_grid(frames, fps)[:num_frames]+ # The released checkpoint's canvas rule: axes on multiples of 32, short edge 768, area 768x1344.+ height, width = resolve_canvas_size(frames.shape[2], frames.shape[1], 32, 768, 768 * 1344)+ if frames.shape[1:3] == (height, width):+ return np.ascontiguousarray(frames)++ def one(frame: np.ndarray) -> np.ndarray:+ return np.asarray(Image.fromarray(frame).resize((width, height), Image.Resampling.LANCZOS))++ with ThreadPoolExecutor(16) as pool:+ return np.stack(list(pool.map(one, frames)))+++def frame_rate(path: str) -> float:+ """A clip's frame rate from its container."""++ import av++ container = open_media(localize(path))+ rate = float(container.streams.video[0].average_rate)+ container.close()+ return rate+++def label_mask(path: str, label: int, fps: float, start: float, span: float):+ """One object's region out of a per-frame label map (VPData's all_masks.npz: every frame of the+ clip, each pixel the number of the object on it), cut to the stretch that is needed, as frames+ the rest of the mask handling reads: white where the object is. It runs on the clip's clock -+ its frame i is the clip's frame i - so the clip's own rate is its rate."""++ labels = np.load(localize(path))["arr_0"]+ first = int(round(start * fps))+ last = int(math.ceil((start + span) * fps)) + 2+ # Already cut at the start: the caller must not cut it again.+ return (labels[first:last] == label).astype(np.uint8)[..., None] * 255, fps+++def frame_count(path: str) -> int:+ """How many frames a clip holds, from its container rather than by decoding it."""++ import av++ container = open_media(localize(path))+ stream = container.streams.video[0]+ count = int(stream.frames or 0)+ if not count:+ seconds = float(stream.duration * stream.time_base) if stream.duration else (container.duration or 0) / 1e6+ count = int(round(seconds * float(stream.average_rate)))+ container.close()+ return count+++def on_grid(frames: np.ndarray, fps: float) -> np.ndarray:+ """A clip put on 24 fps the way H3 puts a reference there: whole frames dropped or held."""++ if fps == FPS:+ return frames+ scale = FPS / fps+ slots = np.floor(np.arange(frames.shape[0]) * scale + 0.5).astype(np.int64)+ end = math.floor(frames.shape[0] * scale + 0.5)+ return np.repeat(frames, np.diff(slots, append=end), axis=0)+++def crop_scale(size: tuple[int, int], canvas_shape: tuple[int, int]) -> tuple[float, float]:+ """What fraction of the height and width a centre-crop to the canvas aspect keeps."""++ height, width = size+ wanted = canvas_shape[1] / canvas_shape[0]+ if abs(width / height - wanted) < 1e-3:+ return 1.0, 1.0+ if width / height > wanted:+ return 1.0, round(height * wanted) / width+ return round(width / wanted) / height, 1.0+++def fit(frames: np.ndarray, canvas_shape: tuple[int, int]) -> np.ndarray:+ """Centre-crop to the canvas's own aspect, because the encoder will not stretch to it.++ The reference blocks fit a frame inside the canvas and keep its aspect, so a clip that is not+ 1344:768 comes out on a smaller grid than the model's - 48x64 for four-thirds, 48x48 for a+ square render - and the reader will not admit it. Cropping keeps every pixel it keeps real;+ padding would teach the model to draw the bars.+ """++ height, width = frames.shape[1:3]+ wanted = canvas_shape[1] / canvas_shape[0]+ if abs(width / height - wanted) < 1e-3:+ return frames+ if width / height > wanted: # too wide: take a full-height slice+ keep = int(round(height * wanted))+ start = (width - keep) // 2+ return frames[:, :, start : start + keep]+ keep = int(round(width / wanted)) # too tall: take a full-width slice+ start = (height - keep) // 2+ return frames[:, start : start + keep, :]+++def region(frames: np.ndarray, box) -> np.ndarray:+ """The part of every frame a dataset's own crop box keeps, ``[x0, x1, y0, y1]`` in the clip's+ pixels. Some sets cut their clips down to a region of the frame - past a subtitle, a watermark,+ a letterbox, or onto the part that holds the subject - and name the box rather than re-encode+ millions of files; whatever is laid over the clip frame for frame is cut with it."""++ if not box:+ return frames+ x0, x1, y0, y1 = (int(v) for v in box)+ height, width = frames.shape[1:3]+ if not (0 <= x0 < x1 <= width and 0 <= y0 < y1 <= height):+ raise ValueError(f"crop box {list(box)} does not fit the {width}x{height} frame")+ return frames[:, y0:y1, x0:x1]+++def within(image, limit: float = 4.0):+ """The centre of an image cut to the widest and tallest shapes H3 takes, 4:1 and 1:4."""++ width, height = image.size+ if width > limit * height:+ keep = int(limit * height)+ return image.crop(((width - keep) // 2, 0, (width - keep) // 2 + keep, height))+ if height > limit * width:+ keep = int(limit * width)+ return image.crop((0, (height - keep) // 2, width, (height - keep) // 2 + keep))+ return image+++def aligned_image(image, canvas_shape: tuple[int, int], box=None):+ """An image laid over the target's canvas (a first frame, a still of an aligned stream): cut to the+ canvas's aspect and LANCZOS-resized to its size, as every frame of the target is. Handed to H3 as+ an image, a first frame became a 128x224 latent - off the target's 48x84 grid - and some seven+ thousand extra prompt tokens; ``reference`` hands it over as a one-frame video at this size."""++ from PIL import Image++ pixels = fit(region(np.asarray(image)[None], box), canvas_shape)[0]+ height, width = canvas_shape+ if pixels.shape[:2] == (height, width):+ return Image.fromarray(pixels)+ return Image.fromarray(pixels).resize((width, height), Image.Resampling.LANCZOS)+++def reference(+ path: str,+ canvas_shape: tuple[int, int],+ video=None,+ crop: bool = True,+ start: float = 0.0,+ num_frames: int = 158,+ box=None,+):+ """One reference for H3. ``crop`` puts it on the canvas's shape; only what is laid over the+ target's canvas - the target itself, and an observation aligned with it frame for frame - has to+ be. A free reference (a portrait, an object, another shot) sits on its own spatial grid, as it+ does in H3 itself, so it keeps its own shape: cutting a full-length portrait to the canvas's+ 7:4 kept a band across its middle, a torso with no face and no feet."""++ from diffusers.modular_pipelines.minimax_h3 import (+ MiniMaxH3ImageReference,+ MiniMaxH3VideoReference,+ )+ from PIL import Image++ if path.lower().endswith(IMAGE_EXT):+ image = Image.open(localize(path)).convert("RGB")+ if not crop:+ return MiniMaxH3ImageReference(image=within(image))+ # A still laid over the target is one frame of it, so it is encoded the way the target's frames+ # are: as video, on the target's grid (as an image H3 brings it to its own still size). H3+ # takes no video shorter than STILL_FRAMES, so the still is held that long; the causal VAE's+ # first latent sees the first frame alone, and that latent is all the sample keeps.+ frame = np.asarray(aligned_image(image, canvas_shape, box))[None]+ return MiniMaxH3VideoReference(frames=np.repeat(frame, STILL_FRAMES, axis=0), fps=FPS)+ frames, fps = video if video is not None else read_video(path, start + num_frames / FPS + 0.5)+ if start:+ frames = frames[int(round(start * fps)) :]+ # The target's own crop box, for a reference laid over it; a free one is never cut.+ frames = fit(region(frames, box), canvas_shape) if crop else frames+ return MiniMaxH3VideoReference(frames=prepared(frames, fps, num_frames), fps=FPS)+++def build_pipelines(device: str):+ from diffusers import ComponentsManager, ModularPipeline+ from diffusers.modular_pipelines import SequentialPipelineBlocks++ blocks = ModularPipeline.from_pretrained(ROOT).blocks+ manager = ComponentsManager()+ # Weights stay on the card. Offloading moved the text encoder over PCIe on every call, which+ # cost little per sample once warm but made the first call of every process several times+ # slower. The heaviest sample, 47 latents with four observations, fits on the card;+ # offloading remains, behind a switch, for a shape that would not fit.+ offload = os.environ.get("OMNI_CPU_OFFLOAD") == "1"+ if offload:+ manager.enable_auto_cpu_offload(device=device)++ def chain(workflow: str, names: tuple[str, ...]):+ sub = blocks.get_workflow(workflow).sub_blocks+ return SequentialPipelineBlocks.from_blocks_dict(+ {name: sub[name] for name in names}+ ).init_pipeline(ROOT, components_manager=manager)++ conditioned = chain("ref2va", ("before_encode", "text_encoder", "vae_encoder"))+ conditioned.load_components(dtype=torch.bfloat16)+ if not offload:+ for name in conditioned.pretrained_component_names:+ component = getattr(conditioned, name, None)+ if isinstance(component, torch.nn.Module):+ component.to(device)+ # One copy of every weight: the other chains borrow the components (the text encoder is 62 GB).+ loaded = {name: getattr(conditioned, name) for name in conditioned.pretrained_component_names}+ latents_only = chain("ref2va", ("before_encode", "vae_encoder"))+ text_only = chain("t2va", ("text_encoder",))+ for pipe in (latents_only, text_only):+ pipe.update_components(+ **{k: v for k, v in loaded.items() if k in pipe.pretrained_component_names}+ )+ return conditioned, latents_only, text_only+++def canvas(path: str, box=None) -> tuple[int, int]:+ """The canvas a clip is encoded on: landscape or portrait by the part of the frame that is kept."""++ import av++ stream = open_media(localize(path)).streams.video[0]+ width, height = stream.width, stream.height+ if box:+ width, height = int(box[1]) - int(box[0]), int(box[3]) - int(box[2])+ return (768, 1344) if width >= height else (1344, 768)+++def source_size(path: str) -> tuple[int, int]:+ import av++ stream = open_media(localize(path)).streams.video[0]+ return stream.height, stream.width+++def encode(sample: dict, pipes, num_frames: int) -> tuple[dict, dict]:+ conditioned, latents_only, text_only = pipes+ box = sample.get("crop")+ # A crop box moves the picture's origin and scales its field of view, and a camera track's+ # intrinsics would have to follow; nothing here does that, so the two are refused together+ # before any card time is spent.+ if box and sample.get("camera"):+ raise ValueError("a crop box and a camera track together are not supported")+ height, width = canvas(sample["target"], box)+ size = {"height": height, "width": width, "num_frames": num_frames}+ shape = (height, width)+ # Where in the clip the sample starts, in seconds. The encoder keeps 6.6 s of a clip; a sample+ # whose moment is later in a longer clip - a keyframe at 20 s of 22 - names where its stretch+ # begins, and the target, every observation on the clip's clock, the camera and the mask all+ # start there together. Free references are not on the clip's clock and are not moved.+ start = float(sample.get("start") or 0.0)+ # A start given as a frame number is turned into seconds by the clip's own rate: a list's+ # rounded "fps 23" for a 23.976 clip would put frame 736 1.3 s off.+ if sample.get("start_frame"):+ start = int(sample["start_frame"]) / frame_rate(sample["target"])+ span = start + num_frames / FPS + 0.5 # what of the clip the latents can hold, and a margin+ video = read_video(sample["target"], span)+ video = (region(video[0], box), video[1])+ # The clip as it was made, before anything is cut off: counted by the container, since only its+ # start was decoded. A camera track made one pose per frame of the whole clip needs it.+ whole_frames = max(frame_count(sample["target"]), int(video[0].shape[0]))+ if start:+ video = (video[0][int(round(start * video[1])) :], video[1])+ if len(video[0]) < 2:+ raise ValueError(f"start {start:.2f} s is past the end of the clip")+ # Where the sample's stretch of the clip ends, as a frame of the clip, when a set curates a+ # stretch shorter than the encoder's window: reading on past it would take in what the set cut+ # away - often the next shot. Everything on the clip's clock is held to the same stretch,+ # because it is counted from the target's frames below.+ if sample.get("end_frame"):+ keep = int(sample["end_frame"]) - int(round(start * video[1]))+ if keep < 2:+ raise ValueError(f"end frame {sample['end_frame']} leaves {keep} frames after the start")+ video = (video[0][:keep], video[1])+ clip_frames, clip_fps = int(video[0].shape[0]), float(video[1])+ target_state = latents_only(+ references=[reference(sample["target"], shape, video, num_frames=num_frames)],+ prompt="",+ **size,+ )+ del video+ target = target_state.get("condition_latents")[0][0] # [24, F, h, w]+ frames = int(target.shape[1])+ # What of the clip the target latents hold: H3 puts the clip on 24 fps and keeps the first+ # num_frames, so a clip longer than that is encoded from its start and the rest is not there.+ # Anything else that runs on the clip's clock - a camera track, a mask that came with the+ # clip - has to be read over this same stretch, or it describes seconds the latents never saw.+ window = min(int(math.floor(clip_frames * FPS / clip_fps + 0.5)), num_frames)++ def seconds(latent: int) -> float:+ """Where a latent sits in the clip, in seconds from its start."""+ return latent * (window - 1) / max(frames - 1, 1) / FPS+ # Two latents is five frames, a fifth of a second: a still with a timestamp. Nothing about+ # motion, continuation or timing can be learned from it, so it is refused by name here and+ # counted, rather than encoded and quietly drawn in training.+ if frames <= 2:+ raise ValueError(f"target is {frames} latents ({17 * (frames - 2) // 5 + 5} frames), too short to be a video")+ tensors = {"target_latents": target.to(torch.bfloat16).contiguous()}+ observations = []+ items = sample.get("observations") or []+ if items:+ state = conditioned(+ prompt=sample["text"],+ references=[+ reference(+ o["media"],+ shape,+ crop=o.get("frames") is not None,+ start=start if o.get("frames") is not None else 0.0,+ num_frames=num_frames,+ box=box if o.get("frames") is not None else None,+ )+ for o in items+ ],+ **size,+ )+ for index, (item, latents) in enumerate(+ zip(items, state.get("condition_latents"), strict=True)+ ):+ latents = latents[0]+ if item["media"].lower().endswith(IMAGE_EXT) and item.get("frames") is not None:+ latents = latents[:, :1] # an aligned still: its first latent (see ``reference``)+ aligned = item.get("frames")+ if aligned == "all":+ aligned = list(range(min(int(latents.shape[1]), frames)))+ latents = latents[:, : len(aligned)]+ if aligned is not None and len(aligned) != int(latents.shape[1]):+ raise ValueError(+ f"observation {index} has {int(latents.shape[1])} latent frames "+ f"for {len(aligned)} target frames"+ )+ tensors[f"observation_{index}_latents"] = latents.to(torch.bfloat16).contiguous()+ observations.append(+ {+ "frames": aligned,+ "degree": float(item.get("degree", 0.999)),+ "boxes": item.get("boxes"),+ }+ )+ else:+ state = text_only(prompt=sample["text"])+ prompt = state.get("prompt_embeds")+ tensors["prompt_embeds"] = prompt.reshape(-1, prompt.shape[-1]).to(torch.bfloat16).contiguous()+ tensors["text_token_tags"] = (+ state.get("text_token_tags").reshape(-1).to(torch.int64).contiguous()+ )+ if sample.get("camera"):+ # The sample hands over absolute camera-to-world poses and normalised intrinsics; the+ # reader makes them relative.+ loaded = np.load(localize(sample["camera"]))+ views_all = torch.as_tensor(np.asarray(loaded["c2w"]), dtype=torch.float32)+ k_all = torch.as_tensor(np.asarray(loaded["K"]), dtype=torch.float32)+ total = int(views_all.shape[0])+ if k_all.ndim != 2 and int(k_all.shape[0]) < total:+ raise ValueError(f"camera holds {total} poses but {int(k_all.shape[0])} intrinsics")+ # One camera per latent, taken at the latent's time. A track says its own rate when it was+ # made over part of a clip; one without says nothing because it was made one pose per+ # frame of the whole clip. Spreading the whole track over the latents instead - as this+ # once did - is right only for a clip no longer than the window; for a 20 s clip it laid+ # 20 s of camera motion over the 6.6 s the latents hold.+ track_fps = float(loaded["fps"]) if "fps" in loaded.files else total * clip_fps / whole_frames+ # A track starts where the clip does unless it says otherwise, so a sample that starts+ # later reads it from that far in.+ offset = start - (float(loaded["start"]) if "start" in loaded.files else 0.0)+ if offset < 0:+ raise ValueError(f"camera track starts {-offset:.2f} s after the sample does")+ if offset + seconds(frames - 1) > (total - 1) / track_fps + 1.0 / FPS:+ raise ValueError(+ f"camera track covers {(total - 1) / track_fps:.2f} s, the sample needs "+ f"{offset + seconds(frames - 1):.2f} s"+ )+ picks = [+ min(int(round((offset + seconds(index)) * track_fps)), total - 1)+ for index in range(frames)+ ]+ tensors["camera_c2w"] = views_all[picks].contiguous()+ intrinsics = (k_all if k_all.ndim == 2 else k_all[picks]).clone()+ # The intrinsics describe the clip as it was shot; the encoder sees a centre-crop of it.+ # A normalised focal length is pixels over the frame's own size, so cropping the frame+ # raises it by exactly the fraction cropped away - and a field of view that disagrees with+ # the picture is worse than no intrinsics at all.+ keep_h, keep_w = crop_scale(source_size(sample["target"]), (height, width))+ if keep_h != 1.0 or keep_w != 1.0:+ intrinsics[..., 0, 0] /= keep_w+ intrinsics[..., 1, 1] /= keep_h+ # A long lens on a frame that had to be cropped hard ends up narrower than the released+ # contract can express (normalised focal at most 4, about a 14 degree vertical view). The+ # sample is dropped by name here rather than written out for the reader to reject later,+ # because a rejected member is a silent loss and a skipped sample is a counted one.+ focal = torch.stack([intrinsics[..., 0, 0].max(), intrinsics[..., 1, 1].max()])+ if float(focal.max()) > 4.0 or float(focal.min()) <= 0.0:+ raise ValueError(+ f"cropped focal {float(focal.max()):.2f} is outside the contract's 0..4; "+ "this lens is narrower than the canvas can carry"+ )+ tensors["camera_K"] = intrinsics.contiguous()+ if sample.get("known_mask"):+ # Which clock the mask runs on. One that came with the clip runs on the clip's: its frame j+ # is the clip at j / mask_fps, so it is put on 24 fps and cut to the window exactly as the+ # target was, and frame for frame it covers what the latents cover. One from the generated+ # library says what is given over the window, whatever the clip's length, and is stretched+ # over it. A dataset's mask marks the thing in the clip, so that is its default.+ clock = sample.get("known_mask_clock") or (+ "clip" if sample.get("known_mask_marks_the_region") else "window"+ )+ # white = keep (given), black = generate. A mask on the window's clock is read whole.+ if sample["known_mask"].endswith(".npz"):+ mask, mask_fps = label_mask(+ sample["known_mask"], int(sample["known_mask_label"]), clip_fps, start,+ num_frames / FPS + 0.5,+ )+ skip = 0 # a label map is returned already cut at the start+ else:+ mask, mask_fps = read_video(sample["known_mask"], span if clock == "clip" else 0.0)+ skip = int(round(start * mask_fps))+ if clock == "clip":+ # A mask of the clip's own shares its pixels, so the target's crop box cuts it too.+ mask = on_grid(region(mask[skip:], box), mask_fps)[:window]+ if len(mask) < window: # a mask a frame short of its clip holds its last frame+ mask = np.concatenate([mask, np.repeat(mask[-1:], window - len(mask), axis=0)])+ # The target is centre-cropped to the canvas, so a mask that names a real region has to be+ # cropped the same way or it points at the wrong part of the picture. A mask already at the+ # canvas's aspect - every mask in the generated library is - comes back unchanged.+ mask = fit(mask, (height, width))+ keep = torch.from_numpy(mask[..., 0].astype(np.float32) / 255.0)[None, None]+ # A mask that came with a dataset marks the thing, not what is kept around it. Saying so+ # on the sample is cheaper and more honest than re-encoding the file: the sample records+ # which way round its own mask is written, and the convention downstream stays one.+ if sample.get("known_mask_marks_the_region"):+ keep = 1.0 - keep+ grid = torch.nn.functional.interpolate(+ keep,+ size=(frames, target.shape[2] // 2, target.shape[3] // 2),+ mode="trilinear",+ align_corners=False,+ )+ # A row is given only where the whole patch is: VAE fields leak across a mask edge.+ tensors["known_degrees"] = (grid[0, 0] > 0.98).to(torch.float16).reshape(-1).contiguous()+ meta = {+ "observations": observations,+ "latent_frames": frames,+ "canvas": [height, width],+ # How long the clip was and how much of it the latents hold, so a member that covers only+ # the start of a long clip says so.+ "clip_seconds": round(whole_frames / clip_fps, 3),+ "start_seconds": round(start, 3),+ "window_seconds": round(window / FPS, 3),+ "source": sample.get("source", ""),+ "crop": list(box) if box else None,+ # What this sample was made from, kept beside it. The text is the one thing that can be+ # rewritten later without re-encoding the pixels - a pass over the members recomputing+ # prompt_embeds costs the text encoder only - but that pass can only work if a member+ # still says which clip it came from. Without these two lines a rewrite would depend on+ # a sample list that has since been regenerated, and the corpus would have to be built+ # again to change a word of its text.+ "target": sample.get("target", ""),+ "media": [item.get("media", "") for item in items],+ "known_mask": sample.get("known_mask", ""),+ "text": sample["text"][:16000],+ }+ return tensors, meta+++# A row that failed is left alone for this long before anyone tries it again. Most failures are the+# clip itself - a stream that will not decode, a clip too short to be a video - and fail the same way+# every time; retried on every pass by every worker, a set's last thousand such rows held every+# card idle. A day later they are tried once more, for the few that failed on a passing+# hiccup (a read from storage that timed out, a list naming media still being unpacked).+RETRY_FAILED_AFTER = float(os.environ.get("OMNI_RETRY_FAILED_AFTER", 86400))+++def failed_recently(out: str, name: str) -> bool:+ """Whether a worker marked this row failed within the retry window: a single request."""++ if "://" not in out:+ return False+ found = storage.stat(f"{out.rstrip('/')}/failed/{name}.txt")+ if found is None:+ return False+ return found[1] is None or time.time() - found[1] < RETRY_FAILED_AFTER+++def not_the_row(error: BaseException) -> bool:+ """Whether a failure says nothing about the row: the card ran out of memory or CUDA failed+ (another process on the same card is enough for either), or the storage refused the member the+ worker had already encoded. A row is left unmarked for these, and tried again next pass."""++ text = str(error)+ return (+ isinstance(error, (torch.OutOfMemoryError, WriteRefused))+ or "CUDA" in text+ or "out of memory" in text+ )+++def mark_failed(out: str, name: str, reason: str) -> None:+ """Leave a note in the output location that this row failed and why, so no one tries it again soon."""++ if "://" in out:+ with contextlib.suppress(Exception):+ storage.write_bytes(f"{out.rstrip('/')}/failed/{name}.txt", reason.encode())+++def written(out: str, name: str) -> bool:+ """Whether one member is in the output location now: a single request, asked just before encoding it.++ The listing a worker holds is minutes old, and in a set's last minutes that is long enough for+ a helper and an owner - or two helpers - to each encode what the other finished a moment ago.+ Asking about the one key costs a tenth of a second against half a minute of encoding.+ """++ if "://" not in out:+ return False+ return storage.stat(f"{out.rstrip('/')}/samples/{name}.safetensors") is not None+++class Present:+ """The names the set's output location holds, kept fresh on a thread of its own.++ Two writers can hold the same row in their hands at once - a worker on its own shard and a+ sibling that finished early and came to help - and the only thing both can see is the output location.+ Listing it takes seconds once a set holds a few hundred thousand members, so the listing runs+ beside the encoder instead of in its way, and the encoder reads whatever the last one found.+ """++ def __init__(self, out: str, every: float) -> None:+ self.out, self.every = out.rstrip("/"), every+ self.names: set[str] = set()+ self.failed: dict[str, float] = {} # row -> when it last failed+ self.ok = False # whether a listing has ever come back; until then it says nothing+ if "://" in self.out:+ self.refresh()+ threading.Thread(target=self.loop, daemon=True).start()++ def refresh(self) -> None:+ # A listing that failed (None) says nothing about what is there; keep the last one that worked.+ samples = storage.listing(f"{self.out}/samples/")+ if samples is not None:+ self.names = {name[: -len(".safetensors")] for name in samples if name.endswith(".safetensors")}+ self.ok = True+ marks = storage.listing(f"{self.out}/failed/")+ if marks is not None:+ now = time.time()+ failed = {name[: -len(".txt")]: (stamp or now) for name, stamp in marks.items() if name.endswith(".txt")}+ self.failed = {**self.failed, **failed}++ def loop(self) -> None:+ while True:+ time.sleep(self.every)+ try:+ self.refresh()+ except Exception:+ pass++ def add(self, name: str) -> None:+ self.names.add(name)++ def fail(self, name: str) -> None:+ self.failed[name] = time.time()++ def __contains__(self, name: str) -> bool:+ return name in self.names or time.time() - self.failed.get(name, 0.0) < RETRY_FAILED_AFTER+++def local_weights(root: str) -> str:+ """The encoder's weights on this node's own disk, copied there once; the path to load from.++ Every pass starts a fresh set of workers, and the weights (74 GB) may live on a shared network+ filesystem whose client cache goes cold between passes - so each pass would read them all again,+ and workers starting their passes together leave every card idle while they load. A local copy stays in this node's page cache from one pass to the next. The copy is made+ by whichever worker gets the lock first; the rest wait for it. Only what the encoder uses is+ copied - not the 62 GB transformer - and the index is pointed at the copy.+ """++ import fcntl+ import re+ import shutil++ local = os.environ.get("OMNI_LOCAL_WEIGHTS", "/tmp/h3_weights")+ if not local:+ return root+ ready = f"{local}/.ready"+ if os.path.exists(ready):+ return f"{local}/ref"+ os.makedirs(local, exist_ok=True)+ with open(f"{local}.lock", "w") as lock:+ fcntl.flock(lock, fcntl.LOCK_EX)+ if os.path.exists(ready):+ return f"{local}/ref"+ index = open(f"{root}/modular_model_index.json").read()+ pipes = sorted(set(re.findall(r'"pretrained_model_name_or_path":\s*"([^"]+)"', index)) - {root})+ for src in pipes:+ dst = f"{local}/pipe{len(pipes) > 1 and '_' + str(pipes.index(src)) or ''}"+ names = [n for n in os.listdir(src) if n != "transformer"]+ # Parallel copies: one stream from a network filesystem is a fraction of what it can give.+ subprocess.run(+ ["bash", "-c", f"mkdir -p {dst} && cd {src} && printf '%s\\n' "+ + " ".join(f"'{n}'" for n in names)+ + f" | xargs -P 16 -I{{}} cp -r {{}} {dst}/"],+ check=True,+ )+ index = index.replace(f'"{src}"', f'"{dst}"')+ shutil.rmtree(f"{local}/ref", ignore_errors=True)+ os.makedirs(f"{local}/ref")+ for name in os.listdir(root):+ if name.startswith("transformer"):+ continue # the diffusion transformer is not the encoder's; its entry keeps the original path+ if os.path.isdir(f"{root}/{name}"):+ shutil.copytree(f"{root}/{name}", f"{local}/ref/{name}")+ else:+ shutil.copy(f"{root}/{name}", f"{local}/ref/{name}")+ open(f"{local}/ref/modular_model_index.json", "w").write(index)+ open(ready, "w").write("ok\n")+ return f"{local}/ref"+++def main() -> None:+ parser = argparse.ArgumentParser()+ parser.add_argument("--samples", required=True, help="sample JSONL, local or remote path")+ parser.add_argument("--out", required=True, help="output prefix (remote object storage or a local directory) for samples/ and index shards")+ parser.add_argument("--shard", type=int, default=int(os.environ.get("RANK", 0)))+ parser.add_argument("--shards", type=int, default=int(os.environ.get("WORLD_SIZE", 1)))+ parser.add_argument("--shard-offset", type=int, default=0, help="global shard = offset + rank")+ parser.add_argument(+ "--no-help",+ action="store_true",+ help="stop when this shard is done instead of taking up what is left",+ )+ # The length a clip is encoded at is the one thing about a member that cannot be changed+ # afterwards without encoding it again, and generation asks for 47 latents (158 frames).+ # Training on 37 would hand the rolling schedule a shorter window than it is asked to+ # produce. This is a cap, not a demand: a clip with fewer frames yields fewer latents.+ parser.add_argument("--num-frames", type=int, default=158)+ # A pass that runs until its rows are gone holds the node for as long as the largest set lasts,+ # and a set that matters more - one still being derived - waits the whole time. With a limit+ # the node stops taking rows, goes back to the top of its queue, and the first set with work+ # gets it; whatever it did not reach is still unwritten and is picked up next time.+ parser.add_argument("--max-seconds", type=float, default=0, help="stop taking rows after this")+ args = parser.parse_args()+ from safetensors.torch import save_file++ local = int(os.environ.get("LOCAL_RANK", 0))+ device = f"cuda:{local}"+ torch.cuda.set_device(local)+ torch.zeros(1, device=device) # bind the card at once, before the long load+ args.shard += args.shard_offset+ with open(+ fetch_fresh(args.samples, f"{SCRATCH}/samples.{os.environ.get('RANK', 0)}.jsonl")+ ) as lines:+ rows = [json.loads(line) for line in lines if line.strip()]+ # Shard by the sample's own name, not by where it sits in the list. A list that grows while+ # the set is being encoded - a signal still being derived, a pool still being unpacked - puts+ # new rows in the middle, and a position-based split would hand rows that were already done to+ # a different worker, which would encode them a second time.+ def owner(name: str) -> int:+ return int(hashlib.sha1(name.encode()).hexdigest()[:8], 16) % args.shards++ mine = [row for row in rows if owner(row["id"]) == args.shard]+ # Whose rows this worker will pick up once its own are done. A shard is an equal count of+ # clips, not an equal amount of work - a 1400-frame clip costs ten times a 130-frame one - so+ # the fast workers of a node finish long before the slow ones and their cards sit idle for the+ # rest of the pass. Rather than balance the split, which cannot be done without knowing the+ # cost in advance, a finished worker goes looking for rows nobody has written yet.+ # Strided over the whole list by the worker's place among all the set's workers, not among+ # its own node's eight. Striding by the node rank gives worker 3 of every node the same rows, and+ # two nodes in their tail at once then walk them side by side, each writing every row the+ # other writes. By the global shard the helpers' rows are disjoint across the whole set.+ others = [+ row+ for index, row in enumerate(rows)+ if index % args.shards == args.shard and owner(row["id"]) != args.shard+ ]+ sink = Sink(args.out, args.shard)+ # What this shard wrote, and what the set already holds however it got there. Asking the+ # storage as well as the index means a row another worker has already written is never+ # written again - whether that worker was a sibling helping out, or this set's split was a+ # different number of shards last time it ran.+ # Asked again every few minutes, not only at the start: helpers walk this shard's rows from+ # the far end while it walks them from the near end, and whatever a helper has already+ # written this shard must not write again when it gets there.+ present = Present(args.out, float(os.environ.get("OMNI_PRESENT_EVERY", 600)))+ # The storage is the account of what is done; this shard's index is only what some worker once+ # said it wrote under this shard number. A member moved out of the set to be encoded again is+ # still named in an index, and so is one written when the set ran under another split whose+ # shard numbers overlap this one's. Only when the storage cannot be listed is the index used.+ done = set() if present.ok else sink.done()+ global ROOT+ try:+ ROOT = local_weights(ROOT)+ except Exception: # a node whose disk will not take the copy still encodes, from the original path+ print(f"local weights unavailable: {traceback.format_exc()[-300:]}", file=sys.stderr, flush=True)+ pipes = build_pipelines(device)+ start, count = time.time(), 0+ with open(sink.index_path, "a") as index, torch.no_grad():+ for row in mine:+ if args.max_seconds and time.time() - start > args.max_seconds:+ break+ if row["id"] in done or row["id"] in present:+ continue+ if written(args.out, row["id"]):+ present.add(row["id"])+ continue+ if failed_recently(args.out, row["id"]):+ present.fail(row["id"])+ continue+ try:+ tensors, meta = encode(row, pipes, args.num_frames)+ name = f"samples/{row['id']}.safetensors"+ member = sink.member(name)+ os.makedirs(os.path.dirname(member), exist_ok=True)+ save_file(tensors, member, metadata={"omni": json.dumps(meta)})+ sink.publish(name)+ index.write(+ json.dumps(+ {+ "id": row["id"],+ "member": name,+ **{k: meta[k] for k in ("latent_frames", "canvas", "source")},+ "observations": len(meta["observations"]),+ }+ )+ + "\n"+ )+ index.flush()+ count += 1+ if count % 20 == 0:+ sink.publish_index()+ sweep(f"{SCRATCH}/in")+ print(+ f"[{args.shard}/{args.shards}] {count} encoded, "+ f"{(time.time() - start) / count:.1f} s/sample",+ flush=True,+ )+ except Exception as error: # one bad sample must not stop the shard+ reason = traceback.format_exc()[-600:]+ print(f"SKIP {row['id']}: {reason}", file=sys.stderr, flush=True)+ if not_the_row(error):+ torch.cuda.empty_cache()+ else:+ mark_failed(args.out, row["id"], reason)+ present.fail(row["id"])+ sink.publish_index()+ print(f"[{args.shard}/{args.shards}] DONE {count} new samples, helping", flush=True)++ # The helper pass. It walks the other shards' rows from the end of the list backwards. Their+ # owners are walking the same rows forwards, so a helper takes the rows its owner would reach+ # last and the two meet once, in the middle, instead of racing down the same rows side by side+ # - which is what a helper walking forwards did, writing each row just before its owner wrote+ # it again. Both sides skip whatever the storage already holds; a row encoded twice is never+ # wrong, only wasted, and meeting in the middle keeps it to the rows in flight when they meet.+ # Then everything else nobody has written yet, so a worker that is done does not sit on an idle+ # card while the pass waits for its slowest sibling. Each worker starts at its own place in+ # that list, so the ones that arrive together spread out instead of queueing on the same row.+ stride = {id(row) for row in others}+ rest = [row for row in reversed(rows) if owner(row["id"]) != args.shard and id(row) not in stride]+ turn = (args.shard * len(rest)) // max(args.shards, 1)+ order = list(reversed(others)) + rest[turn:] + rest[:turn]+ helped = 0+ if order and not args.no_help:+ with open(sink.index_path, "a") as index, torch.no_grad():+ for row in order:+ if args.max_seconds and time.time() - start > args.max_seconds:+ break+ if row["id"] in present or row["id"] in done:+ continue+ if written(args.out, row["id"]):+ present.add(row["id"])+ continue+ if failed_recently(args.out, row["id"]):+ present.fail(row["id"])+ continue+ try:+ tensors, meta = encode(row, pipes, args.num_frames)+ member = sink.member(f"samples/{row['id']}.safetensors")+ os.makedirs(os.path.dirname(member), exist_ok=True)+ save_file(tensors, member, metadata={"omni": json.dumps(meta)})+ sink.publish(f"samples/{row['id']}.safetensors")+ index.write(+ json.dumps(+ {+ "id": row["id"],+ "member": f"samples/{row['id']}.safetensors",+ "latent_frames": meta["latent_frames"],+ "canvas": meta["canvas"],+ "source": meta["source"],+ "observations": len(meta["observations"]),+ }+ )+ + "\n"+ )+ index.flush()+ present.add(row["id"])+ helped += 1+ if helped % 20 == 0:+ sink.publish_index()+ sweep(f"{SCRATCH}/in")+ print(f"[{args.shard}/{args.shards}] helped {helped}", flush=True)+ except Exception as error:+ reason = traceback.format_exc()[-600:]+ print(f"SKIP {row['id']}: {reason}", file=sys.stderr, flush=True)+ if not_the_row(error):+ torch.cuda.empty_cache()+ else:+ mark_failed(args.out, row["id"], reason)+ present.fail(row["id"])+ sink.publish_index()+ print(f"[{args.shard}/{args.shards}] DONE {count} of its own, {helped} of others", flush=True)+++if __name__ == "__main__":+ main()