Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
3.9 KB
Raw Blame
"""Reuse the pinned upstream encoder and renderer inside an isolated model process."""

from __future__ import annotations

import importlib.util
import sys
import tempfile
from pathlib import Path

from solarrenderer_model import SolarAnchor, SolarSettings


class SolarBackend:
    def __init__(self, settings: SolarSettings):
        self.settings = settings
        for path in (
            settings.source / "scripts/render.py",
            settings.h3 / "modular_model_index.json",
            settings.control,
            settings.fewstep,
        ):
            if not path.is_file():
                raise FileNotFoundError(f"Missing SolarRenderer asset: {path}")
        sys.path[:0] = [str(settings.source / "src"), str(settings.source / "tools")]
        spec = importlib.util.spec_from_file_location(
            "solarrenderer_native_render", settings.source / "scripts/render.py"
        )
        self.native = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(self.native)
        self.renderer = self.run = self.decoder = None
        self.device = "cuda:0"

    def reset(self):
        self.run = self.decoder = None

    def begin(self, anchor: SolarAnchor):
        import torch
        from solarrenderer_decode import SolarDecoder
        from solarrenderer_sampling import SolarCausalRun
        from solarwm.backends.minimax_h3.omni_data import H3OmniStream

        self.reset()
        # The native entrypoint never holds Qwen and the transformer on the card together.
        self.renderer = None
        self.native.free_cuda()
        with tempfile.TemporaryDirectory(
            prefix="solar-", dir=self.settings.scratch
        ) as directory:
            directory = Path(directory)
            structure = directory / "structure.mp4"
            structure.write_bytes(anchor.structure)
            first = directory / "first.png"
            if anchor.reference is not None:
                first.write_bytes(anchor.reference)
            member = directory / "member.safetensors"
            encoder = self.native.load_encoder(str(self.settings.h3), self.device)
            sample = self.native.sample_of(
                "reactor",
                anchor.prompt,
                str(structure),
                str(first) if anchor.reference is not None else None,
            )
            self.native.encode_sample(encoder, sample, str(member))
            del encoder
            self.native.free_cuda()
            self.renderer = self.native.load_renderer(
                str(self.settings.h3),
                str(self.settings.control),
                str(self.settings.fewstep),
                self.device,
                causal=True,
            )
            stream = object.__new__(H3OmniStream)
            stream.root, stream.current_plan_fingerprint, stream.aligned_control = (
                "/",
                "render",
                False,
            )
            batch = stream.read({"id": "reactor", "member": str(member)})
        core, conditioners, device = self.renderer
        torch.manual_seed(anchor.seed)
        with torch.no_grad():
            self.run = SolarCausalRun(
                core, batch, torch.Generator(device=device).manual_seed(anchor.seed)
            )
        self.decoder = SolarDecoder(conditioners.video_vae, self.run.frames, device)

    def step(self):
        import torch

        block = self.run.step()
        decoded = self.decoder.push(block)
        frames = None
        if decoded is not None:
            mean = torch.tensor((0.485, 0.456, 0.406)).view(1, 3, 1, 1, 1)
            std = torch.tensor((0.229, 0.224, 0.225)).view(1, 3, 1, 1, 1)
            pixels = (decoded.float() * std + mean).clamp(0, 1)
            frames = (
                pixels[0]
                .permute(1, 2, 3, 0)
                .mul(255)
                .round()
                .clamp(0, 255)
                .to(torch.uint8)
                .numpy()
            )
        return frames, self.run.complete