Skip to content

SolarRenderer: Few-Step Generative Reference Rendering

Python
4.5 KB
Raw Blame
"""Collective-free rank-local EMA for FSDP-exposed parameter shards."""

from __future__ import annotations

from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from typing import Any

from solarwm.errors import BackendContractError


def ema_decay_for_step(*, target_decay: float, global_step: int, warmup_steps: int = 0) -> float:
    decay = float(target_decay)
    step = int(global_step)
    warmup = int(warmup_steps)
    if not 0 <= decay <= 1 or step < 0 or warmup < 0:
        raise BackendContractError("invalid EMA decay/step/warmup")
    return 0.0 if step < warmup else decay


class ShardedEMA:
    """Track only locally exposed shards; save-time gathering belongs to the backend."""

    def __init__(
        self,
        module: Any,
        *,
        decay: float,
        device: Any,
        dtype: Any,
        trainable_only: bool = False,
    ) -> None:
        try:
            import torch
        except ImportError as exc:
            raise BackendContractError("torch is required for EMA") from exc
        if not 0 <= float(decay) <= 1:
            raise BackendContractError("EMA decay must be in [0,1]")
        if not isinstance(dtype, torch.dtype) or not dtype.is_floating_point:
            raise BackendContractError("EMA dtype must be floating point")
        self.decay = float(decay)
        self.device = torch.device(device)
        self.dtype = dtype
        self.trainable_only = bool(trainable_only)
        self.num_updates = 0
        self.shadow = {
            name: parameter.detach().to(device=self.device, dtype=dtype).clone()
            for name, parameter in self._named(module)
        }
        if not self.shadow:
            raise BackendContractError("EMA selected no parameters")

    def _named(self, module: Any) -> list[tuple[str, Any]]:
        root = getattr(module, "module", module)
        return [
            (name, parameter)
            for name, parameter in root.named_parameters()
            if not self.trainable_only or parameter.requires_grad
        ]

    def _validate(self, module: Any) -> list[tuple[str, Any]]:
        named = self._named(module)
        if {name for name, _ in named} != set(self.shadow):
            raise BackendContractError("EMA parameter names differ from the live model")
        for name, parameter in named:
            shadow = self.shadow[name]
            if shadow.shape != parameter.shape or shadow.dtype != self.dtype:
                raise BackendContractError(f"EMA shard layout differs for {name}")
        return named

    def update(self, module: Any, *, decay: float | None = None) -> None:
        import torch

        value = self.decay if decay is None else float(decay)
        if not 0 <= value <= 1:
            raise BackendContractError("EMA update decay must be in [0,1]")
        with torch.no_grad():
            for name, parameter in self._validate(module):
                live = parameter.detach().to(device=self.device, dtype=self.dtype)
                self.shadow[name].mul_(value).add_(live, alpha=1 - value)
        self.num_updates += 1

    def state_dict(self) -> dict[str, Any]:
        return {name: value.clone() for name, value in self.shadow.items()}

    def load_state_dict(self, values: Mapping[str, Any], *, num_updates: int) -> None:
        if set(values) != set(self.shadow):
            raise BackendContractError("EMA checkpoint names differ from the live layout")
        restored = {}
        for name, current in self.shadow.items():
            value = values[name]
            if value.shape != current.shape or not value.is_floating_point():
                raise BackendContractError(f"EMA checkpoint layout differs for {name}")
            restored[name] = value.detach().to(device=self.device, dtype=self.dtype, copy=True)
        if int(num_updates) < 0:
            raise BackendContractError("EMA update count must be non-negative")
        self.shadow = restored
        self.num_updates = int(num_updates)

    @contextmanager
    def swapped_into(self, module: Any) -> Iterator[None]:
        import torch

        named = self._validate(module)
        live = {name: parameter.detach().clone() for name, parameter in named}
        try:
            with torch.no_grad():
                for name, parameter in named:
                    parameter.copy_(
                        self.shadow[name].to(device=parameter.device, dtype=parameter.dtype)
                    )
            yield
        finally:
            with torch.no_grad():
                for name, parameter in self._named(module):
                    parameter.copy_(live[name])