Idol: a single image to 3D Gaussians. Inference SDK and demo.
Python
import torch
import torch.nn.functional as F
from tensordict import TensorDict
IMAGENET_MEAN = (0.485, 0.456, 0.406)
IMAGENET_STD = (0.229, 0.224, 0.225)
DTYPES = {'fp32': torch.float32, 'bf16': torch.bfloat16, 'fp16': torch.float16}
_ACCESSIBLE_REPOS = set()
class EncoderAccessError(RuntimeError):
"""The Hugging Face account in use cannot download one of the gated image encoders."""
def require_encoder_access(repos):
"""Stop with an explanation, before anything large is downloaded, when the gated encoder repos are not readable.
Network problems are not treated as missing access: the encoders may already be in the local cache."""
import os
from huggingface_hub import auth_check, get_token
from huggingface_hub.errors import GatedRepoError, RepositoryNotFoundError
if os.environ.get('HF_HUB_OFFLINE', '0').lower() in ('1', 'true', 'yes'):
return
denied = []
for repo in repos:
if repo in _ACCESSIBLE_REPOS:
continue
try:
auth_check(repo)
_ACCESSIBLE_REPOS.add(repo)
except (GatedRepoError, RepositoryNotFoundError):
denied.append(repo)
except Exception:
pass
if not denied:
return
pages = '\n'.join(f' https://huggingface.co/{repo}' for repo in denied)
if get_token() is None:
reason = 'You are not logged in to Hugging Face, and these image encoders are gated:'
steps = ('1. Accept the terms on each page above with your Hugging Face account.\n'
'2. Log in: run `hf auth login`, or set the HF_TOKEN environment variable to a token of that account.')
else:
reason = 'Your Hugging Face account does not have access yet to these gated image encoders:'
steps = ('1. Accept the terms on each page above (approval can take a while).\n'
'2. Make sure the token in use (`hf auth whoami`) belongs to the account that accepted them.')
raise EncoderAccessError(f'Idol needs two image encoders that are downloaded from Hugging Face.\n{reason}\n'
f'{pages}\n{steps}')
class ImageEncoder:
def __init__(self, dinov3_repo: str, flux2_vae_repo: str,
dinov3_dtype: str = 'bf16', device: str = 'cuda'):
from diffusers.models import AutoencoderKLFlux2
from transformers import AutoModel
self.device = torch.device(device)
self.dinov3 = AutoModel.from_pretrained(dinov3_repo, torch_dtype=DTYPES[dinov3_dtype])
self.dinov3 = self.dinov3.eval().requires_grad_(False).to(self.device)
self.flux2_vae = AutoencoderKLFlux2.from_pretrained(flux2_vae_repo, subfolder='vae',
torch_dtype=torch.bfloat16)
self.flux2_vae = self.flux2_vae.eval().requires_grad_(False).to(self.device)
def _dinov3_tokens(self, images):
mean = torch.tensor(IMAGENET_MEAN, device=images.device).view(1, 3, 1, 1)
std = torch.tensor(IMAGENET_STD, device=images.device).view(1, 3, 1, 1)
images = (images.float() - mean) / std
dtype = next(self.dinov3.parameters()).dtype
tokens = self.dinov3(pixel_values=images.to(dtype)).last_hidden_state.float()
return F.layer_norm(tokens, tokens.shape[-1:])
def _flux2_tokens(self, images):
vae = self.flux2_vae
latents = vae.encode(images.to(torch.bfloat16) * 2 - 1).latent_dist.sample()
batch, channels, height, width = latents.shape
latents = latents.view(batch, channels, height // 2, 2, width // 2, 2).permute(0, 1, 3, 5, 2, 4)
latents = latents.reshape(batch, channels * 4, height // 2, width // 2)
mean = vae.bn.running_mean.view(1, -1, 1, 1).to(latents.device, latents.dtype)
std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps).to(latents.device, latents.dtype)
tokens = ((latents - mean) / std).float().flatten(2).transpose(1, 2).contiguous()
padding = torch.zeros(batch, 5, tokens.shape[2], dtype=tokens.dtype, device=tokens.device)
return torch.cat([padding, tokens], dim=1)
@torch.no_grad()
def __call__(self, images: torch.Tensor) -> TensorDict:
"""images: [B, 3, 1024, 1024] in [0, 1] -> TensorDict(feature1=DINOv3 tokens, feature2=FLUX.2 VAE tokens)."""
images = images.to(self.device)
feature1 = self._dinov3_tokens(images)
feature2 = self._flux2_tokens(images)
return TensorDict({'feature1': feature1, 'feature2': feature2}, batch_size=feature1.shape[0])