SUPRA-Tex: 3D Generation with Super-Aligned Texture
import torch, torchaudio
from typing import Optional
from tqdm import tqdm
from ..core import ModelConfig
from ..core.device.npu_compatible_device import get_device_type
from ..diffusion import FlowMatchScheduler
from ..diffusion.base_pipeline import BasePipeline, PipelineUnit
from ..models.ace_step_conditioner import AceStepConditionEncoder
from ..models.ace_step_text_encoder import AceStepTextEncoder
from ..models.ace_step_vae import AceStepVAE
from ..models.demucs import HTDemucs
from ..models.diffsynth_music_dit import DiffSynthMusicDiTModel
from transformers import AutoTokenizer
class DiffSynthMusicPipeline(BasePipeline):
def __init__(self, device=get_device_type(), torch_dtype=torch.bfloat16):
super().__init__(device=device, torch_dtype=torch_dtype)
self.scheduler = FlowMatchScheduler("FLUX.2")
self.text_encoder: AceStepTextEncoder = None
self.conditioner: AceStepConditionEncoder = None
self.dit: DiffSynthMusicDiTModel = None
self.vae: AceStepVAE = None
self.track_separator: HTDemucs = None
self.tokenizer: AutoTokenizer = None
self.in_iteration_models = ("dit",)
self.units = [
DiffSynthMusic_PromptEmbedder(),
DiffSynthMusic_ConditionEmbedder(),
DiffSynthMusic_NoiseInitializer(),
DiffSynthMusic_InputAudioEmbedder(),
]
self.model_fn = model_fn_ace_step
self.compilable_models = ["dit"]
self.default_negative_prompt = (
"A barren, low-fidelity audio mess with a strong sense of technical failure. "
"The track kicks off with an awkward silence where the backing instrumentation is completely absent, "
"leaving only a hollow void. A layer of harsh, static-like noise and environmental hiss dominates the soundscape, "
"drowning out any potential clarity. A distorted male vocal enters with severe clipping and digital artifacts, "
"cracking painfully on every syllable. The singing is consistently off-key and wandering, lacking any rhythmic anchor "
"or melodic coherence. The overall mood is grating, amateurish, and unintentionally comical."
)
@staticmethod
def from_pretrained(
torch_dtype: torch.dtype = torch.bfloat16,
device: str = get_device_type(),
model_configs: list[ModelConfig] = [],
tokenizer_config: ModelConfig = ModelConfig(model_id="DiffSynth-Studio/DiffSynth-Music-Tools", origin_file_pattern="text_encoder/"),
vram_limit: float = None,
):
pipe = DiffSynthMusicPipeline(device=device, torch_dtype=torch_dtype)
model_pool = pipe.download_and_load_models(model_configs, vram_limit)
pipe.text_encoder = model_pool.fetch_model("ace_step_text_encoder")
pipe.conditioner = model_pool.fetch_model("ace_step_conditioner")
pipe.dit = model_pool.fetch_model("diffsynth_music_dit")
pipe.vae = model_pool.fetch_model("ace_step_vae")
if pipe.vae is not None: pipe.vae.remove_weight_norm()
pipe.track_separator = model_pool.fetch_model("demucs")
if tokenizer_config is not None:
tokenizer_config.download_if_necessary()
pipe.tokenizer = AutoTokenizer.from_pretrained(tokenizer_config.path)
pipe.vram_management_enabled = pipe.check_vram_management_state()
return pipe
@torch.no_grad()
def extract_track(self, audio, sample_rate=48000, track="vocals"):
self.load_models_to_device(["track_separator"])
audio = audio.to(dtype=next(iter(self.track_separator.parameters())).dtype, device=self.device)
output = self.track_separator.extract_track(audio, sample_rate, track)
output = torchaudio.functional.resample(output, 44100, sample_rate)
return output
@torch.no_grad()
def fuse_track(self, audio, target_audio, target_track):
if not isinstance(target_track, list):
target_track = [target_track]
kept_track = [i for i in self.track_separator.sources if i not in target_track]
target_audio = self.extract_track(target_audio, track=target_track)
audio = self.extract_track(audio, track=kept_track)
audio, target_audio = self._balance_volumes(audio, target_audio)
if audio.shape[1] > target_audio.shape[1]:
audio[:, :target_audio.shape[1]] += target_audio
else:
audio += target_audio[:, :audio.shape[1]]
return audio
@staticmethod
def _balance_volumes(audio_a, audio_b, silence_threshold=1e-4):
rms_a = audio_a.pow(2).mean().sqrt().item()
rms_b = audio_b.pow(2).mean().sqrt().item()
a_silent = rms_a < silence_threshold
b_silent = rms_b < silence_threshold
if a_silent or b_silent:
return audio_a, audio_b
target_rms = min(rms_a, rms_b)
scale_a = target_rms / rms_a
scale_b = target_rms / rms_b
return audio_a * scale_a, audio_b * scale_b
@torch.no_grad()
def __call__(
self,
# Prompt
prompt: str = "",
negative_prompt: str = "",
lyrics: str = "",
vocal_language: str = None,
cfg_scale: float = 1.0,
# Metadata
bpm: float = None,
timesignature: str = None,
keyscale: str = None,
# Input Audio
input_audio: Optional[torch.Tensor] = None,
denoising_strength: float = 1.0,
# Duration
duration: int = 160,
# Rand
seed: int = None,
rand_device: str = "cpu",
# Tiled VAE
tiled: bool = False,
tile_size: int = 512,
tile_stride: int = 256,
# KV Cache
kv_cache = None,
negative_kv_cache = None,
# Force Control
target_audio = None,
target_track = "vocals",
# Steps
num_inference_steps: int = 50,
progress_bar_cmd=tqdm,
):
# Scheduler
self.scheduler.set_timesteps(num_inference_steps=num_inference_steps, denoising_strength=denoising_strength, dynamic_shift_len=4096)
# Parameters
inputs_posi = {"prompt": prompt, "lyrics": lyrics, "kv_cache": kv_cache}
inputs_nega = {"prompt": negative_prompt, "lyrics": "", "kv_cache": negative_kv_cache}
inputs_shared = {
"cfg_scale": cfg_scale,
"bpm": bpm, "timesignature": timesignature, "keyscale": keyscale, "vocal_language": vocal_language,
"input_audio": input_audio,
"duration": duration,
"seed": seed,
"rand_device": rand_device,
}
for unit in self.units:
inputs_shared, inputs_posi, inputs_nega = self.unit_runner(unit, self, inputs_shared, inputs_posi, inputs_nega)
# Denoise
self.load_models_to_device(self.in_iteration_models)
models = {name: getattr(self, name) for name in self.in_iteration_models}
for progress_id, timestep in enumerate(progress_bar_cmd(self.scheduler.timesteps)):
timestep = timestep.unsqueeze(0).to(dtype=self.torch_dtype, device=self.device)
noise_pred = self.cfg_guided_model_fn(
self.model_fn, cfg_scale,
inputs_shared, inputs_posi, inputs_nega,
**models, timestep=timestep, progress_id=progress_id,
)
inputs_shared["latents"] = self.step(self.scheduler, progress_id=progress_id, noise_pred=noise_pred, **inputs_shared)
# Decode
self.load_models_to_device(['vae'])
audio = self.vae_output_to_audio(inputs_shared["latents"], tiled, tile_size, tile_stride)
self.load_models_to_device([])
if target_audio is not None:
audio = self.fuse_track(audio, target_audio, target_track)
return audio
def vae_output_to_audio(self, vae_output, tiled=False, tile_size=512, tile_stride=256):
audio = self.vae.decode(vae_output.transpose(1, 2), tiled=tiled, tile_size=tile_size, tile_stride=tile_stride)
peak = torch.max(torch.abs(audio))
if peak < 1e-6: return audio
audio = audio * (10 ** (-1 / 20.0) / peak)
audio = self.output_audio_format_check(audio)
return audio
class DiffSynthMusic_PromptEmbedder(PipelineUnit):
LYRIC_PROMPT = "# Languages\n{}\n\n# Lyric\n{}<|endoftext|>"
def __init__(self):
super().__init__(
seperate_cfg=True,
input_params_posi={"prompt": "prompt", "lyrics": "lyrics"},
input_params_nega={"prompt": "prompt", "lyrics": "lyrics"},
input_params=("bpm", "timesignature", "keyscale", "duration", "vocal_language"),
output_params=("text_hidden_states", "text_attention_mask", "lyric_hidden_states", "lyric_attention_mask"),
onload_model_names=("text_encoder",)
)
def _encode_text(self, pipe: DiffSynthMusicPipeline, text, max_length=256):
text_inputs = pipe.tokenizer(
text,
max_length=max_length,
truncation=True,
return_tensors="pt",
)
input_ids = text_inputs.input_ids.to(pipe.device)
attention_mask = text_inputs.attention_mask.bool().to(pipe.device)
hidden_states = pipe.text_encoder(input_ids, attention_mask)
return hidden_states, attention_mask
def _encode_lyrics(self, pipe: DiffSynthMusicPipeline, lyric_text, max_length=2048):
text_inputs = pipe.tokenizer(
lyric_text,
max_length=max_length,
truncation=True,
return_tensors="pt",
)
input_ids = text_inputs.input_ids.to(pipe.device)
attention_mask = text_inputs.attention_mask.bool().to(pipe.device)
hidden_states = pipe.text_encoder.model.embed_tokens(input_ids)
return hidden_states, attention_mask
def process(self, pipe: DiffSynthMusicPipeline, prompt, lyrics, bpm, timesignature, keyscale, duration, vocal_language):
pipe.load_models_to_device(['text_encoder'])
if bpm is None: bpm = 100
if timesignature is None: timesignature = "4"
if keyscale is None: keyscale = "B minor"
if duration is None: duration = "300"
prompt = f"""
# Instruction
Fill the audio semantic mask based on the given conditions:
# Caption
{prompt}
# Metas
- bpm: {bpm}
- timesignature: {timesignature}
- keyscale: {keyscale}
- duration: {duration} seconds
<|endoftext|>
""".strip()
text_hidden_states, text_attention_mask = self._encode_text(pipe, prompt, max_length=256)
if vocal_language is None: vocal_language = "N/A"
lyric_text = self.LYRIC_PROMPT.format(vocal_language, lyrics)
lyric_hidden_states, lyric_attention_mask = self._encode_lyrics(pipe, lyric_text, max_length=2048)
return {
"text_hidden_states": text_hidden_states,
"text_attention_mask": text_attention_mask,
"lyric_hidden_states": lyric_hidden_states,
"lyric_attention_mask": lyric_attention_mask,
}
class DiffSynthMusic_ConditionEmbedder(PipelineUnit):
def __init__(self):
super().__init__(
seperate_cfg=True,
input_params_posi={
"text_hidden_states": "text_hidden_states",
"text_attention_mask": "text_attention_mask",
"lyric_hidden_states": "lyric_hidden_states",
"lyric_attention_mask": "lyric_attention_mask"
},
input_params_nega={
"text_hidden_states": "text_hidden_states",
"text_attention_mask": "text_attention_mask",
"lyric_hidden_states": "lyric_hidden_states",
"lyric_attention_mask": "lyric_attention_mask"
},
output_params=("encoder_hidden_states",),
onload_model_names=("conditioner",),
)
def process(self, pipe: DiffSynthMusicPipeline, text_hidden_states, text_attention_mask, lyric_hidden_states, lyric_attention_mask):
pipe.load_models_to_device(["conditioner"])
encoder_hidden_states, _ = pipe.conditioner(
text_hidden_states=text_hidden_states,
text_attention_mask=text_attention_mask,
lyric_hidden_states=lyric_hidden_states,
lyric_attention_mask=lyric_attention_mask,
reference_latents=None,
refer_audio_order_mask=None,
)
return {"encoder_hidden_states": encoder_hidden_states}
class DiffSynthMusic_NoiseInitializer(PipelineUnit):
def __init__(self):
super().__init__(
input_params=("duration", "seed", "rand_device"),
output_params=("noise",),
)
def process(self, pipe: DiffSynthMusicPipeline, duration, seed, rand_device):
noise = pipe.generate_noise((1, int(round(duration * 48000 / 1920 / 2)) * 2, 64), seed=seed, rand_device=rand_device, rand_torch_dtype=pipe.torch_dtype)
return {"noise": noise}
class DiffSynthMusic_InputAudioEmbedder(PipelineUnit):
def __init__(self):
super().__init__(
input_params=("noise", "input_audio"),
output_params=("latents", "input_latents"),
onload_model_names=("vae",),
)
def encode_audio(self, pipe: DiffSynthMusicPipeline, audio):
if audio.shape[-1] == 64:
return audio
else:
audio = torch.clamp(audio, -1.0, 1.0).unsqueeze(0)
audio = pipe.vae.encode(audio.to(dtype=pipe.torch_dtype, device=pipe.device)).transpose(1, 2)
return audio
def process(self, pipe: DiffSynthMusicPipeline, noise, input_audio):
if input_audio is None:
return {"latents": noise}
pipe.load_models_to_device(self.onload_model_names)
input_latents = self.encode_audio(pipe, input_audio)
# prevent potential size mismatch between context_latents and input_latents by cropping input_latents to the same temporal length as noise
input_latents = input_latents[:, :noise.shape[1]]
if input_latents.shape[1] < noise.shape[1]:
input_latents = torch.concat([input_latents, torch.zeros_like(noise)[:, :noise.shape[1] - input_latents.shape[1]]], dim=1)
if pipe.scheduler.training:
return {"input_latents": input_latents, "latents": noise}
else:
latents = pipe.scheduler.add_noise(input_latents, noise, timestep=pipe.scheduler.timesteps[0])
return {"latents": latents}
def model_fn_ace_step(
dit: DiffSynthMusicDiTModel,
latents=None,
timestep=None,
encoder_hidden_states=None,
kv_cache=None,
use_gradient_checkpointing=False,
use_gradient_checkpointing_offload=False,
**kwargs,
):
decoder_outputs = dit(
x=latents,
timestep=timestep,
y=encoder_hidden_states,
kv_cache=kv_cache,
use_gradient_checkpointing=use_gradient_checkpointing,
use_gradient_checkpointing_offload=use_gradient_checkpointing_offload,
)
return decoder_outputs