The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
SEEDS: update noise decomposition and refactor (#9633)
chaObserv32a627bf1fea
diff --git a/comfy/k_diffusion/sampling.py b/comfy/k_diffusion/sampling.pyindex fe6844b17..2d7e09838 100644--- a/comfy/k_diffusion/sampling.py+++ b/comfy/k_diffusion/sampling.py@@ -171,6 +171,16 @@ def offset_first_sigma_for_snr(sigmas, model_sampling, percent_offset=1e-4):return sigmas+def ei_h_phi_1(h: torch.Tensor) -> torch.Tensor:+ """Compute the result of h*phi_1(h) in exponential integrator methods."""+ return torch.expm1(h)+++def ei_h_phi_2(h: torch.Tensor) -> torch.Tensor:+ """Compute the result of h*phi_2(h) in exponential integrator methods."""+ return (torch.expm1(h) - h) / h++@torch.no_grad()def sample_euler(model, x, sigmas, extra_args=None, callback=None, disable=None, s_churn=0., s_tmin=0., s_tmax=float('inf'), s_noise=1.):"""Implements Algorithm 2 (Euler steps) from Karras et al. (2022)."""@@ -1550,13 +1560,12 @@ def sample_er_sde(model, x, sigmas, extra_args=None, callback=None, disable=None@torch.no_grad()def sample_seeds_2(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r=0.5):"""SEEDS-2 - Stochastic Explicit Exponential Derivative-free Solvers (VP Data Prediction) stage 2.- arXiv: https://arxiv.org/abs/2305.14267+ arXiv: https://arxiv.org/abs/2305.14267 (NeurIPS 2023)"""extra_args = {} if extra_args is None else extra_argsseed = extra_args.get("seed", None)noise_sampler = default_noise_sampler(x, seed=seed) if noise_sampler is None else noise_samplers_in = x.new_ones([x.shape[0]])-inject_noise = eta > 0 and s_noise > 0model_sampling = model.inner_model.model_patcher.get_model_object('model_sampling')@@ -1564,55 +1573,53 @@ def sample_seeds_2(model, x, sigmas, extra_args=None, callback=None, disable=Nonlambda_fn = partial(sigma_to_half_log_snr, model_sampling=model_sampling)sigmas = offset_first_sigma_for_snr(sigmas, model_sampling)+ fac = 1 / (2 * r)+for i in trange(len(sigmas) - 1, disable=disable):denoised = model(x, sigmas[i] * s_in, **extra_args)if callback is not None:callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})+if sigmas[i + 1] == 0:x = denoised- else:- lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])- h = lambda_t - lambda_s- h_eta = h * (eta + 1)- lambda_s_1 = lambda_s + r * h- fac = 1 / (2 * r)- sigma_s_1 = sigma_fn(lambda_s_1)-- # alpha_t = sigma_t * exp(log(alpha_t / sigma_t)) = sigma_t * exp(lambda_t)- alpha_s_1 = sigma_s_1 * lambda_s_1.exp()- alpha_t = sigmas[i + 1] * lambda_t.exp()-- coeff_1, coeff_2 = (-r * h_eta).expm1(), (-h_eta).expm1()- if inject_noise:- # 0 < r < 1- noise_coeff_1 = (-2 * r * h * eta).expm1().neg().sqrt()- noise_coeff_2 = (-r * h * eta).exp() * (-2 * (1 - r) * h * eta).expm1().neg().sqrt()- noise_1, noise_2 = noise_sampler(sigmas[i], sigma_s_1), noise_sampler(sigma_s_1, sigmas[i + 1])-- # Step 1- x_2 = sigma_s_1 / sigmas[i] * (-r * h * eta).exp() * x - alpha_s_1 * coeff_1 * denoised- if inject_noise:- x_2 = x_2 + sigma_s_1 * (noise_coeff_1 * noise_1) * s_noise- denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)-- # Step 2- denoised_d = (1 - fac) * denoised + fac * denoised_2- x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * coeff_2 * denoised_d- if inject_noise:- x = x + sigmas[i + 1] * (noise_coeff_2 * noise_1 + noise_coeff_1 * noise_2) * s_noise+ continue++ lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])+ h = lambda_t - lambda_s+ h_eta = h * (eta + 1)+ lambda_s_1 = torch.lerp(lambda_s, lambda_t, r)+ sigma_s_1 = sigma_fn(lambda_s_1)++ alpha_s_1 = sigma_s_1 * lambda_s_1.exp()+ alpha_t = sigmas[i + 1] * lambda_t.exp()++ # Step 1+ x_2 = sigma_s_1 / sigmas[i] * (-r * h * eta).exp() * x - alpha_s_1 * ei_h_phi_1(-r * h_eta) * denoised+ if inject_noise:+ sde_noise = (-2 * r * h * eta).expm1().neg().sqrt() * noise_sampler(sigmas[i], sigma_s_1)+ x_2 = x_2 + sde_noise * sigma_s_1 * s_noise+ denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)++ # Step 2+ denoised_d = torch.lerp(denoised, denoised_2, fac)+ x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * ei_h_phi_1(-h_eta) * denoised_d+ if inject_noise:+ segment_factor = (r - 1) * h * eta+ sde_noise = sde_noise * segment_factor.exp()+ sde_noise = sde_noise + segment_factor.mul(2).expm1().neg().sqrt() * noise_sampler(sigma_s_1, sigmas[i + 1])+ x = x + sde_noise * sigmas[i + 1] * s_noisereturn x@torch.no_grad()def sample_seeds_3(model, x, sigmas, extra_args=None, callback=None, disable=None, eta=1., s_noise=1., noise_sampler=None, r_1=1./3, r_2=2./3):"""SEEDS-3 - Stochastic Explicit Exponential Derivative-free Solvers (VP Data Prediction) stage 3.- arXiv: https://arxiv.org/abs/2305.14267+ arXiv: https://arxiv.org/abs/2305.14267 (NeurIPS 2023)"""extra_args = {} if extra_args is None else extra_argsseed = extra_args.get("seed", None)noise_sampler = default_noise_sampler(x, seed=seed) if noise_sampler is None else noise_samplers_in = x.new_ones([x.shape[0]])-inject_noise = eta > 0 and s_noise > 0model_sampling = model.inner_model.model_patcher.get_model_object('model_sampling')@@ -1624,45 +1631,49 @@ def sample_seeds_3(model, x, sigmas, extra_args=None, callback=None, disable=Nondenoised = model(x, sigmas[i] * s_in, **extra_args)if callback is not None:callback({'x': x, 'i': i, 'sigma': sigmas[i], 'sigma_hat': sigmas[i], 'denoised': denoised})+if sigmas[i + 1] == 0:x = denoised- else:- lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])- h = lambda_t - lambda_s- h_eta = h * (eta + 1)- lambda_s_1 = lambda_s + r_1 * h- lambda_s_2 = lambda_s + r_2 * h- sigma_s_1, sigma_s_2 = sigma_fn(lambda_s_1), sigma_fn(lambda_s_2)-- # alpha_t = sigma_t * exp(log(alpha_t / sigma_t)) = sigma_t * exp(lambda_t)- alpha_s_1 = sigma_s_1 * lambda_s_1.exp()- alpha_s_2 = sigma_s_2 * lambda_s_2.exp()- alpha_t = sigmas[i + 1] * lambda_t.exp()-- coeff_1, coeff_2, coeff_3 = (-r_1 * h_eta).expm1(), (-r_2 * h_eta).expm1(), (-h_eta).expm1()- if inject_noise:- # 0 < r_1 < r_2 < 1- noise_coeff_1 = (-2 * r_1 * h * eta).expm1().neg().sqrt()- noise_coeff_2 = (-r_1 * h * eta).exp() * (-2 * (r_2 - r_1) * h * eta).expm1().neg().sqrt()- noise_coeff_3 = (-r_2 * h * eta).exp() * (-2 * (1 - r_2) * h * eta).expm1().neg().sqrt()- noise_1, noise_2, noise_3 = noise_sampler(sigmas[i], sigma_s_1), noise_sampler(sigma_s_1, sigma_s_2), noise_sampler(sigma_s_2, sigmas[i + 1])-- # Step 1- x_2 = sigma_s_1 / sigmas[i] * (-r_1 * h * eta).exp() * x - alpha_s_1 * coeff_1 * denoised- if inject_noise:- x_2 = x_2 + sigma_s_1 * (noise_coeff_1 * noise_1) * s_noise- denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)-- # Step 2- x_3 = sigma_s_2 / sigmas[i] * (-r_2 * h * eta).exp() * x - alpha_s_2 * coeff_2 * denoised + (r_2 / r_1) * alpha_s_2 * (coeff_2 / (r_2 * h_eta) + 1) * (denoised_2 - denoised)- if inject_noise:- x_3 = x_3 + sigma_s_2 * (noise_coeff_2 * noise_1 + noise_coeff_1 * noise_2) * s_noise- denoised_3 = model(x_3, sigma_s_2 * s_in, **extra_args)-- # Step 3- x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * coeff_3 * denoised + (1. / r_2) * alpha_t * (coeff_3 / h_eta + 1) * (denoised_3 - denoised)- if inject_noise:- x = x + sigmas[i + 1] * (noise_coeff_3 * noise_1 + noise_coeff_2 * noise_2 + noise_coeff_1 * noise_3) * s_noise+ continue++ lambda_s, lambda_t = lambda_fn(sigmas[i]), lambda_fn(sigmas[i + 1])+ h = lambda_t - lambda_s+ h_eta = h * (eta + 1)+ lambda_s_1 = torch.lerp(lambda_s, lambda_t, r_1)+ lambda_s_2 = torch.lerp(lambda_s, lambda_t, r_2)+ sigma_s_1, sigma_s_2 = sigma_fn(lambda_s_1), sigma_fn(lambda_s_2)++ alpha_s_1 = sigma_s_1 * lambda_s_1.exp()+ alpha_s_2 = sigma_s_2 * lambda_s_2.exp()+ alpha_t = sigmas[i + 1] * lambda_t.exp()++ # Step 1+ x_2 = sigma_s_1 / sigmas[i] * (-r_1 * h * eta).exp() * x - alpha_s_1 * ei_h_phi_1(-r_1 * h_eta) * denoised+ if inject_noise:+ sde_noise = (-2 * r_1 * h * eta).expm1().neg().sqrt() * noise_sampler(sigmas[i], sigma_s_1)+ x_2 = x_2 + sde_noise * sigma_s_1 * s_noise+ denoised_2 = model(x_2, sigma_s_1 * s_in, **extra_args)++ # Step 2+ a3_2 = r_2 / r_1 * ei_h_phi_2(-r_2 * h_eta)+ a3_1 = ei_h_phi_1(-r_2 * h_eta) - a3_2+ x_3 = sigma_s_2 / sigmas[i] * (-r_2 * h * eta).exp() * x - alpha_s_2 * (a3_1 * denoised + a3_2 * denoised_2)+ if inject_noise:+ segment_factor = (r_1 - r_2) * h * eta+ sde_noise = sde_noise * segment_factor.exp()+ sde_noise = sde_noise + segment_factor.mul(2).expm1().neg().sqrt() * noise_sampler(sigma_s_1, sigma_s_2)+ x_3 = x_3 + sde_noise * sigma_s_2 * s_noise+ denoised_3 = model(x_3, sigma_s_2 * s_in, **extra_args)++ # Step 3+ b3 = ei_h_phi_2(-h_eta) / r_2+ b1 = ei_h_phi_1(-h_eta) - b3+ x = sigmas[i + 1] / sigmas[i] * (-h * eta).exp() * x - alpha_t * (b1 * denoised + b3 * denoised_3)+ if inject_noise:+ segment_factor = (r_2 - 1) * h * eta+ sde_noise = sde_noise * segment_factor.exp()+ sde_noise = sde_noise + segment_factor.mul(2).expm1().neg().sqrt() * noise_sampler(sigma_s_2, sigmas[i + 1])+ x = x + sde_noise * sigmas[i + 1] * s_noisereturn x