From 64a8f9d1b11c9f6427693f097f8b7eb8f5b4aec1 Mon Sep 17 00:00:00 2001 From: arrmansa <41120982+arrmansa@users.noreply.github.com> Date: Mon, 30 Dec 2024 04:14:50 +0530 Subject: [PATCH 1/5] Update img2imgalt.py Fix with documentation --- scripts/img2imgalt.py | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index 1e833fa89..109c4a2ab 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -11,6 +11,10 @@ from modules import processing, shared, sd_samplers, sd_samplers_common import torch import k_diffusion as K +# Debugging notes - the original method apply_model is being called for sd1.5 is in modules.sd_hijack_utils and is ldm.models.diffusion.ddpm.LatentDiffusion +# For sdxl - OpenAIWrapper will be called, which will call the underlying diffusion_model + + def find_noise_for_image(p, cond, uncond, cfg_scale, steps): x = p.init_latent @@ -30,7 +34,13 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): x_in = torch.cat([x] * 2) sigma_in = torch.cat([sigmas[i] * s_in] * 2) - cond_in = torch.cat([uncond, cond]) + + if shared.sd_model.is_sdxl: + cond_tensor = cond['crossattn'] + uncond_tensor = uncond['crossattn'] + cond_in = torch.cat([uncond_tensor, cond_tensor]) + else: + cond_in = torch.cat([uncond, cond]) image_conditioning = torch.cat([p.image_conditioning] * 2) cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]} @@ -38,7 +48,11 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]] t = dnw.sigma_to_t(sigma_in) - eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) + if shared.sd_model.is_sdxl: + eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + else: + eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) + denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2) denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale @@ -64,6 +78,13 @@ Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "origina # Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736 def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): + if shared.sd_model.is_sdxl: + cond_tensor = cond['crossattn'] + uncond_tensor = uncond['crossattn'] + cond_in = torch.cat([uncond_tensor, cond_tensor]) + else: + cond_in = torch.cat([uncond, cond]) + x = p.init_latent s_in = x.new_ones([x.shape[0]]) @@ -82,7 +103,14 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): x_in = torch.cat([x] * 2) sigma_in = torch.cat([sigmas[i - 1] * s_in] * 2) - cond_in = torch.cat([uncond, cond]) + + + if shared.sd_model.is_sdxl: + cond_tensor = cond['crossattn'] + uncond_tensor = uncond['crossattn'] + cond_in = torch.cat([uncond_tensor, cond_tensor]) + else: + cond_in = torch.cat([uncond, cond]) image_conditioning = torch.cat([p.image_conditioning] * 2) cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]} @@ -94,7 +122,12 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): else: t = dnw.sigma_to_t(sigma_in) - eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) + + if shared.sd_model.is_sdxl: + eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + else: + eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) + denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2) denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale From a63cf10650f52f60044cb9b72f7292166959cc62 Mon Sep 17 00:00:00 2001 From: arrmansa <41120982+arrmansa@users.noreply.github.com> Date: Mon, 30 Dec 2024 23:33:43 +0530 Subject: [PATCH 2/5] Update img2imgalt.py WIP --- scripts/img2imgalt.py | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index 109c4a2ab..fa0612aaa 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -49,7 +49,12 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): t = dnw.sigma_to_t(sigma_in) if shared.sd_model.is_sdxl: - eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + num_classes_hack = shared.sd_model.model.diffusion_model.num_classes + shared.sd_model.model.diffusion_model.num_classes = None + try: + eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + finally: + shared.sd_model.model.diffusion_model.num_classes = num_classes_hack else: eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) @@ -78,13 +83,6 @@ Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "origina # Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736 def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): - if shared.sd_model.is_sdxl: - cond_tensor = cond['crossattn'] - uncond_tensor = uncond['crossattn'] - cond_in = torch.cat([uncond_tensor, cond_tensor]) - else: - cond_in = torch.cat([uncond, cond]) - x = p.init_latent s_in = x.new_ones([x.shape[0]]) @@ -124,7 +122,12 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): if shared.sd_model.is_sdxl: - eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + num_classes_hack = shared.sd_model.model.diffusion_model.num_classes + shared.sd_model.model.diffusion_model.num_classes = None + try: + eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + finally: + shared.sd_model.model.diffusion_model.num_classes = num_classes_hack else: eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) @@ -211,9 +214,19 @@ class Script(scripts.Script): and self.cache.sigma_adjustment == sigma_adjustment same_everything = same_params and self.cache.latent.shape == lat.shape and np.abs(self.cache.latent-lat).sum() < 100 + rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p) + if same_everything: rec_noise = self.cache.noise else: + # This prevents a crash, because I don't know how to access the underlying .diffusion_model yet when controlnet is enabled. WIP + # modules.sd_unet -> we're good + # scripts.hook -> we're cooked + if "scripts.hook" in str(shared.sd_model.model.diffusion_model.forward.__module__): + print("turn off any controlnets, do 1 pass and then turn controlnet back on to cache noise") + p.steps = 1 + return sd_samplers.create_sampler(p.sampler_name, p.sd_model).sample_img2img(p, p.init_latent, rand_noise, conditioning, unconditional_conditioning, image_conditioning=p.image_conditioning) + shared.state.job_count += 1 cond = p.sd_model.get_learned_conditioning(p.batch_size * [original_prompt]) uncond = p.sd_model.get_learned_conditioning(p.batch_size * [original_negative_prompt]) @@ -223,8 +236,6 @@ class Script(scripts.Script): rec_noise = find_noise_for_image(p, cond, uncond, cfg, st) self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment) - rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p) - combined_noise = ((1 - randomness) * rec_noise + randomness * rand_noise) / ((randomness**2 + (1-randomness)**2) ** 0.5) sampler = sd_samplers.create_sampler(p.sampler_name, p.sd_model) From 6b6396f4a6c32099834ee1a28167aadc9160f8c2 Mon Sep 17 00:00:00 2001 From: arrmansa <41120982+arrmansa@users.noreply.github.com> Date: Wed, 1 Jan 2025 01:46:57 +0530 Subject: [PATCH 3/5] speedup, second order correction, intensity control --- scripts/img2imgalt.py | 90 +++++++++++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 33 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index fa0612aaa..01e46f98a 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -13,7 +13,7 @@ import k_diffusion as K # Debugging notes - the original method apply_model is being called for sd1.5 is in modules.sd_hijack_utils and is ldm.models.diffusion.ddpm.LatentDiffusion # For sdxl - OpenAIWrapper will be called, which will call the underlying diffusion_model - +# When controlnet is enabled, the underlying model is not available to use, therefore we skip def find_noise_for_image(p, cond, uncond, cfg_scale, steps): x = p.init_latent @@ -78,11 +78,11 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): return x / x.std() -Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment"]) +Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment", "second_order_correction", "noise_sigma_intensity"]) # Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736 -def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): +def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, correction_factor, sigma_intensity): x = p.init_latent s_in = x.new_ones([x.shape[0]]) @@ -98,11 +98,7 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): for i in trange(1, len(sigmas)): shared.state.sampling_step += 1 - - x_in = torch.cat([x] * 2) sigma_in = torch.cat([sigmas[i - 1] * s_in] * 2) - - if shared.sd_model.is_sdxl: cond_tensor = cond['crossattn'] uncond_tensor = uncond['crossattn'] @@ -113,13 +109,53 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): image_conditioning = torch.cat([p.image_conditioning] * 2) cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]} - c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]] - if i == 1: t = dnw.sigma_to_t(torch.cat([sigmas[i] * s_in] * 2)) + dt = (sigmas[i] - sigmas[i - 1]) / (2 * sigmas[i]) else: t = dnw.sigma_to_t(sigma_in) + dt = (sigmas[i] - sigmas[i - 1]) / sigmas[i - 1] + noise = noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip) + + if correction_factor > 0: + recalculated_noise = noise_from_model(x + noise, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip) + noise = recalculated_noise * correction_factor + noise * (1 - correction_factor) + + x += noise + + sd_samplers_common.store_latent(x) + + # This shouldn't be necessary, but solved some VRAM issues + #del x_in, sigma_in, cond_in, c_out, c_in, t + #del eps, denoised_uncond, denoised_cond, denoised, dt + + shared.state.nextjob() + + return x / (x.std()*(1 - sigma_intensity) + sigmas[-1]*sigma_intensity) + +def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip): + + if cfg_scale == 1: # Case where denoised_uncond should not be calculated - 50% speedup, also good for sdxl in experiments + x_in = x + sigma_in = sigma_in[1:2] + c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]] + cond_in = {"c_concat":[cond_in["c_concat"][0][1:2]], "c_crossattn": [cond_in["c_crossattn"][0][1:2]]} + if shared.sd_model.is_sdxl: + num_classes_hack = shared.sd_model.model.diffusion_model.num_classes + shared.sd_model.model.diffusion_model.num_classes = None + try: + eps = shared.sd_model.model(x_in * c_in, t[1:2], {"crossattn": cond_in["c_crossattn"][0]}) + finally: + shared.sd_model.model.diffusion_model.num_classes = num_classes_hack + else: + eps = shared.sd_model.apply_model(x_in * c_in, t[1:2], cond=cond_in) + + return -eps * c_out* dt + else : + x_in = torch.cat([x] * 2) + + c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]] if shared.sd_model.is_sdxl: num_classes_hack = shared.sd_model.model.diffusion_model.num_classes @@ -131,28 +167,11 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps): else: eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) - denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2) + denoised_uncond, denoised_cond = (eps * c_out).chunk(2) denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale - if i == 1: - d = (x - denoised) / (2 * sigmas[i]) - else: - d = (x - denoised) / sigmas[i - 1] - - dt = sigmas[i] - sigmas[i - 1] - x = x + d * dt - - sd_samplers_common.store_latent(x) - - # This shouldn't be necessary, but solved some VRAM issues - del x_in, sigma_in, cond_in, c_out, c_in, t, - del eps, denoised_uncond, denoised_cond, denoised, d, dt - - shared.state.nextjob() - - return x / sigmas[-1] - + return -denoised * dt class Script(scripts.Script): def __init__(self): @@ -183,6 +202,8 @@ class Script(scripts.Script): cfg = gr.Slider(label="Decode CFG scale", minimum=0.0, maximum=15.0, step=0.1, value=1.0, elem_id=self.elem_id("cfg")) randomness = gr.Slider(label="Randomness", minimum=0.0, maximum=1.0, step=0.01, value=0.0, elem_id=self.elem_id("randomness")) sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=False, elem_id=self.elem_id("sigma_adjustment")) + second_order_correction = gr.Slider(label="Correct noise by running model again", minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id=self.elem_id("second_order_correction")) + noise_sigma_intensity = gr.Slider(label="Weight scaling std vs sigma based", minimum=-1.0, maximum=2.0, step=0.01, value=0.5, elem_id=self.elem_id("noise_sigma_intensity")) return [ info, @@ -190,10 +211,11 @@ class Script(scripts.Script): override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, - cfg, randomness, sigma_adjustment, + cfg, randomness, sigma_adjustment, second_order_correction, + noise_sigma_intensity ] - def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment): + def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment, second_order_correction, noise_sigma_intensity): # Override if override_sampler: p.sampler_name = "Euler" @@ -211,7 +233,9 @@ class Script(scripts.Script): same_params = self.cache is not None and self.cache.cfg_scale == cfg and self.cache.steps == st \ and self.cache.original_prompt == original_prompt \ and self.cache.original_negative_prompt == original_negative_prompt \ - and self.cache.sigma_adjustment == sigma_adjustment + and self.cache.sigma_adjustment == sigma_adjustment \ + and self.cache.second_order_correction == second_order_correction \ + and self.cache.noise_sigma_intensity == noise_sigma_intensity same_everything = same_params and self.cache.latent.shape == lat.shape and np.abs(self.cache.latent-lat).sum() < 100 rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p) @@ -231,10 +255,10 @@ class Script(scripts.Script): cond = p.sd_model.get_learned_conditioning(p.batch_size * [original_prompt]) uncond = p.sd_model.get_learned_conditioning(p.batch_size * [original_negative_prompt]) if sigma_adjustment: - rec_noise = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st) + rec_noise = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st, second_order_correction, noise_sigma_intensity) else: rec_noise = find_noise_for_image(p, cond, uncond, cfg, st) - self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment) + self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment, second_order_correction, noise_sigma_intensity) combined_noise = ((1 - randomness) * rec_noise + randomness * rand_noise) / ((randomness**2 + (1-randomness)**2) ** 0.5) From 0b34349f47f45dbd9e297fae4c7670a52401e104 Mon Sep 17 00:00:00 2001 From: arrmansa <41120982+arrmansa@users.noreply.github.com> Date: Wed, 1 Jan 2025 12:22:17 +0530 Subject: [PATCH 4/5] Update img2imgalt.py --- scripts/img2imgalt.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index 01e46f98a..98050831d 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -16,7 +16,7 @@ import k_diffusion as K # When controlnet is enabled, the underlying model is not available to use, therefore we skip def find_noise_for_image(p, cond, uncond, cfg_scale, steps): - x = p.init_latent + x = p.init_latent.clone() s_in = x.new_ones([x.shape[0]]) if shared.sd_model.parameterization == "v": @@ -83,7 +83,7 @@ Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "origina # Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736 def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, correction_factor, sigma_intensity): - x = p.init_latent + x = p.init_latent.clone() s_in = x.new_ones([x.shape[0]]) if shared.sd_model.parameterization == "v": @@ -118,7 +118,7 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, cor noise = noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip) - if correction_factor > 0: + if correction_factor > 0: # runs model with previously calculated noise recalculated_noise = noise_from_model(x + noise, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip) noise = recalculated_noise * correction_factor + noise * (1 - correction_factor) @@ -132,6 +132,7 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, cor shared.state.nextjob() + # Chooses between std and sigmas[-1] return x / (x.std()*(1 - sigma_intensity) + sigmas[-1]*sigma_intensity) def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip): @@ -202,9 +203,10 @@ class Script(scripts.Script): cfg = gr.Slider(label="Decode CFG scale", minimum=0.0, maximum=15.0, step=0.1, value=1.0, elem_id=self.elem_id("cfg")) randomness = gr.Slider(label="Randomness", minimum=0.0, maximum=1.0, step=0.01, value=0.0, elem_id=self.elem_id("randomness")) sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=False, elem_id=self.elem_id("sigma_adjustment")) - second_order_correction = gr.Slider(label="Correct noise by running model again", minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id=self.elem_id("second_order_correction")) - noise_sigma_intensity = gr.Slider(label="Weight scaling std vs sigma based", minimum=-1.0, maximum=2.0, step=0.01, value=0.5, elem_id=self.elem_id("noise_sigma_intensity")) - + second_order_correction = gr.Slider(label="Correct noise by running model again", minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id=self.elem_id("second_order_correction"), + info="use 0 (disabled) for original script behaviour, 0.5 reccomended value. Runs the model again to recalculate noise and correct it by given factor. Higher adheres to original image more.") + noise_sigma_intensity = gr.Slider(label="Weight scaling std vs sigma based", minimum=-1.0, maximum=2.0, step=0.01, value=0.5, elem_id=self.elem_id("noise_sigma_intensity"), + info="use 1 for original script behaviour, 0.5 reccomended value. Decides whether to use fixed sigma value or dynamic standard deviation to scale noise. Lower gives softer images.") return [ info, override_sampler, From 84c4aaec0df753b91fe5cf476573b806f9244efe Mon Sep 17 00:00:00 2001 From: arrmansa <41120982+arrmansa@users.noreply.github.com> Date: Thu, 2 Jan 2025 01:15:12 +0530 Subject: [PATCH 5/5] Code cleanup (final commit unless I fix controlnet) optional vector for sdxl Better functions, better cache Tested everything --- scripts/img2imgalt.py | 117 +++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 69 deletions(-) diff --git a/scripts/img2imgalt.py b/scripts/img2imgalt.py index 98050831d..7af93bae9 100644 --- a/scripts/img2imgalt.py +++ b/scripts/img2imgalt.py @@ -15,7 +15,8 @@ import k_diffusion as K # For sdxl - OpenAIWrapper will be called, which will call the underlying diffusion_model # When controlnet is enabled, the underlying model is not available to use, therefore we skip -def find_noise_for_image(p, cond, uncond, cfg_scale, steps): +@torch.no_grad() +def find_noise_for_image(p, cond, uncond, cfg_scale, steps, skip_sdxl_vector): x = p.init_latent.clone() s_in = x.new_ones([x.shape[0]]) @@ -36,53 +37,27 @@ def find_noise_for_image(p, cond, uncond, cfg_scale, steps): sigma_in = torch.cat([sigmas[i] * s_in] * 2) if shared.sd_model.is_sdxl: - cond_tensor = cond['crossattn'] - uncond_tensor = uncond['crossattn'] - cond_in = torch.cat([uncond_tensor, cond_tensor]) + cond_in = {"crossattn": [torch.cat([uncond['crossattn'], cond['crossattn']])], "vector": [torch.cat([uncond['vector'], cond['vector']])]} else: - cond_in = torch.cat([uncond, cond]) + cond_in = {"c_concat": [torch.cat([p.image_conditioning] * 2)], "c_crossattn": [torch.cat([uncond, cond])]} - image_conditioning = torch.cat([p.image_conditioning] * 2) - cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]} - - c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]] t = dnw.sigma_to_t(sigma_in) - - if shared.sd_model.is_sdxl: - num_classes_hack = shared.sd_model.model.diffusion_model.num_classes - shared.sd_model.model.diffusion_model.num_classes = None - try: - eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) - finally: - shared.sd_model.model.diffusion_model.num_classes = num_classes_hack - else: - eps = shared.sd_model.apply_model(x_in * c_in, t, cond=cond_in) - - denoised_uncond, denoised_cond = (x_in + eps * c_out).chunk(2) - - denoised = denoised_uncond + (denoised_cond - denoised_uncond) * cfg_scale - - d = (x - denoised) / sigmas[i] dt = sigmas[i] - sigmas[i - 1] - - x = x + d * dt + x += noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector) sd_samplers_common.store_latent(x) # This shouldn't be necessary, but solved some VRAM issues - del x_in, sigma_in, cond_in, c_out, c_in, t, - del eps, denoised_uncond, denoised_cond, denoised, d, dt + del x_in, sigma_in, cond_in, t, dt shared.state.nextjob() - return x / x.std() - - -Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment", "second_order_correction", "noise_sigma_intensity"]) + return x, sigmas[-1] # Based on changes suggested by briansemrau in https://github.com/AUTOMATIC1111/stable-diffusion-webui/issues/736 -def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, correction_factor, sigma_intensity): +@torch.no_grad() +def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, correction_factor, skip_sdxl_vector): x = p.init_latent.clone() s_in = x.new_ones([x.shape[0]]) @@ -100,14 +75,9 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, cor shared.state.sampling_step += 1 sigma_in = torch.cat([sigmas[i - 1] * s_in] * 2) if shared.sd_model.is_sdxl: - cond_tensor = cond['crossattn'] - uncond_tensor = uncond['crossattn'] - cond_in = torch.cat([uncond_tensor, cond_tensor]) + cond_in = {"crossattn": [torch.cat([uncond['crossattn'], cond['crossattn']])], "vector": [torch.cat([uncond['vector'], cond['vector']])]} else: - cond_in = torch.cat([uncond, cond]) - - image_conditioning = torch.cat([p.image_conditioning] * 2) - cond_in = {"c_concat": [image_conditioning], "c_crossattn": [cond_in]} + cond_in = {"c_concat": [torch.cat([p.image_conditioning] * 2)], "c_crossattn": [torch.cat([uncond, cond])]} if i == 1: t = dnw.sigma_to_t(torch.cat([sigmas[i] * s_in] * 2)) @@ -116,37 +86,35 @@ def find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg_scale, steps, cor t = dnw.sigma_to_t(sigma_in) dt = (sigmas[i] - sigmas[i - 1]) / sigmas[i - 1] - noise = noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip) + noise = noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector) if correction_factor > 0: # runs model with previously calculated noise - recalculated_noise = noise_from_model(x + noise, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip) + recalculated_noise = noise_from_model(x + noise, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector) noise = recalculated_noise * correction_factor + noise * (1 - correction_factor) x += noise sd_samplers_common.store_latent(x) - # This shouldn't be necessary, but solved some VRAM issues - #del x_in, sigma_in, cond_in, c_out, c_in, t - #del eps, denoised_uncond, denoised_cond, denoised, dt - shared.state.nextjob() - # Chooses between std and sigmas[-1] - return x / (x.std()*(1 - sigma_intensity) + sigmas[-1]*sigma_intensity) + return x, sigmas[-1] -def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip): +@torch.no_grad() +def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip, skip_sdxl_vector): if cfg_scale == 1: # Case where denoised_uncond should not be calculated - 50% speedup, also good for sdxl in experiments x_in = x sigma_in = sigma_in[1:2] c_out, c_in = [K.utils.append_dims(k, x_in.ndim) for k in dnw.get_scalings(sigma_in)[skip:]] - cond_in = {"c_concat":[cond_in["c_concat"][0][1:2]], "c_crossattn": [cond_in["c_crossattn"][0][1:2]]} + cond_in = {k:[v[0][1:2]] for k, v in cond_in.items()} if shared.sd_model.is_sdxl: num_classes_hack = shared.sd_model.model.diffusion_model.num_classes - shared.sd_model.model.diffusion_model.num_classes = None + if skip_sdxl_vector: + shared.sd_model.model.diffusion_model.num_classes = None + cond_in["vector"][0] = None try: - eps = shared.sd_model.model(x_in * c_in, t[1:2], {"crossattn": cond_in["c_crossattn"][0]}) + eps = shared.sd_model.model(x_in * c_in, t[1:2], {"crossattn": cond_in["crossattn"][0], "vector": cond_in["vector"][0]}) finally: shared.sd_model.model.diffusion_model.num_classes = num_classes_hack else: @@ -160,9 +128,11 @@ def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip): if shared.sd_model.is_sdxl: num_classes_hack = shared.sd_model.model.diffusion_model.num_classes - shared.sd_model.model.diffusion_model.num_classes = None + if skip_sdxl_vector: + shared.sd_model.model.diffusion_model.num_classes = None + cond_in["vector"][0] = None try: - eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["c_crossattn"][0]} ) + eps = shared.sd_model.model(x_in * c_in, t, {"crossattn": cond_in["crossattn"][0], "vector": cond_in["vector"][0]} ) finally: shared.sd_model.model.diffusion_model.num_classes = num_classes_hack else: @@ -174,6 +144,9 @@ def noise_from_model(x, t, dt, sigma_in, cond_in, cfg_scale, dnw, skip): return -denoised * dt +Cached = namedtuple("Cached", ["noise", "cfg_scale", "steps", "latent", "original_prompt", "original_negative_prompt", "sigma_adjustment", "second_order_correction", "skip_sdxl_vector"]) + + class Script(scripts.Script): def __init__(self): self.cache = None @@ -189,24 +162,26 @@ class Script(scripts.Script): * `CFG Scale` should be 2 or lower. ''') - override_sampler = gr.Checkbox(label="Override `Sampling method` to Euler?(this method is built for it)", value=True, elem_id=self.elem_id("override_sampler")) + override_sampler = gr.Checkbox(label="Override `Sampling method` to Euler?(this method is built for it)", value=False, elem_id=self.elem_id("override_sampler")) - override_prompt = gr.Checkbox(label="Override `prompt` to the same value as `original prompt`?(and `negative prompt`)", value=True, elem_id=self.elem_id("override_prompt")) + override_prompt = gr.Checkbox(label="Override `prompt` to the same value as `original prompt`?(and `negative prompt`)", value=False, elem_id=self.elem_id("override_prompt")) original_prompt = gr.Textbox(label="Original prompt", lines=1, elem_id=self.elem_id("original_prompt")) original_negative_prompt = gr.Textbox(label="Original negative prompt", lines=1, elem_id=self.elem_id("original_negative_prompt")) override_steps = gr.Checkbox(label="Override `Sampling Steps` to the same value as `Decode steps`?", value=True, elem_id=self.elem_id("override_steps")) - st = gr.Slider(label="Decode steps", minimum=1, maximum=150, step=1, value=50, elem_id=self.elem_id("st")) + st = gr.Slider(label="Decode steps", minimum=1, maximum=150, step=1, value=20, elem_id=self.elem_id("st")) override_strength = gr.Checkbox(label="Override `Denoising strength` to 1?", value=True, elem_id=self.elem_id("override_strength")) cfg = gr.Slider(label="Decode CFG scale", minimum=0.0, maximum=15.0, step=0.1, value=1.0, elem_id=self.elem_id("cfg")) randomness = gr.Slider(label="Randomness", minimum=0.0, maximum=1.0, step=0.01, value=0.0, elem_id=self.elem_id("randomness")) - sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=False, elem_id=self.elem_id("sigma_adjustment")) + sigma_adjustment = gr.Checkbox(label="Sigma adjustment for finding noise for image", value=True, elem_id=self.elem_id("sigma_adjustment")) second_order_correction = gr.Slider(label="Correct noise by running model again", minimum=0.0, maximum=1.0, step=0.01, value=0.5, elem_id=self.elem_id("second_order_correction"), info="use 0 (disabled) for original script behaviour, 0.5 reccomended value. Runs the model again to recalculate noise and correct it by given factor. Higher adheres to original image more.") noise_sigma_intensity = gr.Slider(label="Weight scaling std vs sigma based", minimum=-1.0, maximum=2.0, step=0.01, value=0.5, elem_id=self.elem_id("noise_sigma_intensity"), info="use 1 for original script behaviour, 0.5 reccomended value. Decides whether to use fixed sigma value or dynamic standard deviation to scale noise. Lower gives softer images.") + skip_sdxl_vector = gr.Checkbox(label="Skip sdxl vectors", info="may cause distortion if false", value=True, elem_id=self.elem_id("skip_sdxl_vector")) + return [ info, override_sampler, @@ -214,10 +189,11 @@ class Script(scripts.Script): override_steps, st, override_strength, cfg, randomness, sigma_adjustment, second_order_correction, - noise_sigma_intensity + noise_sigma_intensity, skip_sdxl_vector ] - def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment, second_order_correction, noise_sigma_intensity): + @torch.no_grad() + def run(self, p, _, override_sampler, override_prompt, original_prompt, original_negative_prompt, override_steps, st, override_strength, cfg, randomness, sigma_adjustment, second_order_correction, noise_sigma_intensity, skip_sdxl_vector): # Override if override_sampler: p.sampler_name = "Euler" @@ -237,15 +213,16 @@ class Script(scripts.Script): and self.cache.original_negative_prompt == original_negative_prompt \ and self.cache.sigma_adjustment == sigma_adjustment \ and self.cache.second_order_correction == second_order_correction \ - and self.cache.noise_sigma_intensity == noise_sigma_intensity + and self.cache.skip_sdxl_vector == skip_sdxl_vector + same_everything = same_params and self.cache.latent.shape == lat.shape and np.abs(self.cache.latent-lat).sum() < 100 rand_noise = processing.create_random_tensors(p.init_latent.shape[1:], seeds=seeds, subseeds=subseeds, subseed_strength=p.subseed_strength, seed_resize_from_h=p.seed_resize_from_h, seed_resize_from_w=p.seed_resize_from_w, p=p) if same_everything: - rec_noise = self.cache.noise + rec_noise, sigma_val = self.cache.noise else: - # This prevents a crash, because I don't know how to access the underlying .diffusion_model yet when controlnet is enabled. WIP + # This prevents a crash, because I don't know how to access the underlying .diffusion_model yet when controlnet is enabled. # modules.sd_unet -> we're good # scripts.hook -> we're cooked if "scripts.hook" in str(shared.sd_model.model.diffusion_model.forward.__module__): @@ -257,21 +234,23 @@ class Script(scripts.Script): cond = p.sd_model.get_learned_conditioning(p.batch_size * [original_prompt]) uncond = p.sd_model.get_learned_conditioning(p.batch_size * [original_negative_prompt]) if sigma_adjustment: - rec_noise = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st, second_order_correction, noise_sigma_intensity) + rec_noise, sigma_val = find_noise_for_image_sigma_adjustment(p, cond, uncond, cfg, st, second_order_correction, skip_sdxl_vector) else: - rec_noise = find_noise_for_image(p, cond, uncond, cfg, st) - self.cache = Cached(rec_noise, cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment, second_order_correction, noise_sigma_intensity) + rec_noise, sigma_val = find_noise_for_image(p, cond, uncond, cfg, st, skip_sdxl_vector) + self.cache = Cached((rec_noise, sigma_val), cfg, st, lat, original_prompt, original_negative_prompt, sigma_adjustment, second_order_correction, skip_sdxl_vector) + + rec_noise = rec_noise / (rec_noise.std()*(1 - noise_sigma_intensity) + sigma_val*noise_sigma_intensity) combined_noise = ((1 - randomness) * rec_noise + randomness * rand_noise) / ((randomness**2 + (1-randomness)**2) ** 0.5) sampler = sd_samplers.create_sampler(p.sampler_name, p.sd_model) + p.seed = p.seed + 1 + sigmas = sampler.model_wrap.get_sigmas(p.steps) noise_dt = combined_noise - (p.init_latent / sigmas[0]) - p.seed = p.seed + 1 - return sampler.sample_img2img(p, p.init_latent, noise_dt, conditioning, unconditional_conditioning, image_conditioning=p.image_conditioning) p.sample = sample_extra