From 7814c6a7e77d3c7938e30f950fcb9c46239e16d7 Mon Sep 17 00:00:00 2001 From: Justin John Date: Thu, 16 Nov 2023 20:34:05 +0530 Subject: [PATCH] Integrated Rerender_A_Video into animation directory --- ..._diffusers_to_original_stable_diffusion.py | 333 ------------------ .../convert_lora_safetensor_to_diffusers.py | 128 ------- ..._original_stable_diffusion_to_diffusers.py | 188 ---------- animation/Rerender_A_Video/sd_model_cfg.py | 45 +-- animation/Rerender_A_Video/webUI.py | 36 +- 5 files changed, 17 insertions(+), 713 deletions(-) delete mode 100644 animation/Rerender_A_Video/convert_merge_loras/convert_diffusers_to_original_stable_diffusion.py delete mode 100644 animation/Rerender_A_Video/convert_merge_loras/convert_lora_safetensor_to_diffusers.py delete mode 100644 animation/Rerender_A_Video/convert_merge_loras/convert_original_stable_diffusion_to_diffusers.py diff --git a/animation/Rerender_A_Video/convert_merge_loras/convert_diffusers_to_original_stable_diffusion.py b/animation/Rerender_A_Video/convert_merge_loras/convert_diffusers_to_original_stable_diffusion.py deleted file mode 100644 index 9da4521..0000000 --- a/animation/Rerender_A_Video/convert_merge_loras/convert_diffusers_to_original_stable_diffusion.py +++ /dev/null @@ -1,333 +0,0 @@ -# Script for converting a HF Diffusers saved pipeline to a Stable Diffusion checkpoint. -# *Only* converts the UNet, VAE, and Text Encoder. -# Does not convert optimizer state or any other thing. - -import argparse -import os.path as osp -import re - -import torch -from safetensors.torch import load_file, save_file - - -# =================# -# UNet Conversion # -# =================# - -unet_conversion_map = [ - # (stable-diffusion, HF Diffusers) - ("time_embed.0.weight", "time_embedding.linear_1.weight"), - ("time_embed.0.bias", "time_embedding.linear_1.bias"), - ("time_embed.2.weight", "time_embedding.linear_2.weight"), - ("time_embed.2.bias", "time_embedding.linear_2.bias"), - ("input_blocks.0.0.weight", "conv_in.weight"), - ("input_blocks.0.0.bias", "conv_in.bias"), - ("out.0.weight", "conv_norm_out.weight"), - ("out.0.bias", "conv_norm_out.bias"), - ("out.2.weight", "conv_out.weight"), - ("out.2.bias", "conv_out.bias"), -] - -unet_conversion_map_resnet = [ - # (stable-diffusion, HF Diffusers) - ("in_layers.0", "norm1"), - ("in_layers.2", "conv1"), - ("out_layers.0", "norm2"), - ("out_layers.3", "conv2"), - ("emb_layers.1", "time_emb_proj"), - ("skip_connection", "conv_shortcut"), -] - -unet_conversion_map_layer = [] -# hardcoded number of downblocks and resnets/attentions... -# would need smarter logic for other networks. -for i in range(4): - # loop over downblocks/upblocks - - for j in range(2): - # loop over resnets/attentions for downblocks - hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}." - sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0." - unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix)) - - if i < 3: - # no attention layers in down_blocks.3 - hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}." - sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1." - unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix)) - - for j in range(3): - # loop over resnets/attentions for upblocks - hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}." - sd_up_res_prefix = f"output_blocks.{3*i + j}.0." - unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix)) - - if i > 0: - # no attention layers in up_blocks.0 - hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}." - sd_up_atn_prefix = f"output_blocks.{3*i + j}.1." - unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix)) - - if i < 3: - # no downsample in down_blocks.3 - hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv." - sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op." - unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix)) - - # no upsample in up_blocks.3 - hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." - sd_upsample_prefix = f"output_blocks.{3*i + 2}.{1 if i == 0 else 2}." - unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix)) - -hf_mid_atn_prefix = "mid_block.attentions.0." -sd_mid_atn_prefix = "middle_block.1." -unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix)) - -for j in range(2): - hf_mid_res_prefix = f"mid_block.resnets.{j}." - sd_mid_res_prefix = f"middle_block.{2*j}." - unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix)) - - -def convert_unet_state_dict(unet_state_dict): - # buyer beware: this is a *brittle* function, - # and correct output requires that all of these pieces interact in - # the exact order in which I have arranged them. - mapping = {k: k for k in unet_state_dict.keys()} - for sd_name, hf_name in unet_conversion_map: - mapping[hf_name] = sd_name - for k, v in mapping.items(): - if "resnets" in k: - for sd_part, hf_part in unet_conversion_map_resnet: - v = v.replace(hf_part, sd_part) - mapping[k] = v - for k, v in mapping.items(): - for sd_part, hf_part in unet_conversion_map_layer: - v = v.replace(hf_part, sd_part) - mapping[k] = v - new_state_dict = {v: unet_state_dict[k] for k, v in mapping.items()} - return new_state_dict - - -# ================# -# VAE Conversion # -# ================# - -vae_conversion_map = [ - # (stable-diffusion, HF Diffusers) - ("nin_shortcut", "conv_shortcut"), - ("norm_out", "conv_norm_out"), - ("mid.attn_1.", "mid_block.attentions.0."), -] - -for i in range(4): - # down_blocks have two resnets - for j in range(2): - hf_down_prefix = f"encoder.down_blocks.{i}.resnets.{j}." - sd_down_prefix = f"encoder.down.{i}.block.{j}." - vae_conversion_map.append((sd_down_prefix, hf_down_prefix)) - - if i < 3: - hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0." - sd_downsample_prefix = f"down.{i}.downsample." - vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix)) - - hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." - sd_upsample_prefix = f"up.{3-i}.upsample." - vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix)) - - # up_blocks have three resnets - # also, up blocks in hf are numbered in reverse from sd - for j in range(3): - hf_up_prefix = f"decoder.up_blocks.{i}.resnets.{j}." - sd_up_prefix = f"decoder.up.{3-i}.block.{j}." - vae_conversion_map.append((sd_up_prefix, hf_up_prefix)) - -# this part accounts for mid blocks in both the encoder and the decoder -for i in range(2): - hf_mid_res_prefix = f"mid_block.resnets.{i}." - sd_mid_res_prefix = f"mid.block_{i+1}." - vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix)) - - -vae_conversion_map_attn = [ - # (stable-diffusion, HF Diffusers) - ("norm.", "group_norm."), - ("q.", "query."), - ("k.", "key."), - ("v.", "value."), - ("proj_out.", "proj_attn."), -] - - -def reshape_weight_for_sd(w): - # convert HF linear weights to SD conv2d weights - return w.reshape(*w.shape, 1, 1) - - -def convert_vae_state_dict(vae_state_dict): - mapping = {k: k for k in vae_state_dict.keys()} - for k, v in mapping.items(): - for sd_part, hf_part in vae_conversion_map: - v = v.replace(hf_part, sd_part) - mapping[k] = v - for k, v in mapping.items(): - if "attentions" in k: - for sd_part, hf_part in vae_conversion_map_attn: - v = v.replace(hf_part, sd_part) - mapping[k] = v - new_state_dict = {v: vae_state_dict[k] for k, v in mapping.items()} - weights_to_convert = ["q", "k", "v", "proj_out"] - for k, v in new_state_dict.items(): - for weight_name in weights_to_convert: - if f"mid.attn_1.{weight_name}.weight" in k: - print(f"Reshaping {k} for SD format") - new_state_dict[k] = reshape_weight_for_sd(v) - return new_state_dict - - -# =========================# -# Text Encoder Conversion # -# =========================# - - -textenc_conversion_lst = [ - # (stable-diffusion, HF Diffusers) - ("resblocks.", "text_model.encoder.layers."), - ("ln_1", "layer_norm1"), - ("ln_2", "layer_norm2"), - (".c_fc.", ".fc1."), - (".c_proj.", ".fc2."), - (".attn", ".self_attn"), - ("ln_final.", "transformer.text_model.final_layer_norm."), - ("token_embedding.weight", "transformer.text_model.embeddings.token_embedding.weight"), - ("positional_embedding", "transformer.text_model.embeddings.position_embedding.weight"), -] -protected = {re.escape(x[1]): x[0] for x in textenc_conversion_lst} -textenc_pattern = re.compile("|".join(protected.keys())) - -# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp -code2idx = {"q": 0, "k": 1, "v": 2} - - -def convert_text_enc_state_dict_v20(text_enc_dict): - new_state_dict = {} - capture_qkv_weight = {} - capture_qkv_bias = {} - for k, v in text_enc_dict.items(): - if ( - k.endswith(".self_attn.q_proj.weight") - or k.endswith(".self_attn.k_proj.weight") - or k.endswith(".self_attn.v_proj.weight") - ): - k_pre = k[: -len(".q_proj.weight")] - k_code = k[-len("q_proj.weight")] - if k_pre not in capture_qkv_weight: - capture_qkv_weight[k_pre] = [None, None, None] - capture_qkv_weight[k_pre][code2idx[k_code]] = v - continue - - if ( - k.endswith(".self_attn.q_proj.bias") - or k.endswith(".self_attn.k_proj.bias") - or k.endswith(".self_attn.v_proj.bias") - ): - k_pre = k[: -len(".q_proj.bias")] - k_code = k[-len("q_proj.bias")] - if k_pre not in capture_qkv_bias: - capture_qkv_bias[k_pre] = [None, None, None] - capture_qkv_bias[k_pre][code2idx[k_code]] = v - continue - - relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k) - new_state_dict[relabelled_key] = v - - for k_pre, tensors in capture_qkv_weight.items(): - if None in tensors: - raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing") - relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre) - new_state_dict[relabelled_key + ".in_proj_weight"] = torch.cat(tensors) - - for k_pre, tensors in capture_qkv_bias.items(): - if None in tensors: - raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing") - relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre) - new_state_dict[relabelled_key + ".in_proj_bias"] = torch.cat(tensors) - - return new_state_dict - - -def convert_text_enc_state_dict(text_enc_dict): - return text_enc_dict - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument("--model_path", default=None, type=str, required=True, help="Path to the model to convert.") - parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output model.") - parser.add_argument("--half", action="store_true", help="Save weights in half precision.") - parser.add_argument( - "--use_safetensors", action="store_true", help="Save weights use safetensors, default is ckpt." - ) - - args = parser.parse_args() - - assert args.model_path is not None, "Must provide a model path!" - - assert args.checkpoint_path is not None, "Must provide a checkpoint path!" - - # Path for safetensors - unet_path = osp.join(args.model_path, "unet", "diffusion_pytorch_model.safetensors") - vae_path = osp.join(args.model_path, "vae", "diffusion_pytorch_model.safetensors") - text_enc_path = osp.join(args.model_path, "text_encoder", "model.safetensors") - - # Load models from safetensors if it exists, if it doesn't pytorch - if osp.exists(unet_path): - unet_state_dict = load_file(unet_path, device="cpu") - else: - unet_path = osp.join(args.model_path, "unet", "diffusion_pytorch_model.bin") - unet_state_dict = torch.load(unet_path, map_location="cpu") - - if osp.exists(vae_path): - vae_state_dict = load_file(vae_path, device="cpu") - else: - vae_path = osp.join(args.model_path, "vae", "diffusion_pytorch_model.bin") - vae_state_dict = torch.load(vae_path, map_location="cpu") - - if osp.exists(text_enc_path): - text_enc_dict = load_file(text_enc_path, device="cpu") - else: - text_enc_path = osp.join(args.model_path, "text_encoder", "pytorch_model.bin") - text_enc_dict = torch.load(text_enc_path, map_location="cpu") - - # Convert the UNet model - unet_state_dict = convert_unet_state_dict(unet_state_dict) - unet_state_dict = {"model.diffusion_model." + k: v for k, v in unet_state_dict.items()} - - # Convert the VAE model - vae_state_dict = convert_vae_state_dict(vae_state_dict) - vae_state_dict = {"first_stage_model." + k: v for k, v in vae_state_dict.items()} - - # Easiest way to identify v2.0 model seems to be that the text encoder (OpenCLIP) is deeper - is_v20_model = "text_model.encoder.layers.22.layer_norm2.bias" in text_enc_dict - - if is_v20_model: - # Need to add the tag 'transformer' in advance so we can knock it out from the final layer-norm - text_enc_dict = {"transformer." + k: v for k, v in text_enc_dict.items()} - text_enc_dict = convert_text_enc_state_dict_v20(text_enc_dict) - text_enc_dict = {"cond_stage_model.model." + k: v for k, v in text_enc_dict.items()} - else: - text_enc_dict = convert_text_enc_state_dict(text_enc_dict) - text_enc_dict = {"cond_stage_model.transformer." + k: v for k, v in text_enc_dict.items()} - - # Put together new checkpoint - state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict} - if args.half: - state_dict = {k: v.half() for k, v in state_dict.items()} - - if args.use_safetensors: - save_file(state_dict, args.checkpoint_path) - else: - state_dict = {"state_dict": state_dict} - torch.save(state_dict, args.checkpoint_path) diff --git a/animation/Rerender_A_Video/convert_merge_loras/convert_lora_safetensor_to_diffusers.py b/animation/Rerender_A_Video/convert_merge_loras/convert_lora_safetensor_to_diffusers.py deleted file mode 100644 index f8e05d6..0000000 --- a/animation/Rerender_A_Video/convert_merge_loras/convert_lora_safetensor_to_diffusers.py +++ /dev/null @@ -1,128 +0,0 @@ -# coding=utf-8 -# Copyright 2023, Haofan Wang, Qixun Wang, All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" Conversion script for the LoRA's safetensors checkpoints. """ - -import argparse - -import torch -from safetensors.torch import load_file - -from diffusers import StableDiffusionPipeline - - -def convert(base_model_path, checkpoint_path, LORA_PREFIX_UNET, LORA_PREFIX_TEXT_ENCODER, alpha): - # load base model - pipeline = StableDiffusionPipeline.from_pretrained(base_model_path, torch_dtype=torch.float32) - - # load LoRA weight from .safetensors - state_dict = load_file(checkpoint_path) - - visited = [] - - # directly update weight in diffusers model - for key in state_dict: - # it is suggested to print out the key, it usually will be something like below - # "lora_te_text_model_encoder_layers_0_self_attn_k_proj.lora_down.weight" - - # as we have set the alpha beforehand, so just skip - if ".alpha" in key or key in visited: - continue - - if "text" in key: - layer_infos = key.split(".")[0].split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_") - curr_layer = pipeline.text_encoder - else: - layer_infos = key.split(".")[0].split(LORA_PREFIX_UNET + "_")[-1].split("_") - curr_layer = pipeline.unet - - # find the target layer - temp_name = layer_infos.pop(0) - while len(layer_infos) > -1: - try: - curr_layer = curr_layer.__getattr__(temp_name) - if len(layer_infos) > 0: - temp_name = layer_infos.pop(0) - elif len(layer_infos) == 0: - break - except Exception: - if len(temp_name) > 0: - temp_name += "_" + layer_infos.pop(0) - else: - temp_name = layer_infos.pop(0) - - pair_keys = [] - if "lora_down" in key: - pair_keys.append(key.replace("lora_down", "lora_up")) - pair_keys.append(key) - else: - pair_keys.append(key) - pair_keys.append(key.replace("lora_up", "lora_down")) - - # update weight - if len(state_dict[pair_keys[0]].shape) == 4: - weight_up = state_dict[pair_keys[0]].squeeze(3).squeeze(2).to(torch.float32) - weight_down = state_dict[pair_keys[1]].squeeze(3).squeeze(2).to(torch.float32) - curr_layer.weight.data += alpha * torch.mm(weight_up, weight_down).unsqueeze(2).unsqueeze(3) - else: - weight_up = state_dict[pair_keys[0]].to(torch.float32) - weight_down = state_dict[pair_keys[1]].to(torch.float32) - curr_layer.weight.data += alpha * torch.mm(weight_up, weight_down) - - # update visited list - for item in pair_keys: - visited.append(item) - - return pipeline - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "--base_model_path", default=None, type=str, required=True, help="Path to the base model in diffusers format." - ) - parser.add_argument( - "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." - ) - parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") - parser.add_argument( - "--lora_prefix_unet", default="lora_unet", type=str, help="The prefix of UNet weight in safetensors" - ) - parser.add_argument( - "--lora_prefix_text_encoder", - default="lora_te", - type=str, - help="The prefix of text encoder weight in safetensors", - ) - parser.add_argument("--alpha", default=0.75, type=float, help="The merging ratio in W = W0 + alpha * deltaW") - parser.add_argument( - "--to_safetensors", action="store_true", help="Whether to store pipeline in safetensors format or not." - ) - parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") - - args = parser.parse_args() - - base_model_path = args.base_model_path - checkpoint_path = args.checkpoint_path - dump_path = args.dump_path - lora_prefix_unet = args.lora_prefix_unet - lora_prefix_text_encoder = args.lora_prefix_text_encoder - alpha = args.alpha - - pipe = convert(base_model_path, checkpoint_path, lora_prefix_unet, lora_prefix_text_encoder, alpha) - - pipe = pipe.to(args.device) - pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/animation/Rerender_A_Video/convert_merge_loras/convert_original_stable_diffusion_to_diffusers.py b/animation/Rerender_A_Video/convert_merge_loras/convert_original_stable_diffusion_to_diffusers.py deleted file mode 100644 index 2ca7096..0000000 --- a/animation/Rerender_A_Video/convert_merge_loras/convert_original_stable_diffusion_to_diffusers.py +++ /dev/null @@ -1,188 +0,0 @@ -# coding=utf-8 -# Copyright 2023 The HuggingFace Inc. team. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" Conversion script for the LDM checkpoints. """ - -import argparse -import importlib - -import torch - -from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt - - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." - ) - # !wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml - parser.add_argument( - "--original_config_file", - default=None, - type=str, - help="The YAML config file corresponding to the original architecture.", - ) - parser.add_argument( - "--config_files", - default=None, - type=str, - help="The YAML config file corresponding to the architecture.", - ) - parser.add_argument( - "--num_in_channels", - default=None, - type=int, - help="The number of input channels. If `None` number of input channels will be automatically inferred.", - ) - parser.add_argument( - "--scheduler_type", - default="pndm", - type=str, - help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']", - ) - parser.add_argument( - "--pipeline_type", - default=None, - type=str, - help=( - "The pipeline type. One of 'FrozenOpenCLIPEmbedder', 'FrozenCLIPEmbedder', 'PaintByExample'" - ". If `None` pipeline will be automatically inferred." - ), - ) - parser.add_argument( - "--image_size", - default=None, - type=int, - help=( - "The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2" - " Base. Use 768 for Stable Diffusion v2." - ), - ) - parser.add_argument( - "--prediction_type", - default=None, - type=str, - help=( - "The prediction type that the model was trained on. Use 'epsilon' for Stable Diffusion v1.X and Stable" - " Diffusion v2 Base. Use 'v_prediction' for Stable Diffusion v2." - ), - ) - parser.add_argument( - "--extract_ema", - action="store_true", - help=( - "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" - " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" - " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." - ), - ) - parser.add_argument( - "--upcast_attention", - action="store_true", - help=( - "Whether the attention computation should always be upcasted. This is necessary when running stable" - " diffusion 2.1." - ), - ) - parser.add_argument( - "--from_safetensors", - action="store_true", - help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.", - ) - parser.add_argument( - "--to_safetensors", - action="store_true", - help="Whether to store pipeline in safetensors format or not.", - ) - parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") - parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") - parser.add_argument( - "--stable_unclip", - type=str, - default=None, - required=False, - help="Set if this is a stable unCLIP model. One of 'txt2img' or 'img2img'.", - ) - parser.add_argument( - "--stable_unclip_prior", - type=str, - default=None, - required=False, - help="Set if this is a stable unCLIP txt2img model. Selects which prior to use. If `--stable_unclip` is set to `txt2img`, the karlo prior (https://huggingface.co/kakaobrain/karlo-v1-alpha/tree/main/prior) is selected by default.", - ) - parser.add_argument( - "--clip_stats_path", - type=str, - help="Path to the clip stats file. Only required if the stable unclip model's config specifies `model.params.noise_aug_config.params.clip_stats_path`.", - required=False, - ) - parser.add_argument( - "--controlnet", action="store_true", default=None, help="Set flag if this is a controlnet checkpoint." - ) - parser.add_argument("--half", action="store_true", help="Save weights in half precision.") - parser.add_argument( - "--vae_path", - type=str, - default=None, - required=False, - help="Set to a path, hub id to an already converted vae to not convert it again.", - ) - parser.add_argument( - "--pipeline_class_name", - type=str, - default=None, - required=False, - help="Specify the pipeline class name", - ) - - args = parser.parse_args() - - if args.pipeline_class_name is not None: - library = importlib.import_module("diffusers") - class_obj = getattr(library, args.pipeline_class_name) - pipeline_class = class_obj - else: - pipeline_class = None - - pipe = download_from_original_stable_diffusion_ckpt( - checkpoint_path_or_dict=args.checkpoint_path, - original_config_file=args.original_config_file, - config_files=args.config_files, - image_size=args.image_size, - prediction_type=args.prediction_type, - model_type=args.pipeline_type, - extract_ema=args.extract_ema, - scheduler_type=args.scheduler_type, - num_in_channels=args.num_in_channels, - upcast_attention=args.upcast_attention, - from_safetensors=args.from_safetensors, - device=args.device, - stable_unclip=args.stable_unclip, - stable_unclip_prior=args.stable_unclip_prior, - clip_stats_path=args.clip_stats_path, - controlnet=args.controlnet, - vae_path=args.vae_path, - pipeline_class=pipeline_class, - ) - - if args.half: - pipe.to(torch_dtype=torch.float16) - - if args.controlnet: - # only save the controlnet model - pipe.controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) - else: - pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/animation/Rerender_A_Video/sd_model_cfg.py b/animation/Rerender_A_Video/sd_model_cfg.py index 7c38c20..359932c 100644 --- a/animation/Rerender_A_Video/sd_model_cfg.py +++ b/animation/Rerender_A_Video/sd_model_cfg.py @@ -1,42 +1,9 @@ # The model dict is used for webUI only -from typing import Dict -import unittest -import pathlib - -def path_to_dict(path: pathlib.Path) -> Dict[str, str]: - path = pathlib.Path(path) - result = {} - for file in path.iterdir(): - if file.is_file() and file.suffix == '.safetensors': - # Convert file name to camel case with underscores between words - name = '_'.join(word.capitalize() for word in file.stem.split('_')) - # Add entry to dictionary - result[name] = str(file.relative_to(path.parent)) - result['Stable Diffusion 1.5'] = '' - print("result: ", result) - return result - - -# model_dict = { -# 'Stable Diffusion 1.5': '', -# 'revAnimated_v11': 'models/revAnimated_v11.safetensors', -# 'realisticVisionV20_v20': 'models/realisticVisionV20_v20.safetensors', -# 'CounterfeitV30_25': 'models/CounterfeitV30_25.safetensors', -# 'ghibli_style_offset': 'models/ghibli_style_offset.safetensors' -# } -model_dict = path_to_dict("./models") - -loRA_dict = { - 'None': '', - 'Dark Sushi': 'loRAs/dark_sushi' +model_dict = { + 'Stable Diffusion 1.5': '', + 'revAnimated_v11': 'models/revAnimated_v11.safetensors', + 'realisticVisionV20_v20': 'models/realisticVisionV20_v20.safetensors', + 'CounterfeitV30_25': 'CounterfeitV30_25.safetensors', + 'ghibli_style_offset': 'ghibli_style_offset.safetensors', } - -class TestPathToDict(unittest.TestCase): - def test_path_to_dict(self): - # Create a temporary directory - result = path_to_dict(pathlib.Path("/home/tensor/code/storyteller/storyteller-ml/animation/Rerender_A_Video/models")) - print(result) - -if __name__ == '__main__': - unittest.main() \ No newline at end of file diff --git a/animation/Rerender_A_Video/webUI.py b/animation/Rerender_A_Video/webUI.py index 8171a7f..8331c82 100644 --- a/animation/Rerender_A_Video/webUI.py +++ b/animation/Rerender_A_Video/webUI.py @@ -24,7 +24,7 @@ from deps.ControlNet.annotator.util import HWC3 from deps.ControlNet.cldm.model import create_model, load_state_dict from deps.gmflow.gmflow.gmflow import GMFlow from flow.flow_utils import get_warped_and_mask -from sd_model_cfg import model_dict,loRA_dict +from sd_model_cfg import model_dict from src.config import RerenderConfig from src.controller import AttentionControl from src.ddim_v_hacked import DDIMVSampler @@ -32,7 +32,7 @@ from src.freeu import freeu_forward from src.img_util import find_flat_region, numpy2tensor from src.video_util import (frame_to_video, get_fps, get_frame_count, prepare_frames) -from pathlib import Path + inversed_model_dict = dict() for k, v in model_dict.items(): inversed_model_dict[v] = k @@ -41,7 +41,6 @@ to_tensor = T.PILToTensor() blur = T.GaussianBlur(kernel_size=(9, 9), sigma=(18, 18)) - class ProcessingState(Enum): NULL = 0 FIRST_IMG = 1 @@ -104,7 +103,7 @@ class GlobalState: model.load_state_dict( load_state_dict('./models/control_sd15_openpose.pth', location='cuda')) - model = model.cuda() + model = model.cuda() sd_model_path = model_dict[sd_model] if len(sd_model_path) > 0: model_ext = os.path.splitext(sd_model_path)[1] @@ -145,7 +144,7 @@ class GlobalState: return canny_detector(x, low_threshold, high_threshold) self.detector = apply_canny - + elif control_type == 'depth': midas = MidasDetector() @@ -162,7 +161,7 @@ class GlobalState: detected_map, _ = openpose(x) return detected_map - self.detector = apply_openpose + self.detector = apply_openpose global_state = GlobalState() @@ -173,7 +172,7 @@ video_frame_count = None def create_cfg(input_path, prompt, image_resolution, control_strength, color_preserve, left_crop, right_crop, top_crop, bottom_crop, control_type, low_threshold, high_threshold, ddim_steps, scale, - seed, sd_model,loRA_weights, a_prompt, n_prompt, interval, keyframe_count, + seed, sd_model, a_prompt, n_prompt, interval, keyframe_count, x0_strength, use_constraints, cross_start, cross_end, style_update_freq, warp_start, warp_end, mask_start, mask_end, ada_start, ada_end, mask_strength, inner_strength, @@ -207,7 +206,6 @@ def create_cfg(input_path, prompt, image_resolution, control_strength, interval=interval, crop=[left_crop, right_crop, top_crop, bottom_crop], sd_model=sd_model, - loRA_weights=loRA_weights, ddim_steps=ddim_steps, scale=scale, control_type=control_type, @@ -241,13 +239,11 @@ def cfg_to_input(filename): ] sd_model = inversed_model_dict.get(cfg.sd_model, 'Stable Diffusion 1.5') - loRA_weights = inversed_model_dict.get(cfg.loRA_weights,"None") - args = [ cfg.input_path, cfg.prompt, cfg.image_resolution, cfg.control_strength, cfg.color_preserve, *cfg.crop, cfg.control_type, cfg.canny_low, - cfg.canny_high, cfg.ddim_steps, cfg.scale, cfg.seed, sd_model,loRA_weights, + cfg.canny_high, cfg.ddim_steps, cfg.scale, cfg.seed, sd_model, cfg.a_prompt, cfg.n_prompt, cfg.interval, keyframe_count, cfg.x0_strength, use_constraints, *cfg.cross_period, cfg.style_update_freq, *cfg.warp_period, *cfg.mask_period, @@ -297,9 +293,6 @@ def process1(*args): global global_state global_state.update_sd_model(cfg.sd_model, cfg.control_type, cfg.freeu_args) - # TODO: support Koha loRA - # global_state.update_loRA_weights() - global_state.update_controller(cfg.inner_strength, cfg.mask_period, cfg.cross_period, cfg.ada_period, cfg.warp_period, cfg.loose_cfattn) @@ -572,9 +565,9 @@ def process2(*args): else: noise_rescale = torch.ones_like(mask) masks = [] - for i in range(cfg.ddim_steps): - if i <= cfg.ddim_steps * cfg.mask_period[ - 0] or i >= cfg.ddim_steps * cfg.mask_period[1]: + for j in range(cfg.ddim_steps): + if j <= cfg.ddim_steps * cfg.mask_period[ + 0] or j >= cfg.ddim_steps * cfg.mask_period[1]: masks += [None] else: masks += [mask * cfg.mask_strength] @@ -761,13 +754,6 @@ with block: sd_model = gr.Dropdown(sd_model_list, label='Base model', value='Stable Diffusion 1.5') - - loRA_weights_list = list(loRA_dict.keys()) - loRA_weights = gr.Dropdown(loRA_weights_list, - label="loRA", - value="None") - - a_prompt = gr.Textbox(label='Added prompt', value='best quality, extremely detailed') n_prompt = gr.Textbox( @@ -929,7 +915,7 @@ with block: prompt, image_resolution, control_strength, color_preserve, left_crop, right_crop, top_crop, bottom_crop, control_type, low_threshold, high_threshold, ddim_steps, scale, seed, - sd_model,loRA_weights, a_prompt, n_prompt, interval, keyframe_count, + sd_model, a_prompt, n_prompt, interval, keyframe_count, x0_strength, use_constraints[0], cross_start, cross_end, style_update_freq, warp_start, warp_end, mask_start, mask_end, ada_start, ada_end, mask_strength,