From 45a50d2dbaa35ed79f53afa10b0023ea3cd89a09 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 23 Oct 2023 19:58:25 -0400 Subject: [PATCH 1/2] Made some fixes to the container. --- tts/VALL-E-X/main.py | 39 ++++++++++++++++++++--------- tts/VALL-E-X/utils/prompt_making.py | 1 + 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tts/VALL-E-X/main.py b/tts/VALL-E-X/main.py index e42fec5..67998e2 100755 --- a/tts/VALL-E-X/main.py +++ b/tts/VALL-E-X/main.py @@ -150,22 +150,22 @@ voice_designer = VoiceDesigner() def main(args): - print(args.mode) + print("Starting Inference on Vall-E-X With Inputs") + print(f"Mode: {args.mode}") - print(args.text) - print(args.audio_wav_files) - print(args.whisper_folder_path) - print(args.vocos_folder_path) + print(f"Text: {args.text}") + print(f"List of Files For Create: {args.audio_wav_files}") + print(f"Whisper Folder Path: {args.whisper_folder_path}") + print(f"Vocos Folder Path: {args.vocos_folder_path}") + print(f"Vall-E-X Path: {args.vallex_path}") - print(args.vallex_path) + print(f"Prompt Path: {args.prompt_path}") + print(f"Prompt Name: {args.prompt_name}") - print(args.prompt_path) - print(args.prompt_name) + print(f"Audio Name: {args.audio_name}") + print(f"Audio Path: {args.audio_path}") - print(args.audio_name) - print(args.audio_path) - - print(args.tmp_work_dir) + print(f"Temp Work Dir: {args.tmp_work_dir}") voice_designer.temp_path = pathlib.Path(args.tmp_work_dir) @@ -174,6 +174,7 @@ def main(args): pathlib.Path(args.whisper_folder_path)) if args.mode == 0: # run inference + print("Running Inference") voice_designer.tts_with_prompt(prompt_dir=pathlib.Path(args.prompt_path), prompt_name=pathlib.Path(args.prompt_name), audio_output_path=pathlib.Path(args.audio_path), @@ -234,3 +235,17 @@ if __name__ == "__main__": # create voice example #/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --mode 1 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt" --tmp-work-dir "/tmp" + +#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python +#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py +# --mode 1 +# --audio-wav-files +# "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" +# "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" +# --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" +# --whisper-model medium +# --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" +# --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" +# --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" +# --prompt-name "test_prompt" +# --tmp-work-dir "/tmp" diff --git a/tts/VALL-E-X/utils/prompt_making.py b/tts/VALL-E-X/utils/prompt_making.py index d71a359..41f3848 100644 --- a/tts/VALL-E-X/utils/prompt_making.py +++ b/tts/VALL-E-X/utils/prompt_making.py @@ -84,6 +84,7 @@ def make_prompt(name:pathlib.Path, audio_prompt_path:pathlib.Path,audio_prompt_o else: save_path = os.path.join("./customs/", f"{name}.npz") np.savez(save_path, audio_tokens=audio_tokens, text_tokens=text_tokens, lang_code=lang2code[lang_pr]) + print(f"Embedding Save Path {save_path}") logging.info(f"Successful. Prompt saved to {save_path}") def make_transcript(name, wav, sr, whisper_folder_path: pathlib.Path,transcript=None): From a81c6f083c732ff28e594704b9034333158cdb1a Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 9 Nov 2023 15:59:56 -0500 Subject: [PATCH 2/2] Shows loRA menu now needs just update logic and merging logic --- animation/Rerender_A_Video/sd_model_cfg.py | 5 +++++ animation/Rerender_A_Video/webUI.py | 19 ++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/animation/Rerender_A_Video/sd_model_cfg.py b/animation/Rerender_A_Video/sd_model_cfg.py index 741bc7a..4fb7af0 100644 --- a/animation/Rerender_A_Video/sd_model_cfg.py +++ b/animation/Rerender_A_Video/sd_model_cfg.py @@ -7,3 +7,8 @@ model_dict = { 'CounterfeitV30_25': 'models/CounterfeitV30_25.safetensors', 'ghibli_style_offset': 'models/ghibli_style_offset.safetensors' } + +loRA_dict = { + 'None': '', + 'Dark Sushi': 'loRAs/dark_sushi' +} \ No newline at end of file diff --git a/animation/Rerender_A_Video/webUI.py b/animation/Rerender_A_Video/webUI.py index 3a69102..105daf0 100644 --- a/animation/Rerender_A_Video/webUI.py +++ b/animation/Rerender_A_Video/webUI.py @@ -24,7 +24,7 @@ from deps.ControlNet.annotator.util import HWC3 from deps.ControlNet.cldm.model import create_model, load_state_dict from deps.gmflow.gmflow.gmflow import GMFlow from flow.flow_utils import get_warped_and_mask -from sd_model_cfg import model_dict +from sd_model_cfg import model_dict,loRA_dict from src.config import RerenderConfig from src.controller import AttentionControl from src.ddim_v_hacked import DDIMVSampler @@ -206,6 +206,7 @@ def create_cfg(input_path, prompt, image_resolution, control_strength, interval=interval, crop=[left_crop, right_crop, top_crop, bottom_crop], sd_model=sd_model, + loRA_weights=loRA_weights, ddim_steps=ddim_steps, scale=scale, control_type=control_type, @@ -239,11 +240,13 @@ def cfg_to_input(filename): ] sd_model = inversed_model_dict.get(cfg.sd_model, 'Stable Diffusion 1.5') + loRA_weights = inversed_model_dict.get(cfg.loRA_weights,"None") + args = [ cfg.input_path, cfg.prompt, cfg.image_resolution, cfg.control_strength, cfg.color_preserve, *cfg.crop, cfg.control_type, cfg.canny_low, - cfg.canny_high, cfg.ddim_steps, cfg.scale, cfg.seed, sd_model, + cfg.canny_high, cfg.ddim_steps, cfg.scale, cfg.seed, sd_model,loRA_weights, cfg.a_prompt, cfg.n_prompt, cfg.interval, keyframe_count, cfg.x0_strength, use_constraints, *cfg.cross_period, cfg.style_update_freq, *cfg.warp_period, *cfg.mask_period, @@ -293,6 +296,9 @@ def process1(*args): global global_state global_state.update_sd_model(cfg.sd_model, cfg.control_type, cfg.freeu_args) + # TODO: support Koha loRA + # global_state.update_loRA_weights() + global_state.update_controller(cfg.inner_strength, cfg.mask_period, cfg.cross_period, cfg.ada_period, cfg.warp_period, cfg.loose_cfattn) @@ -754,6 +760,13 @@ with block: sd_model = gr.Dropdown(sd_model_list, label='Base model', value='Stable Diffusion 1.5') + + loRA_weights_list = list(loRA_dict.keys()) + loRA_weights = gr.Dropdown(loRA_weights_list, + label="loRA", + value="None") + + a_prompt = gr.Textbox(label='Added prompt', value='best quality, extremely detailed') n_prompt = gr.Textbox( @@ -915,7 +928,7 @@ with block: prompt, image_resolution, control_strength, color_preserve, left_crop, right_crop, top_crop, bottom_crop, control_type, low_threshold, high_threshold, ddim_steps, scale, seed, - sd_model, a_prompt, n_prompt, interval, keyframe_count, + sd_model,loRA_weights, a_prompt, n_prompt, interval, keyframe_count, x0_strength, use_constraints[0], cross_start, cross_end, style_update_freq, warp_start, warp_end, mask_start, mask_end, ada_start, ada_end, mask_strength,