Merge branch 'master' of https://github.com/storytold/storyteller-ml

animation/Rerender_A_Video/sd_model_cfg.py

@@ -7,3 +7,8 @@ model_dict = {
     'CounterfeitV30_25': 'models/CounterfeitV30_25.safetensors',
     'ghibli_style_offset': 'models/ghibli_style_offset.safetensors'
 }
+
+loRA_dict = {
+    'None': '',
+    'Dark Sushi': 'loRAs/dark_sushi'
+}

animation/Rerender_A_Video/webUI.py

@@ -24,7 +24,7 @@ from deps.ControlNet.annotator.util import HWC3
 from deps.ControlNet.cldm.model import create_model, load_state_dict
 from deps.gmflow.gmflow.gmflow import GMFlow
 from flow.flow_utils import get_warped_and_mask
-from sd_model_cfg import model_dict
+from sd_model_cfg import model_dict,loRA_dict
 from src.config import RerenderConfig
 from src.controller import AttentionControl
 from src.ddim_v_hacked import DDIMVSampler
@@ -206,6 +206,7 @@ def create_cfg(input_path, prompt, image_resolution, control_strength,
         interval=interval,
         crop=[left_crop, right_crop, top_crop, bottom_crop],
         sd_model=sd_model,
+        loRA_weights=loRA_weights,        
         ddim_steps=ddim_steps,
         scale=scale,
         control_type=control_type,
@@ -239,11 +240,13 @@ def cfg_to_input(filename):
     ]
 
     sd_model = inversed_model_dict.get(cfg.sd_model, 'Stable Diffusion 1.5')
+    loRA_weights = inversed_model_dict.get(cfg.loRA_weights,"None")
+      
 
     args = [
         cfg.input_path, cfg.prompt, cfg.image_resolution, cfg.control_strength,
         cfg.color_preserve, *cfg.crop, cfg.control_type, cfg.canny_low,
-        cfg.canny_high, cfg.ddim_steps, cfg.scale, cfg.seed, sd_model,
+        cfg.canny_high, cfg.ddim_steps, cfg.scale, cfg.seed, sd_model,loRA_weights,
         cfg.a_prompt, cfg.n_prompt, cfg.interval, keyframe_count,
         cfg.x0_strength, use_constraints, *cfg.cross_period,
         cfg.style_update_freq, *cfg.warp_period, *cfg.mask_period,
@@ -293,6 +296,9 @@ def process1(*args):
     global global_state
     global_state.update_sd_model(cfg.sd_model, cfg.control_type,
                                  cfg.freeu_args)
+    # TODO: support Koha loRA
+    # global_state.update_loRA_weights()
+    
     global_state.update_controller(cfg.inner_strength, cfg.mask_period,
                                    cfg.cross_period, cfg.ada_period,
                                    cfg.warp_period, cfg.loose_cfattn)
@@ -754,6 +760,13 @@ with block:
                 sd_model = gr.Dropdown(sd_model_list,
                                        label='Base model',
                                        value='Stable Diffusion 1.5')
+                
+                loRA_weights_list = list(loRA_dict.keys())
+                loRA_weights = gr.Dropdown(loRA_weights_list,
+                                           label="loRA",
+                                           value="None")
+                
+                
                 a_prompt = gr.Textbox(label='Added prompt',
                                       value='best quality, extremely detailed')
                 n_prompt = gr.Textbox(
@@ -915,7 +928,7 @@ with block:
                     prompt, image_resolution, control_strength, color_preserve,
                     left_crop, right_crop, top_crop, bottom_crop, control_type,
                     low_threshold, high_threshold, ddim_steps, scale, seed,
-                    sd_model, a_prompt, n_prompt, interval, keyframe_count,
+                    sd_model,loRA_weights, a_prompt, n_prompt, interval, keyframe_count,
                     x0_strength, use_constraints[0], cross_start, cross_end,
                     style_update_freq, warp_start, warp_end, mask_start,
                     mask_end, ada_start, ada_end, mask_strength,

tts/VALL-E-X/main.py

@@ -150,22 +150,22 @@ voice_designer = VoiceDesigner()
 
 def main(args):
     
-    print(args.mode)
+    print("Starting Inference on Vall-E-X With Inputs")
+    print(f"Mode: {args.mode}")
     
-    print(args.text)
-    print(args.audio_wav_files)
-    print(args.whisper_folder_path)
-    print(args.vocos_folder_path)
+    print(f"Text: {args.text}")
+    print(f"List of Files For Create: {args.audio_wav_files}")
+    print(f"Whisper Folder Path: {args.whisper_folder_path}")
+    print(f"Vocos Folder Path: {args.vocos_folder_path}")
+    print(f"Vall-E-X Path: {args.vallex_path}")
     
-    print(args.vallex_path)
+    print(f"Prompt Path: {args.prompt_path}")
+    print(f"Prompt Name: {args.prompt_name}")
     
-    print(args.prompt_path)
-    print(args.prompt_name)
+    print(f"Audio Name: {args.audio_name}")
+    print(f"Audio Path: {args.audio_path}")
     
-    print(args.audio_name)
-    print(args.audio_path)
-    
-    print(args.tmp_work_dir)
+    print(f"Temp Work Dir: {args.tmp_work_dir}")
     
     voice_designer.temp_path = pathlib.Path(args.tmp_work_dir)
 
@@ -174,6 +174,7 @@ def main(args):
                                              pathlib.Path(args.whisper_folder_path))
     
     if args.mode == 0: # run inference
+        print("Running Inference")
         voice_designer.tts_with_prompt(prompt_dir=pathlib.Path(args.prompt_path),
                                        prompt_name=pathlib.Path(args.prompt_name),
                                        audio_output_path=pathlib.Path(args.audio_path),
@@ -234,3 +235,17 @@ if __name__ == "__main__":
 
 # create voice example
 #/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --mode 1 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt" --tmp-work-dir "/tmp"
+
+#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python 
+#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py 
+# --mode 1 
+# --audio-wav-files 
+# "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" 
+# "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" 
+# --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" 
+# --whisper-model medium 
+# --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" 
+# --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" 
+# --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" 
+# --prompt-name "test_prompt" 
+# --tmp-work-dir "/tmp"

tts/VALL-E-X/utils/prompt_making.py

@@ -84,6 +84,7 @@ def make_prompt(name:pathlib.Path, audio_prompt_path:pathlib.Path,audio_prompt_o
     else:
         save_path = os.path.join("./customs/", f"{name}.npz")
     np.savez(save_path, audio_tokens=audio_tokens, text_tokens=text_tokens, lang_code=lang2code[lang_pr])
+    print(f"Embedding Save Path {save_path}")
     logging.info(f"Successful. Prompt saved to {save_path}")
 
 def make_transcript(name, wav, sr, whisper_folder_path: pathlib.Path,transcript=None):