From 7f1969c8293e1700cede4674fe87add101774e47 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Mon, 16 Oct 2023 23:31:49 -0400 Subject: [PATCH] Create works again, now porting this over to the ml repo --- tts/VALL-E-X/main.py | 60 ++++++++++++++--------------- tts/VALL-E-X/utils/generation.py | 3 +- tts/VALL-E-X/utils/prompt_making.py | 13 ++++--- 3 files changed, 38 insertions(+), 38 deletions(-) diff --git a/tts/VALL-E-X/main.py b/tts/VALL-E-X/main.py index bd927e5..9a2ed4c 100755 --- a/tts/VALL-E-X/main.py +++ b/tts/VALL-E-X/main.py @@ -99,28 +99,28 @@ class VoiceDesigner(): def initialize_voice_designer(self,vall_e_x_file_path:pathlib.Path,vocos_folder_path:pathlib.Path,whisper_folder_path:pathlib.Path): load_models_from_pathes(vocos_folder_path=vocos_folder_path,vall_e_x_file_path=vall_e_x_file_path,whisper_folder_path=whisper_folder_path) - def create_prompt(self,audio_file_paths:List[pathlib.Path],prompt_file_name:str,prompt_output_path:str): + def create_prompt(self,audio_file_paths:List[pathlib.Path], + prompt_file_name:pathlib.Path, + prompt_output_path:pathlib.Path, + whisper_folder_path:pathlib.Path): # multiple audio files concat # concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways if len(audio_file_paths) > 0: # TODO: going to assume brandon renames the files as they are uploaded - try:# ./customs/ <-- folder it adds by default until we modified. + # try:# ./customs/ <-- folder it adds by default until we modified. # load the audio files and then concat them. - tensor_result = AudioOps.stitch_wav_files_resample(files=audio_file_paths) - # save to disk - generated_name = f"{uuid.uuid4()}" + tensor_result = AudioOps.stitch_wav_files_resample(files=audio_file_paths) + # save to disk + generated_name = f"{uuid.uuid4()}" + + concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav") + sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE) + # save result into processing... + # trim to 15 seconds. + audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav") + AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path) + make_prompt(name=prompt_file_name,audio_prompt_output_path=prompt_output_path,audio_prompt_path=audio_cut_path,whisper_folder_path=whisper_folder_path) - concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav") - sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE) - # save result into processing... - # trim to 15 seconds. - audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav") - AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path) - make_prompt(name=prompt_file_name,audio_prompt_output_path=prompt_output_path,audio_prompt_path=audio_cut_path) - - except Exception as e: - self.logger.debug(f"Exception: {e.with_traceback()}") - raise else: self.logger.debug(f"No audio files to load") @@ -189,7 +189,9 @@ def main(args): elif args.mode == 1: # run create voice voice_designer.create_prompt(audio_file_paths=args.audio_wav_files, - prompt_file_name=pathlib.Path(args.prompt_name)) + prompt_output_path=pathlib.Path(args.prompt_path), + prompt_file_name=pathlib.Path(args.prompt_name), + whisper_folder_path=pathlib.Path(args.whisper_folder_path)) if __name__ == "__main__": parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.") @@ -217,26 +219,24 @@ if __name__ == "__main__": main(args) -# all params command - -#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python -#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" +# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python +# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py +# --text "hello world" # --mode 0 # --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" -# --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt" +# --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" +# --whisper-model medium # --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" # --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" # --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" -# --prompt-name "test_prompt.npz" -# --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" -# --audio-name "test.wav" +# --prompt-name "goku.npz" +# --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" +# --audio-name "hello_world_test.wav" # --tmp-work-dir "/tmp" -# all params command copy paste -# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt" --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "test.wav" --tmp-work-dir "/tmp" - # inference example +#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "goku.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "hello_world_test.wav" --tmp-work-dir "/tmp" -#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "goku.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "hello_world_test.wav" --tmp-work-dir "/tmp" - +# create voice example +#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --mode 1 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt.npz" --tmp-work-dir "/tmp" diff --git a/tts/VALL-E-X/utils/generation.py b/tts/VALL-E-X/utils/generation.py index 1b5eae5..c913aa3 100644 --- a/tts/VALL-E-X/utils/generation.py +++ b/tts/VALL-E-X/utils/generation.py @@ -88,10 +88,9 @@ def load_models_from_pathes(vocos_folder_path:pathlib.Path,vall_e_x_file_path:pa vocos_model.eval() vocos = vocos_model vocos.to(device) - # TODO replace + # TODO replace model name whisper_model_name = "medium" whisper_model = whisper.load_model(whisper_model_name,download_root=whisper_folder_path).cpu() - # DEPRECATED def load_models_from_mount(model_path:pathlib.Path,vocos_path:pathlib.Path,vall_e_x:pathlib.Path): global model, codec, vocos diff --git a/tts/VALL-E-X/utils/prompt_making.py b/tts/VALL-E-X/utils/prompt_making.py index 4f2dcf1..d71a359 100644 --- a/tts/VALL-E-X/utils/prompt_making.py +++ b/tts/VALL-E-X/utils/prompt_making.py @@ -26,8 +26,8 @@ if torch.cuda.is_available(): codec = AudioTokenizer(device) -if not os.path.exists("./whisper/"): os.mkdir("./whisper/") -# whisper_model = None +#if not os.path.exists("./whisper/"): os.mkdir("./whisper/") +whisper_model = None @torch.no_grad() def transcribe_one(model, audio_path): @@ -54,7 +54,7 @@ def transcribe_one(model, audio_path): text_pr += "." return lang, text_pr -def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=None): +def make_prompt(name:pathlib.Path, audio_prompt_path:pathlib.Path,audio_prompt_output_path:pathlib.Path,whisper_folder_path:pathlib.Path): global model, text_collater, text_tokenizer, codec wav_pr, sr = torchaudio.load(audio_prompt_path) # check length @@ -62,7 +62,7 @@ def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=No raise ValueError(f"Prompt too long, expect length below 15 seconds, got {wav_pr / sr} seconds.") if wav_pr.size(0) == 2: wav_pr = wav_pr.mean(0, keepdim=True) - text_pr, lang_pr = make_transcript(name, wav_pr, sr, None) + text_pr, lang_pr = make_transcript(name, wav_pr, sr,whisper_folder_path,None) # tokenize audio encoded_frames = tokenize_audio(codec, (wav_pr, sr)) @@ -86,7 +86,7 @@ def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=No np.savez(save_path, audio_tokens=audio_tokens, text_tokens=text_tokens, lang_code=lang2code[lang_pr]) logging.info(f"Successful. Prompt saved to {save_path}") -def make_transcript(name, wav, sr, transcript=None): +def make_transcript(name, wav, sr, whisper_folder_path: pathlib.Path,transcript=None): if not isinstance(wav, torch.FloatTensor): wav = torch.tensor(wav) @@ -101,7 +101,8 @@ def make_transcript(name, wav, sr, transcript=None): logging.info("Transcript not given, using Whisper...") global whisper_model if whisper_model is None: - whisper_model = whisper.load_model("medium", download_root=os.path.join(os.getcwd(), "whisper")) + # TODO: replace + whisper_model = whisper.load_model("medium", download_root=whisper_folder_path) whisper_model.to(device) torchaudio.save(f"./prompts/{name}.wav", wav, sr) lang, text = transcribe_one(whisper_model, f"./prompts/{name}.wav")