storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-28 09:08:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create works again, now porting this over to the ml repo

Browse Source
This commit is contained in:
Michael Chung
2023-10-16 23:31:49 -04:00
parent 410c73fffe
commit 7f1969c829
3 changed files with 38 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tts/VALL-E-X/main.py
+30 -30
View File
@@ -99,28 +99,28 @@ class VoiceDesigner():
def initialize_voice_designer(self,vall_e_x_file_path:pathlib.Path,vocos_folder_path:pathlib.Path,whisper_folder_path:pathlib.Path): def initialize_voice_designer(self,vall_e_x_file_path:pathlib.Path,vocos_folder_path:pathlib.Path,whisper_folder_path:pathlib.Path):
load_models_from_pathes(vocos_folder_path=vocos_folder_path,vall_e_x_file_path=vall_e_x_file_path,whisper_folder_path=whisper_folder_path) load_models_from_pathes(vocos_folder_path=vocos_folder_path,vall_e_x_file_path=vall_e_x_file_path,whisper_folder_path=whisper_folder_path)
def create_prompt(self,audio_file_paths:List[pathlib.Path],prompt_file_name:str,prompt_output_path:str): def create_prompt(self,audio_file_paths:List[pathlib.Path],
prompt_file_name:pathlib.Path,
prompt_output_path:pathlib.Path,
whisper_folder_path:pathlib.Path):
# multiple audio files concat # multiple audio files concat
# concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways # concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways
if len(audio_file_paths) > 0: if len(audio_file_paths) > 0:
# TODO: going to assume brandon renames the files as they are uploaded # TODO: going to assume brandon renames the files as they are uploaded
try:# ./customs/ <-- folder it adds by default until we modified. # try:# ./customs/ <-- folder it adds by default until we modified.
# load the audio files and then concat them. # load the audio files and then concat them.
tensor_result = AudioOps.stitch_wav_files_resample(files=audio_file_paths) tensor_result = AudioOps.stitch_wav_files_resample(files=audio_file_paths)
# save to disk # save to disk
generated_name = f"{uuid.uuid4()}" generated_name = f"{uuid.uuid4()}"
concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav")
sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE)
# save result into processing...
# trim to 15 seconds.
audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav")
AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path)
make_prompt(name=prompt_file_name,audio_prompt_output_path=prompt_output_path,audio_prompt_path=audio_cut_path,whisper_folder_path=whisper_folder_path)
concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav")
sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE)
# save result into processing...
# trim to 15 seconds.
audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav")
AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path)
make_prompt(name=prompt_file_name,audio_prompt_output_path=prompt_output_path,audio_prompt_path=audio_cut_path)
except Exception as e:
self.logger.debug(f"Exception: {e.with_traceback()}")
raise
else: else:
self.logger.debug(f"No audio files to load") self.logger.debug(f"No audio files to load")
@@ -189,7 +189,9 @@ def main(args):
elif args.mode == 1: # run create voice elif args.mode == 1: # run create voice
voice_designer.create_prompt(audio_file_paths=args.audio_wav_files, voice_designer.create_prompt(audio_file_paths=args.audio_wav_files,
prompt_file_name=pathlib.Path(args.prompt_name)) prompt_output_path=pathlib.Path(args.prompt_path),
prompt_file_name=pathlib.Path(args.prompt_name),
whisper_folder_path=pathlib.Path(args.whisper_folder_path))
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.") parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.")
@@ -217,26 +219,24 @@ if __name__ == "__main__":
main(args) main(args)
# all params command
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world"
# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python
# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py
# --text "hello world"
# --mode 0 # --mode 0
# --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" # --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav"
# --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt" # --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper"
# --whisper-model medium
# --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" # --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz"
# --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" # --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt"
# --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" # --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts"
# --prompt-name "test_prompt.npz" # --prompt-name "goku.npz"
# --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" # --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/"
# --audio-name "test.wav" # --audio-name "hello_world_test.wav"
# --tmp-work-dir "/tmp" # --tmp-work-dir "/tmp"
# all params command copy paste
# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt" --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "test.wav" --tmp-work-dir "/tmp"
# inference example # inference example
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "goku.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "hello_world_test.wav" --tmp-work-dir "/tmp"
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "goku.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "hello_world_test.wav" --tmp-work-dir "/tmp" # create voice example
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --mode 1 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt.npz" --tmp-work-dir "/tmp"
tts/VALL-E-X/utils/generation.py
+1 -2
View File
@@ -88,10 +88,9 @@ def load_models_from_pathes(vocos_folder_path:pathlib.Path,vall_e_x_file_path:pa
vocos_model.eval() vocos_model.eval()
vocos = vocos_model vocos = vocos_model
vocos.to(device) vocos.to(device)
# TODO replace # TODO replace model name
whisper_model_name = "medium" whisper_model_name = "medium"
whisper_model = whisper.load_model(whisper_model_name,download_root=whisper_folder_path).cpu() whisper_model = whisper.load_model(whisper_model_name,download_root=whisper_folder_path).cpu()
# DEPRECATED # DEPRECATED
def load_models_from_mount(model_path:pathlib.Path,vocos_path:pathlib.Path,vall_e_x:pathlib.Path): def load_models_from_mount(model_path:pathlib.Path,vocos_path:pathlib.Path,vall_e_x:pathlib.Path):
global model, codec, vocos global model, codec, vocos
tts/VALL-E-X/utils/prompt_making.py
+7 -6
View File
@@ -26,8 +26,8 @@ if torch.cuda.is_available():
codec = AudioTokenizer(device) codec = AudioTokenizer(device)
if not os.path.exists("./whisper/"): os.mkdir("./whisper/") #if not os.path.exists("./whisper/"): os.mkdir("./whisper/")
# whisper_model = None whisper_model = None
@torch.no_grad() @torch.no_grad()
def transcribe_one(model, audio_path): def transcribe_one(model, audio_path):
@@ -54,7 +54,7 @@ def transcribe_one(model, audio_path):
text_pr += "." text_pr += "."
return lang, text_pr return lang, text_pr
def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=None): def make_prompt(name:pathlib.Path, audio_prompt_path:pathlib.Path,audio_prompt_output_path:pathlib.Path,whisper_folder_path:pathlib.Path):
global model, text_collater, text_tokenizer, codec global model, text_collater, text_tokenizer, codec
wav_pr, sr = torchaudio.load(audio_prompt_path) wav_pr, sr = torchaudio.load(audio_prompt_path)
# check length # check length
@@ -62,7 +62,7 @@ def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=No
raise ValueError(f"Prompt too long, expect length below 15 seconds, got {wav_pr / sr} seconds.") raise ValueError(f"Prompt too long, expect length below 15 seconds, got {wav_pr / sr} seconds.")
if wav_pr.size(0) == 2: if wav_pr.size(0) == 2:
wav_pr = wav_pr.mean(0, keepdim=True) wav_pr = wav_pr.mean(0, keepdim=True)
text_pr, lang_pr = make_transcript(name, wav_pr, sr, None) text_pr, lang_pr = make_transcript(name, wav_pr, sr,whisper_folder_path,None)
# tokenize audio # tokenize audio
encoded_frames = tokenize_audio(codec, (wav_pr, sr)) encoded_frames = tokenize_audio(codec, (wav_pr, sr))
@@ -86,7 +86,7 @@ def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=No
np.savez(save_path, audio_tokens=audio_tokens, text_tokens=text_tokens, lang_code=lang2code[lang_pr]) np.savez(save_path, audio_tokens=audio_tokens, text_tokens=text_tokens, lang_code=lang2code[lang_pr])
logging.info(f"Successful. Prompt saved to {save_path}") logging.info(f"Successful. Prompt saved to {save_path}")
def make_transcript(name, wav, sr, transcript=None): def make_transcript(name, wav, sr, whisper_folder_path: pathlib.Path,transcript=None):
if not isinstance(wav, torch.FloatTensor): if not isinstance(wav, torch.FloatTensor):
wav = torch.tensor(wav) wav = torch.tensor(wav)
@@ -101,7 +101,8 @@ def make_transcript(name, wav, sr, transcript=None):
logging.info("Transcript not given, using Whisper...") logging.info("Transcript not given, using Whisper...")
global whisper_model global whisper_model
if whisper_model is None: if whisper_model is None:
whisper_model = whisper.load_model("medium", download_root=os.path.join(os.getcwd(), "whisper")) # TODO: replace
whisper_model = whisper.load_model("medium", download_root=whisper_folder_path)
whisper_model.to(device) whisper_model.to(device)
torchaudio.save(f"./prompts/{name}.wav", wav, sr) torchaudio.save(f"./prompts/{name}.wav", wav, sr)
lang, text = transcribe_one(whisper_model, f"./prompts/{name}.wav") lang, text = transcribe_one(whisper_model, f"./prompts/{name}.wav")