mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-16 00:13:07 +00:00
Inference works again with abs pathes
This commit is contained in:
Michael Chung
+149
-158
@@ -2,11 +2,8 @@ import argparse
|
||||
|
||||
# python3 main.py -create_prompt_with_list 20.wav 21.wav -create_prompt_name alice
|
||||
# python3 main.py -use_prompt_name alice -output_file_name alice_says.wav -text "This is the text to be converted to speech."
|
||||
|
||||
# python3 main.py -create_prompt_with_list 20.wav 21.wav 20.wav 21.wav -create_prompt_name alice -use_prompt_name alice -output_file_name alice_says.wav -text "This is my voice."
|
||||
|
||||
# python3 main.py -create_prompt_with_list goku.wav -create_prompt_name goku -use_prompt_name goku -output_file_name goku_test.wav -text "oh my god krillin! where did you put my sensu beans!?"
|
||||
|
||||
# python3 main.py -create_prompt_with_list
|
||||
|
||||
from typing import List
|
||||
@@ -16,203 +13,186 @@ import pathlib
|
||||
|
||||
from utils.prompt_making import make_prompt
|
||||
import logging
|
||||
|
||||
from utils.generation import SAMPLE_RATE, generate_audio, preload_models,load_models_from_mount
|
||||
from utils.generation import SAMPLE_RATE, generate_audio,load_models_from_pathes
|
||||
import soundfile as sf
|
||||
import numpy as np
|
||||
import torchaudio
|
||||
from typing import List
|
||||
import uuid
|
||||
import torchaudio.transforms as T
|
||||
|
||||
# class AudioOps():
|
||||
class AudioOps():
|
||||
|
||||
# @staticmethod
|
||||
# def stitch_wav_tensors_with_crossfade(wav_tensor1:np.array,wav_tensor2:np.array,sample_rate:int):
|
||||
# # Load two audio signals as NumPy tensors
|
||||
# # Compute the length of the cross-fade window in samples
|
||||
# fade_len = int(0.01 * sample_rate)
|
||||
@staticmethod
|
||||
def stitch_wav_tensors_with_crossfade(wav_tensor1:np.array,wav_tensor2:np.array,sample_rate:int):
|
||||
# Load two audio signals as NumPy tensors
|
||||
# Compute the length of the cross-fade window in samples
|
||||
fade_len = int(0.01 * sample_rate)
|
||||
|
||||
# # Create a cross-fade window
|
||||
# fade_window = np.hanning(2*fade_len)
|
||||
# Create a cross-fade window
|
||||
fade_window = np.hanning(2*fade_len)
|
||||
|
||||
# # Concatenate the signals with cross-fade
|
||||
# audio_out = np.concatenate((wav_tensor1[:-fade_len],
|
||||
# wav_tensor1[-fade_len:] * fade_window[:fade_len] + wav_tensor2[:fade_len] * fade_window[fade_len:],
|
||||
# wav_tensor2[fade_len:]))
|
||||
# return audio_out
|
||||
# Concatenate the signals with cross-fade
|
||||
audio_out = np.concatenate((wav_tensor1[:-fade_len],
|
||||
wav_tensor1[-fade_len:] * fade_window[:fade_len] + wav_tensor2[:fade_len] * fade_window[fade_len:],
|
||||
wav_tensor2[fade_len:]))
|
||||
return audio_out
|
||||
|
||||
# @staticmethod
|
||||
# def cut_audio_range(file_path, start_seconds, end_seconds, output_path="cut_audio.wav"):
|
||||
# """
|
||||
# Cuts out a specific segment from an audio file and saves the result.
|
||||
@staticmethod
|
||||
def cut_audio_range(file_path, start_seconds, end_seconds, output_path="cut_audio.wav"):
|
||||
"""
|
||||
Cuts out a specific segment from an audio file and saves the result.
|
||||
|
||||
# Args:
|
||||
# - file_path (str): Path to the audio file from which the segment will be cut.
|
||||
# - start_seconds (float): Start time of the segment to be cut out in seconds.
|
||||
# - end_seconds (float): End time of the segment to be cut out in seconds.
|
||||
# - output_path (str): Path to save the resulting audio. Defaults to "cut_audio.wav".
|
||||
Args:
|
||||
- file_path (str): Path to the audio file from which the segment will be cut.
|
||||
- start_seconds (float): Start time of the segment to be cut out in seconds.
|
||||
- end_seconds (float): End time of the segment to be cut out in seconds.
|
||||
- output_path (str): Path to save the resulting audio. Defaults to "cut_audio.wav".
|
||||
|
||||
# Returns:
|
||||
# - None
|
||||
# """
|
||||
# # Load the audio file
|
||||
# waveform, sr = torchaudio.load(file_path)
|
||||
# # Convert the range to sample indices
|
||||
# start_sample = int(start_seconds * sr)
|
||||
# end_sample = int(end_seconds * sr)
|
||||
# cut_tensor = waveform[:,start_sample:end_sample]
|
||||
# torchaudio.save(output_path, cut_tensor, sr)
|
||||
Returns:
|
||||
- None
|
||||
"""
|
||||
# Load the audio file
|
||||
waveform, sr = torchaudio.load(file_path)
|
||||
# Convert the range to sample indices
|
||||
start_sample = int(start_seconds * sr)
|
||||
end_sample = int(end_seconds * sr)
|
||||
cut_tensor = waveform[:,start_sample:end_sample]
|
||||
torchaudio.save(output_path, cut_tensor, sr)
|
||||
|
||||
# @staticmethod
|
||||
# def stitch_wav_files_resample(root_path:pathlib.Path, files:List[str],target_sample_rate:int=SAMPLE_RATE):
|
||||
# resampler = None
|
||||
# result = None
|
||||
# for file in files:
|
||||
# if type(result) == type(None):
|
||||
# tensor, sr = torchaudio.load(root_path / pathlib.Path(file))
|
||||
# resampler = T.Resample(orig_freq=sr,new_freq=target_sample_rate)
|
||||
# resampled_tensor = resampler(tensor)
|
||||
# result = resampled_tensor.squeeze()
|
||||
# result = np.array(result)
|
||||
# else:
|
||||
# tensor, _ = torchaudio.load(root_path / pathlib.Path(file))
|
||||
# resampled_tensor = resampler(tensor)
|
||||
# resampled_tensor = resampled_tensor.squeeze()
|
||||
# resampled_tensor = np.array(resampled_tensor)
|
||||
# result = AudioOps.stitch_wav_tensors_with_crossfade(result,resampled_tensor,sample_rate=target_sample_rate)
|
||||
@staticmethod
|
||||
def stitch_wav_files_resample(files:List[str],target_sample_rate:int=SAMPLE_RATE):
|
||||
resampler = None
|
||||
result = None
|
||||
for file in files:
|
||||
if type(result) == type(None):
|
||||
tensor, sr = torchaudio.load(pathlib.Path(file))
|
||||
resampler = T.Resample(orig_freq=sr,new_freq=target_sample_rate)
|
||||
resampled_tensor = resampler(tensor)
|
||||
result = resampled_tensor.squeeze()
|
||||
result = np.array(result)
|
||||
else:
|
||||
tensor, _ = torchaudio.load(pathlib.Path(file))
|
||||
resampled_tensor = resampler(tensor)
|
||||
resampled_tensor = resampled_tensor.squeeze()
|
||||
resampled_tensor = np.array(resampled_tensor)
|
||||
result = AudioOps.stitch_wav_tensors_with_crossfade(result,resampled_tensor,sample_rate=target_sample_rate)
|
||||
|
||||
# return result
|
||||
return result
|
||||
|
||||
# class VoiceDesigner():
|
||||
# def __init__(self) -> None:
|
||||
# # load models from host system.
|
||||
# self.logger = self.setup_logger()
|
||||
class VoiceDesigner():
|
||||
def __init__(self) -> None:
|
||||
# load models from host system.
|
||||
self.logger = self.setup_logger()
|
||||
# host path for work
|
||||
self.temp_path = None
|
||||
|
||||
# self.mount_folder = pathlib.Path.cwd() / pathlib.Path("Vall-E-mount")
|
||||
# # Path for the inputs bunch of audio files
|
||||
# self.input_path = self.mount_folder / pathlib.Path("input")
|
||||
# # Path for the models to be used
|
||||
# self.models_path = self.mount_folder / pathlib.Path("models")
|
||||
# # Path for the existing presets
|
||||
# self.presets_path = self.mount_folder / pathlib.Path("presets")
|
||||
# # Path for the new prompts created by the user
|
||||
# self.prompts_path = self.mount_folder / pathlib.Path("prompts")
|
||||
# # Path for the temporary files
|
||||
# self.temp_path = self.mount_folder / pathlib.Path("temp")
|
||||
# # Path for output
|
||||
# self.output_path = self.mount_folder / pathlib.Path("output")
|
||||
# # Will have to load whisper from here as well ... todo fix
|
||||
# load_models_from_mount(model_path=self.models_path,
|
||||
# vocos_path=pathlib.Path("vocos-encodec-24khz"),
|
||||
# vall_e_x=pathlib.Path("vallex-checkpoint.pt"))
|
||||
def tts_with_prompt(self, prompt_dir:pathlib.Path,
|
||||
prompt_name:pathlib.Path,
|
||||
audio_output_path:pathlib.Path,
|
||||
output_file_name:pathlib.Path,
|
||||
text:str):
|
||||
|
||||
# def tts_with_prompt(self,prompt:pathlib.Path,text:str,output_file_name:str):
|
||||
# # TODO Filtering bad words
|
||||
# # TODO check if the files are actually wav ...
|
||||
# audio_array = generate_audio(text,prompt=prompt)
|
||||
# # save audio to disk
|
||||
# sf.write(self.output_path / pathlib.Path(output_file_name), audio_array, SAMPLE_RATE)
|
||||
|
||||
# def create_prompt(self,audio_file_paths:List[pathlib.Path],prompt_file_name:str):
|
||||
# # multiple audio files concat
|
||||
# # concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways
|
||||
# if len(audio_file_paths) > 0:
|
||||
# # TODO: going to assume brandon renames the files as they are uploaded
|
||||
# try:# ./customs/ <-- folder it adds by default until we modified.
|
||||
# # load the audio files and then concat them.
|
||||
# tensor_result = AudioOps.stitch_wav_files_resample(root_path=self.input_path,files=audio_file_paths)
|
||||
# # save to disk
|
||||
# generated_name = f"{uuid.uuid4()}"
|
||||
audio_array = generate_audio(text,prompt_name=prompt_name,prompt_dir=prompt_dir)
|
||||
# save audio to disk
|
||||
sf.write(audio_output_path / output_file_name, audio_array, SAMPLE_RATE)
|
||||
|
||||
def initialize_voice_designer(self,vall_e_x_file_path:pathlib.Path,vocos_folder_path:pathlib.Path,whisper_folder_path:pathlib.Path):
|
||||
load_models_from_pathes(vocos_folder_path=vocos_folder_path,vall_e_x_file_path=vall_e_x_file_path,whisper_folder_path=whisper_folder_path)
|
||||
|
||||
def create_prompt(self,audio_file_paths:List[pathlib.Path],prompt_file_name:str,prompt_output_path:str):
|
||||
# multiple audio files concat
|
||||
# concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways
|
||||
if len(audio_file_paths) > 0:
|
||||
# TODO: going to assume brandon renames the files as they are uploaded
|
||||
try:# ./customs/ <-- folder it adds by default until we modified.
|
||||
# load the audio files and then concat them.
|
||||
tensor_result = AudioOps.stitch_wav_files_resample(files=audio_file_paths)
|
||||
# save to disk
|
||||
generated_name = f"{uuid.uuid4()}"
|
||||
|
||||
# concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav")
|
||||
# sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE)
|
||||
# # save result into processing...
|
||||
# # trim to 15 seconds.
|
||||
# audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav")
|
||||
# AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path)
|
||||
# # TODO get whisper x to work in the container and generate transcription that way or use a different container to do that work.
|
||||
# make_prompt(name=prompt_file_name,audio_prompt_output_path=self.prompts_path,audio_prompt_path=audio_cut_path,transcript=None)
|
||||
# except Exception as e:
|
||||
# self.logger.debug(f"Exception: {e.with_traceback()}")
|
||||
# raise
|
||||
# else:
|
||||
# self.logger.debug(f"No audio files to load")
|
||||
concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav")
|
||||
sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE)
|
||||
# save result into processing...
|
||||
# trim to 15 seconds.
|
||||
audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav")
|
||||
AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path)
|
||||
make_prompt(name=prompt_file_name,audio_prompt_output_path=prompt_output_path,audio_prompt_path=audio_cut_path)
|
||||
|
||||
except Exception as e:
|
||||
self.logger.debug(f"Exception: {e.with_traceback()}")
|
||||
raise
|
||||
else:
|
||||
self.logger.debug(f"No audio files to load")
|
||||
|
||||
# def setup_logger(self):
|
||||
# # Create a logger object
|
||||
# logger = logging.getLogger(__name__)
|
||||
def setup_logger(self):
|
||||
# Create a logger object
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# # Set the log level
|
||||
# logger.setLevel(logging.DEBUG)
|
||||
# Set the log level
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
# # Create a file handler to log messages to a file
|
||||
# file_handler = logging.FileHandler('docker_valle-x.log')
|
||||
# file_handler.setLevel(logging.DEBUG)
|
||||
# Create a file handler to log messages to a file
|
||||
file_handler = logging.FileHandler('docker_valle-x.log')
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
|
||||
# # Create a console handler to log messages to the console
|
||||
# console_handler = logging.StreamHandler()
|
||||
# console_handler.setLevel(logging.DEBUG)
|
||||
# Create a console handler to log messages to the console
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.DEBUG)
|
||||
|
||||
# # Create a formatter
|
||||
# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
# Create a formatter
|
||||
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
|
||||
|
||||
# # Set the formatter for the handlers
|
||||
# file_handler.setFormatter(formatter)
|
||||
# console_handler.setFormatter(formatter)
|
||||
# Set the formatter for the handlers
|
||||
file_handler.setFormatter(formatter)
|
||||
console_handler.setFormatter(formatter)
|
||||
|
||||
# # Add the handlers to the logger
|
||||
# logger.addHandler(file_handler)
|
||||
# logger.addHandler(console_handler)
|
||||
# Add the handlers to the logger
|
||||
logger.addHandler(file_handler)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# return logger
|
||||
return logger
|
||||
|
||||
#global voice_designer
|
||||
#voice_designer = VoiceDesigner()
|
||||
global voice_designer
|
||||
voice_designer = VoiceDesigner()
|
||||
|
||||
def main(args):
|
||||
# print(args.create_prompt_with_list)
|
||||
# print(args.create_prompt_name)
|
||||
|
||||
# print(args.text)
|
||||
# print(args.use_prompt_name)
|
||||
# print(args.output_file_name)
|
||||
|
||||
# if args.create_prompt_name is not None and args.create_prompt_name is not None:
|
||||
# voice_designer.create_prompt(audio_file_paths=args.create_prompt_with_list,
|
||||
# prompt_file_name=args.create_prompt_name)
|
||||
|
||||
# if args.text is not None and args.use_prompt_name is not None:
|
||||
# voice_designer.tts_with_prompt(prompt=args.use_prompt_name,
|
||||
# text=args.text,
|
||||
# output_file_name=args.output_file_name)
|
||||
|
||||
|
||||
print(args.mode)
|
||||
|
||||
print(args.text)
|
||||
print(args.audio_wav_files)
|
||||
print(args.whisper_file_path)
|
||||
print(args.whisper_folder_path)
|
||||
print(args.vocos_folder_path)
|
||||
|
||||
print(args.vallex_path)
|
||||
|
||||
print(args.prompt_path)
|
||||
print(args.audio_name)
|
||||
print(args.prompt_name)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(args.audio_name)
|
||||
print(args.audio_path)
|
||||
|
||||
parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.")
|
||||
print(args.tmp_work_dir)
|
||||
|
||||
voice_designer.temp_path = pathlib.Path(args.tmp_work_dir)
|
||||
|
||||
# parser.add_argument('-create_prompt_with_list', metavar='prompt wav files', nargs='+',
|
||||
# help='Takes in a list of .wav files for an audio prompt provide the absolute path. Must be less than 15 seconds otherwise will error out.')
|
||||
# parser.add_argument('-create_prompt_name',metavar="prompt name to create",
|
||||
# help='required with -create-prompt prompt output name this will be written to the mount. In Vall-E-mount/prompts')
|
||||
# parser.add_argument("-use_prompt_name",metavar="prompt you want to use",type=str)
|
||||
# parser.add_argument("-output_file_name",metavar="file name",type=str)
|
||||
# parser.add_argument('-text', metavar='text to synthesize', type=str,help='Text to be converted to speech.')
|
||||
voice_designer.initialize_voice_designer(pathlib.Path(args.vallex_path),
|
||||
pathlib.Path(args.vocos_folder_path),
|
||||
pathlib.Path(args.whisper_folder_path))
|
||||
|
||||
if args.mode == 0: # run inference
|
||||
voice_designer.tts_with_prompt(prompt_dir=pathlib.Path(args.prompt_path),
|
||||
prompt_name=pathlib.Path(args.prompt_name),
|
||||
audio_output_path=pathlib.Path(args.audio_path),
|
||||
output_file_name=pathlib.Path(args.audio_name),
|
||||
text=args.text)
|
||||
|
||||
elif args.mode == 1: # run create voice
|
||||
voice_designer.create_prompt(audio_file_paths=args.audio_wav_files,
|
||||
prompt_file_name=pathlib.Path(args.prompt_name))
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.")
|
||||
# modes
|
||||
parser.add_argument('--mode', metavar='model mode',type=int,help='Mode 0 - (Inference) | Mode 1 - (Create)')
|
||||
parser.add_argument('--text', metavar='text to synthesize', type=str,help='Text to be converted to speech.')
|
||||
@@ -220,7 +200,8 @@ if __name__ == "__main__":
|
||||
parser.add_argument('--audio-wav-files', metavar='wav to voice clone', type=str,nargs='+',help='Takes in a list of .wav files for an audio prompt provide the absolute path. Must be less than 15 seconds otherwise will error out. (Create Voice Only)')
|
||||
|
||||
# model pathes
|
||||
parser.add_argument("--whisper-file-path",metavar="abs whisper model path",type=str,help="Path to the Whisper Model .pt file")
|
||||
parser.add_argument("--whisper-folder-path",metavar="abs whisper folder path",type=str,help="Path to the Whisper folder where the .pt file")
|
||||
parser.add_argument("--whisper-model",metavar="model large or medium",type=str,help="medium or large")
|
||||
parser.add_argument("--vocos-folder-path",metavar="abs vocos model path",type=str,help="Path to vocos codec Model requires the (folder path)")
|
||||
parser.add_argument("--vallex-path",metavar="abs vall-e model path",type=str,help="Path to the Valle Model File")
|
||||
|
||||
@@ -230,13 +211,17 @@ if __name__ == "__main__":
|
||||
parser.add_argument("--prompt-path",metavar="abs path to prompt",type=str,help="Prompt Embedding Path (Inference and Create Voice)")
|
||||
parser.add_argument("--prompt-name",metavar="abs path to prompt",type=str,help="Prompt Name (Inference and Create Voice)")
|
||||
|
||||
parser.add_argument("--tmp-work-dir",metavar="abs tmp dir",type=str,help="Temporary work directory in host system.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
main(args)
|
||||
|
||||
# all params command
|
||||
|
||||
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python
|
||||
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world"
|
||||
|
||||
# --mode 0
|
||||
# --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav"
|
||||
# --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt"
|
||||
@@ -246,6 +231,12 @@ if __name__ == "__main__":
|
||||
# --prompt-name "test_prompt.npz"
|
||||
# --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/"
|
||||
# --audio-name "test.wav"
|
||||
# --tmp-work-dir "/tmp"
|
||||
|
||||
# all params command copy paste
|
||||
# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt" --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "test.wav"
|
||||
# /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-file-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper/medium.pt" --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "test.wav" --tmp-work-dir "/tmp"
|
||||
|
||||
# inference example
|
||||
|
||||
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "goku.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "hello_world_test.wav" --tmp-work-dir "/tmp"
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ import torch
|
||||
from vocos import Vocos
|
||||
from vocos.feature_extractors import FeatureExtractor, EncodecFeatures
|
||||
|
||||
import whisper
|
||||
|
||||
import logging
|
||||
import langid
|
||||
langid.set_languages(['en', 'zh', 'ja'])
|
||||
@@ -40,14 +42,57 @@ checkpoints_dir = "./checkpoints/"
|
||||
model_checkpoint_name = "vallex-checkpoint.pt"
|
||||
|
||||
model = None
|
||||
|
||||
codec = None
|
||||
|
||||
vocos = None
|
||||
whisper_model = None
|
||||
|
||||
text_tokenizer = PhonemeBpeTokenizer(tokenizer_path="./utils/g2p/bpe_69.json")
|
||||
text_collater = get_text_token_collater()
|
||||
|
||||
def load_models_from_pathes(vocos_folder_path:pathlib.Path,vall_e_x_file_path:pathlib.Path,whisper_folder_path:pathlib.Path):
|
||||
global model, codec, vocos, whisper_model
|
||||
|
||||
model = VALLE(
|
||||
N_DIM,
|
||||
NUM_HEAD,
|
||||
NUM_LAYERS,
|
||||
norm_first=True,
|
||||
add_prenet=False,
|
||||
prefix_mode=PREFIX_MODE,
|
||||
share_embedding=True,
|
||||
nar_scale_factor=1.0,
|
||||
prepend_bos=True,
|
||||
num_quantizers=NUM_QUANTIZERS,
|
||||
).to(device)
|
||||
checkpoint = torch.load(pathlib.Path(vall_e_x_file_path), map_location='cpu')
|
||||
missing_keys, unexpected_keys = model.load_state_dict(
|
||||
checkpoint["model"], strict=True
|
||||
)
|
||||
|
||||
assert not missing_keys
|
||||
model.eval()
|
||||
# Encodec
|
||||
codec = AudioTokenizer(device)
|
||||
|
||||
vocos = None
|
||||
vocos_model = Vocos.from_hparams(vocos_folder_path / pathlib.Path("config.yaml"))
|
||||
state_dict = torch.load(vocos_folder_path / pathlib.Path("pytorch_model.bin"), map_location="cpu")
|
||||
if isinstance(vocos_model.feature_extractor, EncodecFeatures):
|
||||
encodec_parameters = {
|
||||
"feature_extractor.encodec." + key: value
|
||||
for key, value in vocos_model.feature_extractor.encodec.state_dict().items()
|
||||
}
|
||||
state_dict.update(encodec_parameters)
|
||||
|
||||
vocos_model.load_state_dict(state_dict)
|
||||
vocos_model.eval()
|
||||
vocos = vocos_model
|
||||
vocos.to(device)
|
||||
# TODO replace
|
||||
whisper_model_name = "medium"
|
||||
whisper_model = whisper.load_model(whisper_model_name,download_root=whisper_folder_path).cpu()
|
||||
|
||||
# DEPRECATED
|
||||
def load_models_from_mount(model_path:pathlib.Path,vocos_path:pathlib.Path,vall_e_x:pathlib.Path):
|
||||
global model, codec, vocos
|
||||
|
||||
@@ -131,13 +176,11 @@ def preload_models():
|
||||
vocos = Vocos.from_pretrained('charactr/vocos-encodec-24khz').to(device)
|
||||
|
||||
@torch.no_grad()
|
||||
def generate_audio(text,
|
||||
prompt=None,
|
||||
def generate_audio(text:str,
|
||||
prompt_name:pathlib.Path=None,
|
||||
prompt_dir:pathlib.Path=None,
|
||||
language='auto',
|
||||
accent='no-accent',
|
||||
mount_dir:pathlib.Path=pathlib.Path.cwd() / pathlib.Path("..") / pathlib.Path("Vall-E-mount"),
|
||||
presets_dir:pathlib.Path=pathlib.Path("presets"),
|
||||
prompt_dir:pathlib.Path=pathlib.Path("prompts")):
|
||||
accent='no-accent'):
|
||||
|
||||
global model, codec, vocos, text_tokenizer, text_collater
|
||||
text = text.replace("\n", "").strip(" ")
|
||||
@@ -149,22 +192,12 @@ def generate_audio(text,
|
||||
text = lang_token + text + lang_token
|
||||
|
||||
# load prompt
|
||||
if prompt is not None:
|
||||
prompt_path = prompt
|
||||
# if not os.path.exists(prompt_path):
|
||||
# prompt_path = "./presets/" + prompt + ".npz"
|
||||
# if not os.path.exists(prompt_path):
|
||||
# prompt_path = "./customs/" + prompt + ".npz"
|
||||
# check preset dir
|
||||
# mounted dirs
|
||||
if pathlib.Path.exists(mount_dir / presets_dir / pathlib.Path(f"{prompt}.npz")):
|
||||
prompt_path = mount_dir / presets_dir / pathlib.Path(f"{prompt}.npz")
|
||||
|
||||
if pathlib.Path.exists(mount_dir / prompt_dir / pathlib.Path(f"{prompt}.npz")):
|
||||
prompt_path = mount_dir / prompt_dir / pathlib.Path(f"{prompt}.npz")
|
||||
|
||||
if not os.path.exists(prompt_path):
|
||||
raise ValueError(f"Cannot find prompt {prompt}")
|
||||
if prompt_name is not None:
|
||||
prompt_path = prompt_name
|
||||
path = prompt_dir / pathlib.Path(f"{prompt_name}")
|
||||
print(f"Loading Prompt From Path: {path}")
|
||||
if pathlib.Path.exists(path):
|
||||
prompt_path = path
|
||||
|
||||
prompt_data = np.load(prompt_path)
|
||||
audio_prompts = prompt_data['audio_tokens']
|
||||
|
||||
@@ -27,7 +27,7 @@ if torch.cuda.is_available():
|
||||
codec = AudioTokenizer(device)
|
||||
|
||||
if not os.path.exists("./whisper/"): os.mkdir("./whisper/")
|
||||
whisper_model = None
|
||||
# whisper_model = None
|
||||
|
||||
@torch.no_grad()
|
||||
def transcribe_one(model, audio_path):
|
||||
@@ -54,7 +54,7 @@ def transcribe_one(model, audio_path):
|
||||
text_pr += "."
|
||||
return lang, text_pr
|
||||
|
||||
def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=None, transcript=None):
|
||||
def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=None):
|
||||
global model, text_collater, text_tokenizer, codec
|
||||
wav_pr, sr = torchaudio.load(audio_prompt_path)
|
||||
# check length
|
||||
@@ -62,7 +62,7 @@ def make_prompt(name, audio_prompt_path,audio_prompt_output_path:pathlib.Path=No
|
||||
raise ValueError(f"Prompt too long, expect length below 15 seconds, got {wav_pr / sr} seconds.")
|
||||
if wav_pr.size(0) == 2:
|
||||
wav_pr = wav_pr.mean(0, keepdim=True)
|
||||
text_pr, lang_pr = make_transcript(name, wav_pr, sr, transcript)
|
||||
text_pr, lang_pr = make_transcript(name, wav_pr, sr, None)
|
||||
|
||||
# tokenize audio
|
||||
encoded_frames = tokenize_audio(codec, (wav_pr, sr))
|
||||
|
||||
Reference in New Issue
Block a user