storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-12 23:48:57 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
7053eec79ca337fed1c64dbfabdc37e1c052d86e
storyteller-ml/tts/VALL-E-X/main.py
T

254 lines
13 KiB
Python
Executable File
Raw Blame History

import argparse
import sys
from typing import List
import pathlib
# https://discuss.pytorch.org/t/how-to-calculate-the-gpu-memory-that-a-model-uses/157486/6 <-- for fine tuning memory usage later.
from utils.prompt_making import make_prompt
import logging
from utils.generation import SAMPLE_RATE, generate_audio,load_models_from_pathes
import soundfile as sf
import numpy as np
import torchaudio
from typing import List
import uuid
import torchaudio.transforms as T
class AudioOps():
@staticmethod
def stitch_wav_tensors_with_crossfade(wav_tensor1:np.array,wav_tensor2:np.array,sample_rate:int):
# Load two audio signals as NumPy tensors
# Compute the length of the cross-fade window in samples
fade_len = int(0.01 * sample_rate)
# Create a cross-fade window
fade_window = np.hanning(2*fade_len)
# Concatenate the signals with cross-fade
audio_out = np.concatenate((wav_tensor1[:-fade_len],
wav_tensor1[-fade_len:] * fade_window[:fade_len] + wav_tensor2[:fade_len] * fade_window[fade_len:],
wav_tensor2[fade_len:]))
return audio_out
@staticmethod
def cut_audio_range(file_path, start_seconds, end_seconds, output_path="cut_audio.wav"):
"""
Cuts out a specific segment from an audio file and saves the result.
Args:
- file_path (str): Path to the audio file from which the segment will be cut.
- start_seconds (float): Start time of the segment to be cut out in seconds.
- end_seconds (float): End time of the segment to be cut out in seconds.
- output_path (str): Path to save the resulting audio. Defaults to "cut_audio.wav".
Returns:
- None
"""
# Load the audio file
waveform, sr = torchaudio.load(file_path)
# Convert the range to sample indices
start_sample = int(start_seconds * sr)
end_sample = int(end_seconds * sr)
cut_tensor = waveform[:,start_sample:end_sample]
torchaudio.save(output_path, cut_tensor, sr)
@staticmethod
def stitch_wav_files_resample(files:List[str],target_sample_rate:int=SAMPLE_RATE):
resampler = None
result = None
for file in files:
if type(result) == type(None):
tensor, sr = torchaudio.load(pathlib.Path(file))
resampler = T.Resample(orig_freq=sr,new_freq=target_sample_rate)
resampled_tensor = resampler(tensor)
result = resampled_tensor.squeeze()
result = np.array(result)
else:
tensor, _ = torchaudio.load(pathlib.Path(file))
resampled_tensor = resampler(tensor)
resampled_tensor = resampled_tensor.squeeze()
resampled_tensor = np.array(resampled_tensor)
result = AudioOps.stitch_wav_tensors_with_crossfade(result,resampled_tensor,sample_rate=target_sample_rate)
return result
class VoiceDesigner():
def __init__(self) -> None:
# load models from host system.
self.logger = self.setup_logger()
# host path for work
self.temp_path = None
def tts_with_prompt(self, prompt_dir:pathlib.Path,
prompt_name:pathlib.Path,
audio_output_path:pathlib.Path,
output_file_name:pathlib.Path,
text:str):
audio_array = generate_audio(text,prompt_name=prompt_name,prompt_dir=prompt_dir)
# save audio to disk
sf.write(audio_output_path / output_file_name, audio_array, SAMPLE_RATE)
def initialize_voice_designer(self,vall_e_x_file_path:pathlib.Path,vocos_folder_path:pathlib.Path,whisper_folder_path:pathlib.Path):
load_models_from_pathes(vocos_folder_path=vocos_folder_path,vall_e_x_file_path=vall_e_x_file_path,whisper_folder_path=whisper_folder_path)
def create_prompt(self,audio_file_paths:List[pathlib.Path],
prompt_file_name:pathlib.Path,
prompt_output_path:pathlib.Path,
whisper_folder_path:pathlib.Path):
# multiple audio files concat
# concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways
if len(audio_file_paths) > 0:
# try:# ./customs/ <-- folder it adds by default until we modified.
# load the audio files and then concat them.
tensor_result = AudioOps.stitch_wav_files_resample(files=audio_file_paths)
# save to disk
generated_name = f"{uuid.uuid4()}"
concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav")
sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE)
# save result into processing...
# trim to 15 seconds.
audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav")
AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path)
make_prompt(name=prompt_file_name,audio_prompt_output_path=prompt_output_path,audio_prompt_path=audio_cut_path,whisper_folder_path=whisper_folder_path)
else:
self.logger.debug(f"No audio files to load")
def setup_logger(self):
# Create a logger object
logger = logging.getLogger(__name__)
# Set the log level
logger.setLevel(logging.DEBUG)
# Create a file handler to log messages to a file
file_handler = logging.FileHandler('docker_valle-x.log')
file_handler.setLevel(logging.DEBUG)
# Create a console handler to log messages to the console
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.DEBUG)
# Create a formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# Set the formatter for the handlers
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
# Add the handlers to the logger
logger.addHandler(file_handler)
logger.addHandler(console_handler)
return logger
global voice_designer
voice_designer = VoiceDesigner()
def main(args):
print("Starting Vall-E-X")
print(f"Python Version: {sys.version_info}")
print("Starting Inference on Vall-E-X With Inputs")
print(f"Mode: {args.mode}")
print(f"Text: {args.text}")
print(f"List of Files For Create: {args.audio_wav_files}")
print(f"Whisper Folder Path: {args.whisper_folder_path}")
print(f"Vocos Folder Path: {args.vocos_folder_path}")
print(f"Vall-E-X Path: {args.vallex_path}")
print(f"Prompt Path: {args.prompt_path}")
print(f"Prompt Name: {args.prompt_name}")
print(f"Audio Name: {args.audio_name}")
print(f"Audio Path: {args.audio_path}")
print(f"Temp Work Dir: {args.tmp_work_dir}")
voice_designer.temp_path = pathlib.Path(args.tmp_work_dir)
voice_designer.initialize_voice_designer(pathlib.Path(args.vallex_path),
pathlib.Path(args.vocos_folder_path),
pathlib.Path(args.whisper_folder_path))
if args.mode == 0: # run inference
print("Running Inference")
voice_designer.tts_with_prompt(prompt_dir=pathlib.Path(args.prompt_path),
prompt_name=pathlib.Path(args.prompt_name),
audio_output_path=pathlib.Path(args.audio_path),
output_file_name=pathlib.Path(args.audio_name),
text=args.text)
elif args.mode == 1: # run create voice
voice_designer.create_prompt(audio_file_paths=args.audio_wav_files,
prompt_output_path=pathlib.Path(args.prompt_path),
prompt_file_name=pathlib.Path(args.prompt_name),
whisper_folder_path=pathlib.Path(args.whisper_folder_path))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.")
# modes
parser.add_argument('--mode', metavar='model mode',type=int,help='Mode 0 - (Inference) | Mode 1 - (Create)')
parser.add_argument('--text', metavar='text to synthesize', type=str,help='Text to be converted to speech.')
# inputs
parser.add_argument('--audio-wav-files', metavar='wav to voice clone', type=str,nargs='+',help='Takes in a list of .wav files for an audio prompt provide the absolute path. Must be less than 15 seconds otherwise will error out. (Create Voice Only)')
# model pathes
parser.add_argument("--whisper-folder-path",metavar="abs whisper folder path",type=str,help="Path to the Whisper folder where the .pt file")
parser.add_argument("--whisper-model",metavar="model large or medium",type=str,help="medium or large")
parser.add_argument("--vocos-folder-path",metavar="abs vocos model path",type=str,help="Path to vocos codec Model requires the (folder path)")
parser.add_argument("--vallex-path",metavar="abs vall-e model path",type=str,help="Path to the Valle Model File")
# output and input pathes
parser.add_argument("--audio-name",metavar="abs output path",type=str,help="Audio Output Saved Name (Inference)")
parser.add_argument("--audio-path",metavar="abs path to prompt",type=str,help="Audio Output Path (Inference)")
parser.add_argument("--prompt-path",metavar="abs path to prompt",type=str,help="Prompt Embedding Path (Inference and Create Voice)")
parser.add_argument("--prompt-name",metavar="abs path to prompt",type=str,help="Prompt Name (Inference and Create Voice)")
parser.add_argument("--tmp-work-dir",metavar="abs tmp dir",type=str,help="Temporary work directory in host system.")
args = parser.parse_args()
main(args)
# inference example
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --text "hello world" --mode 0 --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "goku.npz" --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/" --audio-name "hello_world_test.wav" --tmp-work-dir "/tmp"
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py
# --text "hello world"
# --mode 0
# --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper"
# --whisper-model medium
# --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz"
# --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt"
# --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts"
# --prompt-name "goku"
# --audio-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/output/"
# --audio-name "hello_world_test.wav"
# --tmp-work-dir "/tmp"
# create voice example
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python /home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py --mode 1 --audio-wav-files "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav" "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav" --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper" --whisper-model medium --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz" --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt" --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts" --prompt-name "test_prompt" --tmp-work-dir "/tmp"
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/.venv/bin/python
#/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/main.py
# --mode 1
# --audio-wav-files
# "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/20.wav"
# "/home/tensor/code/TTSDockerContainer/Vall-E-mount/input/21.wav"
# --whisper-folder-path "/home/tensor/code/storyteller/storyteller-ml/tts/VALL-E-X/whisper"
# --whisper-model medium
# --vocos-folder-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vocos-encodec-24khz"
# --vallex-path "/home/tensor/code/TTSDockerContainer/Vall-E-mount/models/vallex-checkpoint.pt"
# --prompt-path "/home/tensor/code/storyteller/VALL-E-X-TTS-Container/Vall-E-mount/prompts"
# --prompt-name "test_prompt"
# --tmp-work-dir "/tmp"
Reference in New Issue View Git Blame Copy Permalink