diff --git a/tts/StyleTTS2TrainOG/Configs/config_harness.yml b/tts/StyleTTS2TrainOG/Configs/config_harness.yml index c3e29a0..06ecf81 100644 --- a/tts/StyleTTS2TrainOG/Configs/config_harness.yml +++ b/tts/StyleTTS2TrainOG/Configs/config_harness.yml @@ -1,5 +1,5 @@ -log_dir: "Models/LJSpeech" -save_freq: 5 +log_dir: "Models/LibriTTS" +save_freq: 1 log_interval: 10 device: "cuda" epochs: 50 # number of finetuning epoch (1 hour of data) diff --git a/tts/StyleTTS2TrainOG/Models/LJSpeech/config_harness.yml b/tts/StyleTTS2TrainOG/Models/LJSpeech/config_harness.yml deleted file mode 100644 index 1a264ae..0000000 --- a/tts/StyleTTS2TrainOG/Models/LJSpeech/config_harness.yml +++ /dev/null @@ -1,110 +0,0 @@ -log_dir: "Models/LJSpeech" -save_freq: 5 -log_interval: 10 -device: "cuda" -epochs: 50 # number of finetuning epoch (1 hour of data) -batch_size: 2 -max_len: 1000 # maximum number of frames -pretrained_model: "Models/LibriTTS/epochs_2nd_00020.pth" -second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage -load_only_params: true # set to true if do not want to load epoch numbers and optimizer parameters - -F0_path: "Utils/JDC/bst.t7" -ASR_config: "Utils/ASR/config.yml" -ASR_path: "Utils/ASR/epoch_00080.pth" -PLBERT_dir: 'Utils/PLBERT/' - -data_params: - train_data: "Data/train_harness.txt" - val_data: "Data/val_harness.txt" - root_path: "" - OOD_data: "Data/OOD_harness.txt" - min_length: 50 # sample until texts with this size are obtained for OOD texts - -preprocess_params: - sr: 24000 - spect_params: - n_fft: 2048 - win_length: 1200 - hop_length: 300 - -model_params: - multispeaker: true - - dim_in: 64 - hidden_dim: 512 - max_conv_dim: 512 - n_layer: 3 - n_mels: 80 - - n_token: 178 # number of phoneme tokens - max_dur: 50 # maximum duration of a single phoneme - style_dim: 128 # style vector size - - dropout: 0.2 - - # config for decoder - decoder: - type: 'hifigan' # either hifigan or istftnet - resblock_kernel_sizes: [3,7,11] - upsample_rates : [10,5,3,2] - upsample_initial_channel: 512 - resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]] - upsample_kernel_sizes: [20,10,6,4] - - # speech language model config - slm: - model: 'microsoft/wavlm-base-plus' - sr: 16000 # sampling rate of SLM - hidden: 768 # hidden size of SLM - nlayers: 13 # number of layers of SLM - initial_channel: 64 # initial channels of SLM discriminator head - - # style diffusion model config - diffusion: - embedding_mask_proba: 0.1 - # transformer config - transformer: - num_layers: 3 - num_heads: 8 - head_features: 64 - multiplier: 2 - - # diffusion distribution config - dist: - sigma_data: 0.2 # placeholder for estimate_sigma_data set to false - estimate_sigma_data: true # estimate sigma_data from the current batch if set to true - mean: -3.0 - std: 1.0 - -loss_params: - lambda_mel: 5. # mel reconstruction loss - lambda_gen: 1. # generator loss - lambda_slm: 1. # slm feature matching loss - - lambda_mono: 1. # monotonic alignment loss (TMA) - lambda_s2s: 1. # sequence-to-sequence loss (TMA) - - lambda_F0: 1. # F0 reconstruction loss - lambda_norm: 1. # norm reconstruction loss - lambda_dur: 1. # duration loss - lambda_ce: 20. # duration predictor probability output CE loss - lambda_sty: 1. # style reconstruction loss - lambda_diff: 1. # score matching loss - - diff_epoch: 10 # style diffusion starting epoch - joint_epoch: 30 # joint training starting epoch - -optimizer_params: - lr: 0.0001 # general learning rate - bert_lr: 0.00001 # learning rate for PLBERT - ft_lr: 0.0001 # learning rate for acoustic modules - -slmadv_params: - min_len: 400 # minimum length of samples - max_len: 500 # maximum length of samples - batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size - iter: 10 # update the discriminator every this iterations of generator update - thresh: 5 # gradient norm above which the gradient is scaled - scale: 0.01 # gradient scaling factor for predictors from SLM discriminators - sig: 1.5 # sigma for differentiable duration modeling \ No newline at end of file diff --git a/tts/StyleTTS2TrainOG/Models/LJSpeech/train.log b/tts/StyleTTS2TrainOG/Models/LJSpeech/train.log deleted file mode 100644 index ea2aa23..0000000 --- a/tts/StyleTTS2TrainOG/Models/LJSpeech/train.log +++ /dev/null @@ -1,18 +0,0 @@ -INFO:2024-03-09 21:54:08,727: Epoch [1/50], Step [10/98], Loss: 0.61368, Disc Loss: 3.67060, Dur Loss: 1.90936, CE Loss: 0.14344, Norm Loss: 1.91490, F0 Loss: 6.29408, LM Loss: 2.80107, Gen Loss: 10.72666, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 9.37693, Mono Loss: 0.08869 -INFO:2024-03-09 21:54:33,761: Epoch [1/50], Step [20/98], Loss: 0.49030, Disc Loss: 3.78289, Dur Loss: 2.00487, CE Loss: 0.10148, Norm Loss: 2.45362, F0 Loss: 4.87550, LM Loss: 1.80477, Gen Loss: 7.66605, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.39594, Mono Loss: 0.08280 -INFO:2024-03-09 21:54:59,296: Epoch [1/50], Step [30/98], Loss: 0.53035, Disc Loss: 3.94322, Dur Loss: 1.44187, CE Loss: 0.08292, Norm Loss: 2.41949, F0 Loss: 4.68680, LM Loss: 2.01445, Gen Loss: 8.00111, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 6.87621, Mono Loss: 0.07573 -INFO:2024-03-09 21:55:26,019: Epoch [1/50], Step [40/98], Loss: 0.47501, Disc Loss: 3.79434, Dur Loss: 1.67735, CE Loss: 0.08687, Norm Loss: 2.38658, F0 Loss: 7.74768, LM Loss: 2.74964, Gen Loss: 9.62809, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.50795, Mono Loss: 0.08165 -INFO:2024-03-09 21:55:53,364: Epoch [1/50], Step [50/98], Loss: 0.54258, Disc Loss: 3.74476, Dur Loss: 1.51332, CE Loss: 0.07991, Norm Loss: 2.22287, F0 Loss: 5.13007, LM Loss: 2.02846, Gen Loss: 8.01354, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.92439, Mono Loss: 0.07453 -INFO:2024-03-09 21:56:27,086: Epoch [1/50], Step [60/98], Loss: 0.52682, Disc Loss: 3.82720, Dur Loss: 1.41939, CE Loss: 0.06902, Norm Loss: 2.05971, F0 Loss: 6.38216, LM Loss: 2.10802, Gen Loss: 9.07284, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.74938, Mono Loss: 0.07205 -INFO:2024-03-09 21:56:58,320: Epoch [1/50], Step [70/98], Loss: 0.48215, Disc Loss: 3.86537, Dur Loss: 1.74399, CE Loss: 0.12341, Norm Loss: 2.31109, F0 Loss: 4.19888, LM Loss: 1.80379, Gen Loss: 7.93936, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.08503, Mono Loss: 0.09210 -INFO:2024-03-09 21:57:27,673: Epoch [1/50], Step [80/98], Loss: 0.46574, Disc Loss: 4.00640, Dur Loss: 1.45382, CE Loss: 0.06951, Norm Loss: 1.50414, F0 Loss: 6.20810, LM Loss: 1.69506, Gen Loss: 10.20019, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.02205, Mono Loss: 0.09662 -INFO:2024-03-09 21:58:02,599: Epoch [1/50], Step [90/98], Loss: 0.49187, Disc Loss: 3.79479, Dur Loss: 1.75609, CE Loss: 0.09244, Norm Loss: 1.41511, F0 Loss: 6.44889, LM Loss: 1.73869, Gen Loss: 9.58825, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 4.80308, Mono Loss: 0.06435 -INFO:2024-03-09 21:58:49,772: Validation loss: 0.694, Dur loss: 1.956, F0 loss: 8.645 - - - -INFO:2024-03-09 22:02:37,284: Epoch [1/50], Step [10/98], Loss: 0.61610, Disc Loss: 3.64814, Dur Loss: 1.88536, CE Loss: 0.14313, Norm Loss: 1.93735, F0 Loss: 6.27854, LM Loss: 2.80288, Gen Loss: 10.86933, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 9.39963, Mono Loss: 0.08832 -INFO:2024-03-09 22:03:01,804: Epoch [1/50], Step [20/98], Loss: 0.48966, Disc Loss: 3.77456, Dur Loss: 2.00324, CE Loss: 0.09982, Norm Loss: 2.48450, F0 Loss: 5.00932, LM Loss: 1.82215, Gen Loss: 7.68481, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.41395, Mono Loss: 0.08262 -INFO:2024-03-09 22:03:27,043: Epoch [1/50], Step [30/98], Loss: 0.54742, Disc Loss: 3.96390, Dur Loss: 1.51634, CE Loss: 0.09582, Norm Loss: 3.07812, F0 Loss: 5.32871, LM Loss: 2.16857, Gen Loss: 8.18428, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 6.97294, Mono Loss: 0.07418 -INFO:2024-03-09 22:10:47,028: Epoch [1/50], Step [10/98], Loss: 0.61520, Disc Loss: 3.66078, Dur Loss: 1.88090, CE Loss: 0.14265, Norm Loss: 1.91994, F0 Loss: 6.26479, LM Loss: 2.79427, Gen Loss: 10.79336, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 9.40144, Mono Loss: 0.08800 -INFO:2024-03-09 22:11:11,800: Epoch [1/50], Step [20/98], Loss: 0.48881, Disc Loss: 3.78570, Dur Loss: 1.98345, CE Loss: 0.09998, Norm Loss: 2.51467, F0 Loss: 4.99696, LM Loss: 1.83913, Gen Loss: 7.60283, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.41842, Mono Loss: 0.08260 diff --git a/tts/StyleTTS2TrainOG/TestHarnessV2/dasmr/698.wav b/tts/StyleTTS2TrainOG/TestHarnessV2/dasmr/698.wav deleted file mode 100644 index 2718c24..0000000 Binary files a/tts/StyleTTS2TrainOG/TestHarnessV2/dasmr/698.wav and /dev/null differ diff --git a/tts/StyleTTS2TrainOG/TestHarnessV2/info.txt b/tts/StyleTTS2TrainOG/TestHarnessV2/info.txt index 96dddb4..8e03a38 100644 --- a/tts/StyleTTS2TrainOG/TestHarnessV2/info.txt +++ b/tts/StyleTTS2TrainOG/TestHarnessV2/info.txt @@ -51,5 +51,4 @@ /Ventress/186.wav|I sense the dark side in you, but you are too weak to embrace it. I tire of being your pawn. /YamiYugi/20.wav|This has created an opening! In a duel, no one should ever hold back. I will not hold back! Here I go! /YamiYugi/216.wav|Is it a new monster? Of course, if I can't master every monster. You're not going to give it to me for free? -/YamiYugi/252.wav|I activate, Magical Hats! I activate, Chain Destruction! Big mistake! I'm sending your attack right back at you! -/dasmr/698.wav |Didn't really have a body... I just kind of melted into shadows and creeped around on floors and walls, but now... I have this new job, and I just had to proove my self by bringing back a mortal soul. \ No newline at end of file +/YamiYugi/252.wav|I activate, Magical Hats! I activate, Chain Destruction! Big mistake! I'm sending your attack right back at you! \ No newline at end of file diff --git a/tts/StyleTTS2TrainOG/test_harness.py b/tts/StyleTTS2TrainOG/test_harness.py index fe71906..398be1a 100644 --- a/tts/StyleTTS2TrainOG/test_harness.py +++ b/tts/StyleTTS2TrainOG/test_harness.py @@ -58,15 +58,21 @@ class TestHarness: }) def sample_ood_voice(self,file:str="testing.wav",sample_rate=24000,character:str="character_name",caption:str="What is supposed to be said"): + print(f"Uploading {file} {character} {caption}") wandb.log({character: wandb.Audio(file, sample_rate=sample_rate, caption=caption)}) + + def sample_pathes(self,audio_files): + len(audio_files) + wandb.log({"audio_samples":audio_files}) def _next_check_point_inference(self,file_input,text,checkpoint_name:str,callback): saved_path = callback(file_input,text,checkpoint_name) return saved_path - def test(self,check_point:Path,top=1): + def test(self,check_point:Path,top=20): model = StyleTTS2(check_point) - + sample_rate = 24000 + audio_objs = [] char_name_ls = TestHarness.get_characters() count = 0 for char_name in char_name_ls.keys(): @@ -81,12 +87,14 @@ class TestHarness: diffusion_steps=10,alpha=0.45, beta=0.6, embedding_scale=1.0) stereo_wav = torch.tensor(mono_wav).repeat(2,1) path = f"./test/{char_name}|{i}-text|{sample.file_name}" - torchaudio.save(path,stereo_wav,sample_rate=24000) - self.sample_ood_voice(file=path,sample_rate=24000,character=char_name,caption=f"Text: {text} | Reference Text:{sample.ref_text}") - + torchaudio.save(path,stereo_wav,sample_rate=sample_rate) + audio_obj = wandb.Audio(path, sample_rate=sample_rate, caption=f"{char_name}|ref:{sample.ref_text}") + audio_objs.append(audio_obj) count += 1 if count == top: break + self.sample_pathes(audio_objs) + #self.sample_ood_voice(file=path,sample_rate=24000,character=char_name,caption=f"Text: {text} | Reference Text:{sample.ref_text}",epoch=epoch) @staticmethod def get_lines_from_file(file_path): with open(file_path,'r') as f: diff --git a/tts/StyleTTS2TrainOG/train_finetune.py b/tts/StyleTTS2TrainOG/train_finetune.py index 21b053a..adde130 100644 --- a/tts/StyleTTS2TrainOG/train_finetune.py +++ b/tts/StyleTTS2TrainOG/train_finetune.py @@ -29,7 +29,7 @@ from Modules.slmadv import SLMAdversarialLoss from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule from optimizers import build_optimizer - +from test_harness import TestHarness # simple fix for dataparallel that allows access to class attributes class MyDataParallel(torch.nn.DataParallel): def __getattr__(self, name): @@ -706,7 +706,7 @@ def main(config_path): } save_path = osp.join(log_dir, 'epoch_2nd_%05d.pth' % epoch) torch.save(state, save_path) - + print(f"Testing Now: {save_path}") harness.test(save_path) # if estimate sigma, save the estimated simga if model_params.diffusion.dist.estimate_sigma_data: @@ -715,6 +715,11 @@ def main(config_path): with open(osp.join(log_dir, osp.basename(config_path)), 'w') as outfile: yaml.dump(config, outfile, default_flow_style=True) - + #main() + if __name__=="__main__": - main() + harness = TestHarness() + config = yaml.safe_load(open('Configs/config_harness.yml')) + log_dir = config['log_dir'] + save_path = osp.join(log_dir, 'epochs_2nd_%05d.pth' % 20) + harness.test(save_path) \ No newline at end of file