mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-09 00:09:55 +00:00
Merge pull request #48 from storytold/feature/updateTestHarness
updating test harness
This commit is contained in:
ArEnSc
@@ -1,5 +1,5 @@
|
||||
log_dir: "Models/LJSpeech"
|
||||
save_freq: 5
|
||||
log_dir: "Models/LibriTTS"
|
||||
save_freq: 1
|
||||
log_interval: 10
|
||||
device: "cuda"
|
||||
epochs: 50 # number of finetuning epoch (1 hour of data)
|
||||
|
||||
@@ -1,110 +0,0 @@
|
||||
log_dir: "Models/LJSpeech"
|
||||
save_freq: 5
|
||||
log_interval: 10
|
||||
device: "cuda"
|
||||
epochs: 50 # number of finetuning epoch (1 hour of data)
|
||||
batch_size: 2
|
||||
max_len: 1000 # maximum number of frames
|
||||
pretrained_model: "Models/LibriTTS/epochs_2nd_00020.pth"
|
||||
second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage
|
||||
load_only_params: true # set to true if do not want to load epoch numbers and optimizer parameters
|
||||
|
||||
F0_path: "Utils/JDC/bst.t7"
|
||||
ASR_config: "Utils/ASR/config.yml"
|
||||
ASR_path: "Utils/ASR/epoch_00080.pth"
|
||||
PLBERT_dir: 'Utils/PLBERT/'
|
||||
|
||||
data_params:
|
||||
train_data: "Data/train_harness.txt"
|
||||
val_data: "Data/val_harness.txt"
|
||||
root_path: ""
|
||||
OOD_data: "Data/OOD_harness.txt"
|
||||
min_length: 50 # sample until texts with this size are obtained for OOD texts
|
||||
|
||||
preprocess_params:
|
||||
sr: 24000
|
||||
spect_params:
|
||||
n_fft: 2048
|
||||
win_length: 1200
|
||||
hop_length: 300
|
||||
|
||||
model_params:
|
||||
multispeaker: true
|
||||
|
||||
dim_in: 64
|
||||
hidden_dim: 512
|
||||
max_conv_dim: 512
|
||||
n_layer: 3
|
||||
n_mels: 80
|
||||
|
||||
n_token: 178 # number of phoneme tokens
|
||||
max_dur: 50 # maximum duration of a single phoneme
|
||||
style_dim: 128 # style vector size
|
||||
|
||||
dropout: 0.2
|
||||
|
||||
# config for decoder
|
||||
decoder:
|
||||
type: 'hifigan' # either hifigan or istftnet
|
||||
resblock_kernel_sizes: [3,7,11]
|
||||
upsample_rates : [10,5,3,2]
|
||||
upsample_initial_channel: 512
|
||||
resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]]
|
||||
upsample_kernel_sizes: [20,10,6,4]
|
||||
|
||||
# speech language model config
|
||||
slm:
|
||||
model: 'microsoft/wavlm-base-plus'
|
||||
sr: 16000 # sampling rate of SLM
|
||||
hidden: 768 # hidden size of SLM
|
||||
nlayers: 13 # number of layers of SLM
|
||||
initial_channel: 64 # initial channels of SLM discriminator head
|
||||
|
||||
# style diffusion model config
|
||||
diffusion:
|
||||
embedding_mask_proba: 0.1
|
||||
# transformer config
|
||||
transformer:
|
||||
num_layers: 3
|
||||
num_heads: 8
|
||||
head_features: 64
|
||||
multiplier: 2
|
||||
|
||||
# diffusion distribution config
|
||||
dist:
|
||||
sigma_data: 0.2 # placeholder for estimate_sigma_data set to false
|
||||
estimate_sigma_data: true # estimate sigma_data from the current batch if set to true
|
||||
mean: -3.0
|
||||
std: 1.0
|
||||
|
||||
loss_params:
|
||||
lambda_mel: 5. # mel reconstruction loss
|
||||
lambda_gen: 1. # generator loss
|
||||
lambda_slm: 1. # slm feature matching loss
|
||||
|
||||
lambda_mono: 1. # monotonic alignment loss (TMA)
|
||||
lambda_s2s: 1. # sequence-to-sequence loss (TMA)
|
||||
|
||||
lambda_F0: 1. # F0 reconstruction loss
|
||||
lambda_norm: 1. # norm reconstruction loss
|
||||
lambda_dur: 1. # duration loss
|
||||
lambda_ce: 20. # duration predictor probability output CE loss
|
||||
lambda_sty: 1. # style reconstruction loss
|
||||
lambda_diff: 1. # score matching loss
|
||||
|
||||
diff_epoch: 10 # style diffusion starting epoch
|
||||
joint_epoch: 30 # joint training starting epoch
|
||||
|
||||
optimizer_params:
|
||||
lr: 0.0001 # general learning rate
|
||||
bert_lr: 0.00001 # learning rate for PLBERT
|
||||
ft_lr: 0.0001 # learning rate for acoustic modules
|
||||
|
||||
slmadv_params:
|
||||
min_len: 400 # minimum length of samples
|
||||
max_len: 500 # maximum length of samples
|
||||
batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size
|
||||
iter: 10 # update the discriminator every this iterations of generator update
|
||||
thresh: 5 # gradient norm above which the gradient is scaled
|
||||
scale: 0.01 # gradient scaling factor for predictors from SLM discriminators
|
||||
sig: 1.5 # sigma for differentiable duration modeling
|
||||
@@ -1,18 +0,0 @@
|
||||
INFO:2024-03-09 21:54:08,727: Epoch [1/50], Step [10/98], Loss: 0.61368, Disc Loss: 3.67060, Dur Loss: 1.90936, CE Loss: 0.14344, Norm Loss: 1.91490, F0 Loss: 6.29408, LM Loss: 2.80107, Gen Loss: 10.72666, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 9.37693, Mono Loss: 0.08869
|
||||
INFO:2024-03-09 21:54:33,761: Epoch [1/50], Step [20/98], Loss: 0.49030, Disc Loss: 3.78289, Dur Loss: 2.00487, CE Loss: 0.10148, Norm Loss: 2.45362, F0 Loss: 4.87550, LM Loss: 1.80477, Gen Loss: 7.66605, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.39594, Mono Loss: 0.08280
|
||||
INFO:2024-03-09 21:54:59,296: Epoch [1/50], Step [30/98], Loss: 0.53035, Disc Loss: 3.94322, Dur Loss: 1.44187, CE Loss: 0.08292, Norm Loss: 2.41949, F0 Loss: 4.68680, LM Loss: 2.01445, Gen Loss: 8.00111, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 6.87621, Mono Loss: 0.07573
|
||||
INFO:2024-03-09 21:55:26,019: Epoch [1/50], Step [40/98], Loss: 0.47501, Disc Loss: 3.79434, Dur Loss: 1.67735, CE Loss: 0.08687, Norm Loss: 2.38658, F0 Loss: 7.74768, LM Loss: 2.74964, Gen Loss: 9.62809, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.50795, Mono Loss: 0.08165
|
||||
INFO:2024-03-09 21:55:53,364: Epoch [1/50], Step [50/98], Loss: 0.54258, Disc Loss: 3.74476, Dur Loss: 1.51332, CE Loss: 0.07991, Norm Loss: 2.22287, F0 Loss: 5.13007, LM Loss: 2.02846, Gen Loss: 8.01354, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.92439, Mono Loss: 0.07453
|
||||
INFO:2024-03-09 21:56:27,086: Epoch [1/50], Step [60/98], Loss: 0.52682, Disc Loss: 3.82720, Dur Loss: 1.41939, CE Loss: 0.06902, Norm Loss: 2.05971, F0 Loss: 6.38216, LM Loss: 2.10802, Gen Loss: 9.07284, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.74938, Mono Loss: 0.07205
|
||||
INFO:2024-03-09 21:56:58,320: Epoch [1/50], Step [70/98], Loss: 0.48215, Disc Loss: 3.86537, Dur Loss: 1.74399, CE Loss: 0.12341, Norm Loss: 2.31109, F0 Loss: 4.19888, LM Loss: 1.80379, Gen Loss: 7.93936, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.08503, Mono Loss: 0.09210
|
||||
INFO:2024-03-09 21:57:27,673: Epoch [1/50], Step [80/98], Loss: 0.46574, Disc Loss: 4.00640, Dur Loss: 1.45382, CE Loss: 0.06951, Norm Loss: 1.50414, F0 Loss: 6.20810, LM Loss: 1.69506, Gen Loss: 10.20019, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 5.02205, Mono Loss: 0.09662
|
||||
INFO:2024-03-09 21:58:02,599: Epoch [1/50], Step [90/98], Loss: 0.49187, Disc Loss: 3.79479, Dur Loss: 1.75609, CE Loss: 0.09244, Norm Loss: 1.41511, F0 Loss: 6.44889, LM Loss: 1.73869, Gen Loss: 9.58825, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 4.80308, Mono Loss: 0.06435
|
||||
INFO:2024-03-09 21:58:49,772: Validation loss: 0.694, Dur loss: 1.956, F0 loss: 8.645
|
||||
|
||||
|
||||
|
||||
INFO:2024-03-09 22:02:37,284: Epoch [1/50], Step [10/98], Loss: 0.61610, Disc Loss: 3.64814, Dur Loss: 1.88536, CE Loss: 0.14313, Norm Loss: 1.93735, F0 Loss: 6.27854, LM Loss: 2.80288, Gen Loss: 10.86933, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 9.39963, Mono Loss: 0.08832
|
||||
INFO:2024-03-09 22:03:01,804: Epoch [1/50], Step [20/98], Loss: 0.48966, Disc Loss: 3.77456, Dur Loss: 2.00324, CE Loss: 0.09982, Norm Loss: 2.48450, F0 Loss: 5.00932, LM Loss: 1.82215, Gen Loss: 7.68481, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.41395, Mono Loss: 0.08262
|
||||
INFO:2024-03-09 22:03:27,043: Epoch [1/50], Step [30/98], Loss: 0.54742, Disc Loss: 3.96390, Dur Loss: 1.51634, CE Loss: 0.09582, Norm Loss: 3.07812, F0 Loss: 5.32871, LM Loss: 2.16857, Gen Loss: 8.18428, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 6.97294, Mono Loss: 0.07418
|
||||
INFO:2024-03-09 22:10:47,028: Epoch [1/50], Step [10/98], Loss: 0.61520, Disc Loss: 3.66078, Dur Loss: 1.88090, CE Loss: 0.14265, Norm Loss: 1.91994, F0 Loss: 6.26479, LM Loss: 2.79427, Gen Loss: 10.79336, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 9.40144, Mono Loss: 0.08800
|
||||
INFO:2024-03-09 22:11:11,800: Epoch [1/50], Step [20/98], Loss: 0.48881, Disc Loss: 3.78570, Dur Loss: 1.98345, CE Loss: 0.09998, Norm Loss: 2.51467, F0 Loss: 4.99696, LM Loss: 1.83913, Gen Loss: 7.60283, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 7.41842, Mono Loss: 0.08260
|
||||
Binary file not shown.
@@ -51,5 +51,4 @@
|
||||
/Ventress/186.wav|I sense the dark side in you, but you are too weak to embrace it. I tire of being your pawn.
|
||||
/YamiYugi/20.wav|This has created an opening! In a duel, no one should ever hold back. I will not hold back! Here I go!
|
||||
/YamiYugi/216.wav|Is it a new monster? Of course, if I can't master every monster. You're not going to give it to me for free?
|
||||
/YamiYugi/252.wav|I activate, Magical Hats! I activate, Chain Destruction! Big mistake! I'm sending your attack right back at you!
|
||||
/dasmr/698.wav |Didn't really have a body... I just kind of melted into shadows and creeped around on floors and walls, but now... I have this new job, and I just had to proove my self by bringing back a mortal soul.
|
||||
/YamiYugi/252.wav|I activate, Magical Hats! I activate, Chain Destruction! Big mistake! I'm sending your attack right back at you!
|
||||
@@ -58,15 +58,21 @@ class TestHarness:
|
||||
})
|
||||
|
||||
def sample_ood_voice(self,file:str="testing.wav",sample_rate=24000,character:str="character_name",caption:str="What is supposed to be said"):
|
||||
print(f"Uploading {file} {character} {caption}")
|
||||
wandb.log({character: wandb.Audio(file, sample_rate=sample_rate, caption=caption)})
|
||||
|
||||
def sample_pathes(self,audio_files):
|
||||
len(audio_files)
|
||||
wandb.log({"audio_samples":audio_files})
|
||||
|
||||
def _next_check_point_inference(self,file_input,text,checkpoint_name:str,callback):
|
||||
saved_path = callback(file_input,text,checkpoint_name)
|
||||
return saved_path
|
||||
|
||||
def test(self,check_point:Path,top=1):
|
||||
def test(self,check_point:Path,top=20):
|
||||
model = StyleTTS2(check_point)
|
||||
|
||||
sample_rate = 24000
|
||||
audio_objs = []
|
||||
char_name_ls = TestHarness.get_characters()
|
||||
count = 0
|
||||
for char_name in char_name_ls.keys():
|
||||
@@ -81,12 +87,14 @@ class TestHarness:
|
||||
diffusion_steps=10,alpha=0.45, beta=0.6, embedding_scale=1.0)
|
||||
stereo_wav = torch.tensor(mono_wav).repeat(2,1)
|
||||
path = f"./test/{char_name}|{i}-text|{sample.file_name}"
|
||||
torchaudio.save(path,stereo_wav,sample_rate=24000)
|
||||
self.sample_ood_voice(file=path,sample_rate=24000,character=char_name,caption=f"Text: {text} | Reference Text:{sample.ref_text}")
|
||||
|
||||
torchaudio.save(path,stereo_wav,sample_rate=sample_rate)
|
||||
audio_obj = wandb.Audio(path, sample_rate=sample_rate, caption=f"{char_name}|ref:{sample.ref_text}")
|
||||
audio_objs.append(audio_obj)
|
||||
count += 1
|
||||
if count == top:
|
||||
break
|
||||
self.sample_pathes(audio_objs)
|
||||
#self.sample_ood_voice(file=path,sample_rate=24000,character=char_name,caption=f"Text: {text} | Reference Text:{sample.ref_text}",epoch=epoch)
|
||||
@staticmethod
|
||||
def get_lines_from_file(file_path):
|
||||
with open(file_path,'r') as f:
|
||||
|
||||
@@ -29,7 +29,7 @@ from Modules.slmadv import SLMAdversarialLoss
|
||||
from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule
|
||||
|
||||
from optimizers import build_optimizer
|
||||
|
||||
from test_harness import TestHarness
|
||||
# simple fix for dataparallel that allows access to class attributes
|
||||
class MyDataParallel(torch.nn.DataParallel):
|
||||
def __getattr__(self, name):
|
||||
@@ -706,7 +706,7 @@ def main(config_path):
|
||||
}
|
||||
save_path = osp.join(log_dir, 'epoch_2nd_%05d.pth' % epoch)
|
||||
torch.save(state, save_path)
|
||||
|
||||
print(f"Testing Now: {save_path}")
|
||||
harness.test(save_path)
|
||||
# if estimate sigma, save the estimated simga
|
||||
if model_params.diffusion.dist.estimate_sigma_data:
|
||||
@@ -715,6 +715,11 @@ def main(config_path):
|
||||
with open(osp.join(log_dir, osp.basename(config_path)), 'w') as outfile:
|
||||
yaml.dump(config, outfile, default_flow_style=True)
|
||||
|
||||
|
||||
#main()
|
||||
|
||||
if __name__=="__main__":
|
||||
main()
|
||||
harness = TestHarness()
|
||||
config = yaml.safe_load(open('Configs/config_harness.yml'))
|
||||
log_dir = config['log_dir']
|
||||
save_path = osp.join(log_dir, 'epochs_2nd_%05d.pth' % 20)
|
||||
harness.test(save_path)
|
||||
Reference in New Issue
Block a user