From 6c72eda84910085374171df809d3a7af121f17aa Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 15 Jun 2020 00:02:37 -0400 Subject: [PATCH] This seems to work for multi-speaker --- .gitmodules | 3 + .../glow-tts-multispeaker | 1 + .../logs/base/config.json | 55 ++++++++++ .../requirements.txt | 16 +++ .../wrap_glow_model.py | 103 ++++++++++++++++++ 5 files changed, 178 insertions(+) create mode 160000 glow-tts-jaywalnut-multispeaker/glow-tts-multispeaker create mode 100644 glow-tts-jaywalnut-multispeaker/logs/base/config.json create mode 100644 glow-tts-jaywalnut-multispeaker/requirements.txt create mode 100755 glow-tts-jaywalnut-multispeaker/wrap_glow_model.py diff --git a/.gitmodules b/.gitmodules index f2035c5..7dba103 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "glow-tts-jaywalnut/apex"] path = glow-tts-jaywalnut/apex url = https://github.com/NVIDIA/apex.git +[submodule "glow-tts-jaywalnut-multispeaker/glow-tts-multispeaker"] + path = glow-tts-jaywalnut-multispeaker/glow-tts-multispeaker + url = git@github.com:ml-applications/glow-tts-multispeaker.git diff --git a/glow-tts-jaywalnut-multispeaker/glow-tts-multispeaker b/glow-tts-jaywalnut-multispeaker/glow-tts-multispeaker new file mode 160000 index 0000000..e42b799 --- /dev/null +++ b/glow-tts-jaywalnut-multispeaker/glow-tts-multispeaker @@ -0,0 +1 @@ +Subproject commit e42b79911c95c7cab20fdba12b66d0618a3dc65e diff --git a/glow-tts-jaywalnut-multispeaker/logs/base/config.json b/glow-tts-jaywalnut-multispeaker/logs/base/config.json new file mode 100644 index 0000000..da11d6c --- /dev/null +++ b/glow-tts-jaywalnut-multispeaker/logs/base/config.json @@ -0,0 +1,55 @@ +{ + "train": { + "use_cuda": true, + "log_interval": 20, + "seed": 1234, + "epochs": 10000, + "learning_rate": 1e0, + "betas": [0.9, 0.98], + "eps": 1e-9, + "warmup_steps": 4000, + "scheduler": "noam", + "batch_size": 32, + "ddi": true, + "fp16_run": true + }, + "data": { + "load_mel_from_disk": false, + "training_files":"/home/bt/datasets/voicecurator_training/training.csv", + "validation_files":"/home/bt/datasets/voicecurator_training/validation.csv", + "text_cleaners":["english_cleaners"], + "max_wav_value": 32768.0, + "sampling_rate": 22050, + "filter_length": 1024, + "hop_length": 256, + "win_length": 1024, + "n_mel_channels": 80, + "mel_fmin": 0.0, + "mel_fmax": 8000.0, + "add_noise": true, + "add_space": false, + "cmudict_path": "data/cmu_dictionary" + }, + "model": { + "hidden_channels": 192, + "filter_channels": 768, + "filter_channels_dp": 256, + "kernel_size": 3, + "p_dropout": 0.1, + "n_blocks_dec": 12, + "n_layers_enc": 6, + "n_heads": 2, + "p_dropout_dec": 0.05, + "dilation_rate": 1, + "kernel_size_dec": 5, + "n_block_layers": 4, + "n_sqz": 2, + "prenet": true, + "mean_only": true, + "hidden_channels_enc": 192, + "hidden_channels_dec": 192, + "window_size": 4, + "n_speakers": 64, + "gin_channels": 256 + } +} diff --git a/glow-tts-jaywalnut-multispeaker/requirements.txt b/glow-tts-jaywalnut-multispeaker/requirements.txt new file mode 100644 index 0000000..2472624 --- /dev/null +++ b/glow-tts-jaywalnut-multispeaker/requirements.txt @@ -0,0 +1,16 @@ +torch==1.3.0 +librosa==0.7.1 +scipy==1.3.0 +PyYAML==5.2 +numpy==1.16.4 + +# To fix warnings: +numba==0.48.0 + +# For monotonic_align +# (cd monotonic_align; python setup.py build_ext --inplace) +Cython==0.29.12 + +# Other library needs +Unidecode==1.1.1 +inflect==4.1.0 diff --git a/glow-tts-jaywalnut-multispeaker/wrap_glow_model.py b/glow-tts-jaywalnut-multispeaker/wrap_glow_model.py new file mode 100755 index 0000000..adfc121 --- /dev/null +++ b/glow-tts-jaywalnut-multispeaker/wrap_glow_model.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# From: https://github.com/pytorch/pytorch/issues/20356#issuecomment-545572400 +import torch + +# Import glow-tts code +import sys +sys.path.append("./glow-tts-multispeaker") +import utils +import models +import commons +from text.symbols import symbols + +# Nvidia Apex +#from apex import amp +#from apex.parallel import DistributedDataParallel as DDP + +# Data +import numpy as np +from text import text_to_sequence, cmudict + +trace_model = False +if sys.argv[1] == 'true': + trace_model = True +elif sys.argv[1] == 'false': + trace_model = False +else: + raise Exception('Scripting or tracing?') + +torch.set_grad_enabled(False) + +def load(model_checkpoint_path): + model_dir = "./logs/base/" + hps = utils.get_hparams_from_dir(model_dir) + print('Checkpoint path: {}'.format(model_checkpoint_path)) + + with torch.no_grad(): + # NB(bt): This should fix the following: + # RuntimeError: Cannot insert a Tensor that requires grad as a constant. + model = models.FlowGenerator( + len(symbols), + out_channels=hps.data.n_mel_channels, + **hps.model) #.to("cuda") + + utils.load_checkpoint(model_checkpoint_path, model) + model.decoder.store_inverse() # do not calcuate jacobians for fast decoding + _ = model.eval() + + return (hps, model) + + +#model_checkpoint_path = '/home/bt/models/glow-tts-jaywalnut/pretrained.pth' +model_checkpoint_path = '/home/bt/models/glow-tts-jaywalnut-multispeaker/G_998.pth' + +hps, glow_model = load(model_checkpoint_path) +output_filename = 'glow_multispeaker.torchjit' + + +cmu_dict = cmudict.CMUDict(hps.data.cmudict_path) + +tst_stn = "This is a test of the machine learning model. How does this sound?" +seq = text_to_sequence(tst_stn, ['english_cleaners'], cmu_dict) +sequence = np.array(seq)[None, :] # NB: None adds a new axis. +print("".join([symbols[c] for c in sequence[0]])) + +torched = torch.from_numpy(sequence) + +x_tst = torch.autograd.Variable(torched).long() +x_tst_lengths = torch.tensor([x_tst.shape[1]]) + + +speaker_id = 1 +speaker_g = torch.IntTensor([int(speaker_id)]).long() + +print('\n tst_stn', len(tst_stn), tst_stn) +print('\n seq', len(seq), seq) +print('\n sequence', len(sequence), sequence.size, sequence) +print('\n torched', len(torched), torched.shape, torched) + +print('\n x_tst', len(x_tst), x_tst) +print('\n x_tst_lengths', len(x_tst_lengths), x_tst_lengths) + +print('\n speaker_g', len(speaker_g), speaker_g) + + +# NB: Tracing evaluates the model on input and unrolls and hardcodes branching +# and loops. Scripting allows these to remain by converting the entire program +# to TorchScript. Scripting seems MUCH faster, is also harder get working. +#trace_model = True # TODO: Try True. +if trace_model: + print('Tracing model and saving as: {}'.format(output_filename)) + #mel_file = '/home/bt/dev/voder/data/mels/LJ002-0320.mel' + #mel = torch.load(mel_file, map_location=torch.device('cpu')) + # NB: Getting `Tracing failed sanity checks! Graphs differed across invocations!` + inputs = (x_tst, x_tst_lengths, speaker_g) + #traced_script_module = torch.jit.trace(glow_model, inputs, check_trace=True) + traced_script_module = torch.jit.trace(glow_model, inputs, check_trace=False) + traced_script_module.save(output_filename) +else: + print('Saving script model as: {}'.format(output_filename)) + container = torch.jit.script(glow_model) + container.save(output_filename) + +