diff --git a/voice_conversion/softvc/README.md b/voice_conversion/softvc/README.md index 0ae7a93..f7183ea 100644 --- a/voice_conversion/softvc/README.md +++ b/voice_conversion/softvc/README.md @@ -11,12 +11,16 @@ These are the repos contained herein: * https://github.com/bshall/acoustic-model * https://github.com/bshall/hifigan -And this is our initial notebook: - -* [Colab Notebook](https://colab.research.google.com/drive/1g0YsQQY8fzXHmZUWAMzQO60c6OUlyKBl#scrollTo=Sf_rjYw0P83I) - - +And this is our initial notebook (from Vegito): +* [SoftVCTraceAndOnnx.ipynb](https://colab.research.google.com/drive/1g0YsQQY8fzXHmZUWAMzQO60c6OUlyKBl#scrollTo=Sf_rjYw0P83I) + + +From Paul: + +* [softvc_pynb.ipynb](https://drive.google.com/file/d/1FzpL3Ze3X9ESt_nB4w9rTgnfGJxunha0/view?usp=share_link) +* [softvc_hifigan_fine_tuning.ipynb](https://drive.google.com/file/d/1X6Z5sLogpzTx8qEdbQ5Ub3N4U8_ZikPy/view?usp=share_link) +* [SoftVCTraceAndOnnx.ipynb](https://drive.google.com/file/d/13n-rFTqReBGJBQMB7N9nQClySZw8KzLV/view?usp=share_link) diff --git a/voice_conversion/softvc/cli_inference.py b/voice_conversion/softvc/cli_inference.py new file mode 100755 index 0000000..2147ebd --- /dev/null +++ b/voice_conversion/softvc/cli_inference.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +import IPython.display as display +import torch, torchaudio +import torch.nn as nn +import sys +import soundfile as sf + +from time import time +import torch.jit +sys.path.insert(0, './hubert/hubert') +from model import HubertSoft + +sys.path.insert(0, './acoustic-model/') +from acoustic.model import AcousticModel + +# =============================== + + +#@title select input wav +#file_name = "/content/drive/MyDrive/VCModels/Source/brandon1.wav"#@param { type: "string"} +file_name = "samples/source/easy/brandon1.wav" + +source, sr = torchaudio.load(f"{file_name}") +source = torchaudio.functional.resample(source, sr, 16000) +source = source.unsqueeze(0).cuda() +print(f"Wav Shape and Type {source.shape} {source.dtype}") +if source.shape[1] == 2: # remove channel + source = torch.mean(source, dim=0).unsqueeze(0) + print(f"Removing Channel to mono Wav Shape and Type {source.shape} {source.dtype}") + +display.Audio(source.squeeze().cpu(), rate=16000) + + +# =============================== + +import datetime +from utils_acoustic import AcousticModelScripted +from utils_hubert import modify_model + +hubert_soft = torch.hub.load("bshall/hubert:main", "hubert_soft").cuda() +hubert_soft.eval() + +script_acoustic = True #@param {type: "boolean"} +print(script_acoustic) + +# Load the acoustic model (either hubert_soft or hubert_discrete) +#input_acoustic_model_file = "/content/drive/MyDrive/VCModels/Padme/model-24000.pt" #@param {type: "string"} +input_acoustic_model_file = "/home/bt/models/voice-conversion/trump-acoustic.pt" +acoustic_dict = torch.load(input_acoustic_model_file) # this used ddp ## REPLACE THIS! <-- ! +if script_acoustic == True: + acoustic = AcousticModelScripted().cuda() +else: + acoustic = AcousticModel().cuda() + +acoustic.eval() + +new_state_dict = modify_model(state_dict=acoustic_dict) +acoustic.load_state_dict(new_state_dict) + +import importlib.util +import sys + +spec = importlib.util.spec_from_file_location("hifigan.utils", "/content/hifigan/hifigan/utils.py") +u = importlib.util.module_from_spec(spec) +sys.modules["hifigan.utils"] = u +spec.loader.exec_module(u) + +spec = importlib.util.spec_from_file_location("hifigan.generator", "/content/hifigan/hifigan/generator.py") +g = importlib.util.module_from_spec(spec) +sys.modules["hifigan.generator"] = g +spec.loader.exec_module(g) + +new_state_dict = modify_model(state_dict=acoustic_dict) +acoustic.load_state_dict(new_state_dict) +input_hifigan_checkpoint_path = "/content/drive/MyDrive/VCModels/Hifigan/Padme.pt" #@param {type: "string"} +if input_hifigan_checkpoint_path: + checkpoint = torch.load(input_hifigan_checkpoint_path)["generator"]["model"] + hifigan = g.HifiganGenerator() + consume_prefix_in_state_dict_if_present(checkpoint, "module.") + hifigan.load_state_dict(checkpoint) + hifigan.remove_weight_norm() +else: + hifigan = torch.hub.load("bshall/hifigan:main", "hifigan_hubert_soft") + +hifigan.cuda() +hifigan.eval() + +#utils.remove_weight_norm(hubert_soft.positional_embedding.conv) +# Convert to the target speaker +units = None +mel = None + +unit_start_time = 0.0 +unit_end_time = 0.0 + +acoustic_start_time = 0.0 +acoustic_end_time = 0.0 + +hifi_start_time = 0.0 +hifi_end_time = 0.0 + +with torch.inference_mode(): + unit_start_time = datetime.datetime.now() + units = hubert_soft.units(source) + print(f"Units and Type {units.shape} {units.dtype}") + unit_end_time = datetime.datetime.now() + + hubert_soft_script = torch.jit.trace_module(hubert_soft,{"units":source}) + torch.jit.save(hubert_soft_script, 'hubert_soft.jit') + + print(f"Unit Generation Time {(unit_end_time - unit_start_time).total_seconds() * 1000}") + + print("Hubert Soft Exported") + + acoustic_start_time = datetime.datetime.now() + mel = acoustic.generate(units).transpose(1, 2) + print(f"Mel and Type {mel.shape} {mel.dtype}") + acoustic_end_time = datetime.datetime.now() + print(f"Acoustic Time {(acoustic_end_time - acoustic_start_time).total_seconds() * 1000}") + if script_acoustic == True: + output_acoustic_scripted = '/content/drive/MyDrive/VCModels/FinishedModels/Padme.recast' #@param {type: "string"} + print("Exporting Scripted") + acoustic_model_script = torch.jit.script(acoustic) + torch.jit.save(acoustic_model_script, output_acoustic_scripted) + else: + print("Exporting Traced") + acoustic_model_script = torch.jit.trace_module(acoustic,{"generate":units}) + output_acoustic_traced = '' #@param {type: "string"} + torch.jit.save(acoustic_model_script, output_acoustic_traced) + + print("Acoustic Model Exported") + hifi_start_time = datetime.datetime.now() + + target = hifigan(mel) + print(f"Wavform and Type {target.shape} {target.dtype}") + hifi_end_time = datetime.datetime.now() + print(f"MEL {target.shape}") + print(f"Hifi Gan Time {(hifi_end_time - hifi_start_time).total_seconds() * 1000}") + + hifi_gan_script = torch.jit.trace_module(hifigan,{"forward":mel}) + output_hifigan_jit = '/content/drive/MyDrive/VCModels/FinishedModels/Padme-hifigan-jit.pt' #@param {type: "string"} + torch.jit.save(hifi_gan_script, output_hifigan_jit) + print("Hifi Gan Model Exported") + diff --git a/voice_conversion/softvc/requirements.txt b/voice_conversion/softvc/requirements.txt index db1630d..15ed540 100644 --- a/voice_conversion/softvc/requirements.txt +++ b/voice_conversion/softvc/requirements.txt @@ -1,24 +1,41 @@ +asttokens==2.2.1 +backcall==0.2.0 certifi==2022.12.7 cffi==1.15.1 charset-normalizer==2.1.1 contourpy==1.0.6 cycler==0.11.0 +decorator==5.1.1 +executing==1.2.0 fonttools==4.38.0 idna==3.4 +ipython==8.7.0 +jedi==0.18.2 kiwisolver==1.4.4 matplotlib==3.6.2 +matplotlib-inline==0.1.6 numpy==1.23.5 packaging==22.0 +parso==0.8.3 +pexpect==4.8.0 +pickleshare==0.7.5 Pillow==9.3.0 +prompt-toolkit==3.0.36 +ptyprocess==0.7.0 +pure-eval==0.2.2 pycparser==2.21 +Pygments==2.13.0 pyparsing==3.0.9 python-dateutil==2.8.2 requests==2.28.1 six==1.16.0 sklearn==0.0.post1 soundfile==0.11.0 +stack-data==0.6.2 torch==1.13.0+cu116 torchaudio==0.13.0+cu116 torchvision==0.14.0+cu116 +traitlets==5.6.0 typing_extensions==4.4.0 urllib3==1.26.13 +wcwidth==0.2.5 diff --git a/voice_conversion/softvc/utils_acoustic.py b/voice_conversion/softvc/utils_acoustic.py new file mode 100644 index 0000000..386c062 --- /dev/null +++ b/voice_conversion/softvc/utils_acoustic.py @@ -0,0 +1,113 @@ +import torch +import torch.nn as nn + +# @torch.jit.export +# def generate(self, x: torch.Tensor) -> torch.Tensor: +# x = self.encoder(x) +# return self.decoder.generate(x) +class AcousticModelScripted(nn.Module): + def __init__(self, discrete: bool = False, upsample: bool = True): + super().__init__() + self.encoder = Encoder(discrete, upsample) + self.decoder = Decoder() + + def forward(self, x: torch.Tensor, mels: torch.Tensor) -> torch.Tensor: + x = self.encoder(x) + return self.decoder(x, mels) + + @torch.jit.export + def generate(self, x: torch.Tensor) -> torch.Tensor: + x = self.encoder(x) + return self.decoder.generate(x) + + +class Encoder(nn.Module): + def __init__(self, discrete: bool = False, upsample: bool = True): + super().__init__() + self.embedding = nn.Embedding(100 + 1, 256) if discrete else None + self.prenet = PreNet(256, 256, 256) + self.convs = nn.Sequential( + nn.Conv1d(256, 512, 5, 1, 2), + nn.ReLU(), + nn.InstanceNorm1d(512), + nn.ConvTranspose1d(512, 512, 4, 2, 1) if upsample else nn.Identity(), + nn.Conv1d(512, 512, 5, 1, 2), + nn.ReLU(), + nn.InstanceNorm1d(512), + nn.Conv1d(512, 512, 5, 1, 2), + nn.ReLU(), + nn.InstanceNorm1d(512), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + if self.embedding is not None: + x = self.embedding(x) + x = self.prenet(x) + x = self.convs(x.transpose(1, 2)) + return x.transpose(1, 2) + + +class Decoder(nn.Module): + def __init__(self): + super().__init__() + self.prenet = PreNet(128, 256, 256) + self.lstm1 = nn.LSTM(512 + 256, 768, batch_first=True) + self.lstm2 = nn.LSTM(768, 768, batch_first=True) + self.lstm3 = nn.LSTM(768, 768, batch_first=True) + self.proj = nn.Linear(768, 128, bias=False) + + def forward(self, x: torch.Tensor, mels: torch.Tensor) -> torch.Tensor: + mels = self.prenet(mels) + x, _ = self.lstm1(torch.cat((x, mels), dim=-1)) + res = x + x, _ = self.lstm2(x) + x = res + x + res = x + x, _ = self.lstm3(x) + x = res + x + return self.proj(x) + + @torch.inference_mode() + def generate(self, xs: torch.Tensor) -> torch.Tensor: + m = torch.zeros(xs.size(0), 128, device=xs.device) + h1 = torch.zeros(1, xs.size(0), 768, device=xs.device) + c1 = torch.zeros(1, xs.size(0), 768, device=xs.device) + h2 = torch.zeros(1, xs.size(0), 768, device=xs.device) + c2 = torch.zeros(1, xs.size(0), 768, device=xs.device) + h3 = torch.zeros(1, xs.size(0), 768, device=xs.device) + c3 = torch.zeros(1, xs.size(0), 768, device=xs.device) + + mel = [] + for x in torch.unbind(xs, dim=1): + m = self.prenet(m) + x = torch.cat((x, m), dim=1).unsqueeze(1) + x1, (h1, c1) = self.lstm1(x, (h1, c1)) + x2, (h2, c2) = self.lstm2(x1, (h2, c2)) + x = x1 + x2 + x3, (h3, c3) = self.lstm3(x, (h3, c3)) + x = x + x3 + m = self.proj(x).squeeze(1) + mel.append(m) + return torch.stack(mel, dim=1) + + +class PreNet(nn.Module): + def __init__( + self, + input_size: int, + hidden_size: int, + output_size: int, + dropout: float = 0.5, + ): + super().__init__() + self.net = nn.Sequential( + nn.Linear(input_size, hidden_size), + nn.ReLU(), + nn.Dropout(dropout), + nn.Linear(hidden_size, output_size), + nn.ReLU(), + nn.Dropout(dropout), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.net(x) diff --git a/voice_conversion/softvc/utils_hubert.py b/voice_conversion/softvc/utils_hubert.py new file mode 100644 index 0000000..40477e1 --- /dev/null +++ b/voice_conversion/softvc/utils_hubert.py @@ -0,0 +1,36 @@ +# Torch Script Inference Hubert +import copy +from typing import Optional, Tuple +import random + +from sklearn.cluster import KMeans + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present +import torch.nn.utils as utils + +from torch.nn.utils.weight_norm import WeightNorm +from typing import Dict, Any +from collections import OrderedDict + +def remove_weight_norm(module): + module_list = [mod for mod in module.children()] + if len(module_list) == 0: + for k, hook in module._forward_pre_hooks.items(): + if isinstance(hook, WeightNorm): + hook.remove(module) + del module._forward_pre_hooks[k] + else: + for mod in module_list: + remove_weight_norm(mod) + + +def modify_model(state_dict:Dict[str,Any]): + new_state_dict = OrderedDict() + for k in state_dict['acoustic-model'].keys(): + value = state_dict['acoustic-model'][k] + name = k[7:] # remove `module.` because they used ddp training to save + new_state_dict[name] = value + return new_state_dict