WIP inference

voice_conversion/softvc/README.md

@@ -11,12 +11,16 @@ These are the repos contained herein:
 * https://github.com/bshall/acoustic-model
 * https://github.com/bshall/hifigan
 
-And this is our initial notebook: 
+And this is our initial notebook (from Vegito): 
-
-* [Colab Notebook](https://colab.research.google.com/drive/1g0YsQQY8fzXHmZUWAMzQO60c6OUlyKBl#scrollTo=Sf_rjYw0P83I)
-
-
 
+* [SoftVCTraceAndOnnx.ipynb](https://colab.research.google.com/drive/1g0YsQQY8fzXHmZUWAMzQO60c6OUlyKBl#scrollTo=Sf_rjYw0P83I)
+
+
+From Paul: 
+
+* [softvc_pynb.ipynb](https://drive.google.com/file/d/1FzpL3Ze3X9ESt_nB4w9rTgnfGJxunha0/view?usp=share_link)
+* [softvc_hifigan_fine_tuning.ipynb](https://drive.google.com/file/d/1X6Z5sLogpzTx8qEdbQ5Ub3N4U8_ZikPy/view?usp=share_link)
+* [SoftVCTraceAndOnnx.ipynb](https://drive.google.com/file/d/13n-rFTqReBGJBQMB7N9nQClySZw8KzLV/view?usp=share_link)
 
 
 

voice_conversion/softvc/cli_inference.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+
+import IPython.display as display
+import torch, torchaudio
+import torch.nn as nn
+import sys  
+import soundfile as sf
+
+from time import time
+import torch.jit
+sys.path.insert(0, './hubert/hubert')
+from model import HubertSoft
+
+sys.path.insert(0, './acoustic-model/')
+from acoustic.model import AcousticModel
+
+# ===============================
+
+
+#@title select input wav
+#file_name = "/content/drive/MyDrive/VCModels/Source/brandon1.wav"#@param { type: "string"}
+file_name = "samples/source/easy/brandon1.wav"
+
+source, sr = torchaudio.load(f"{file_name}")
+source = torchaudio.functional.resample(source, sr, 16000)
+source = source.unsqueeze(0).cuda()
+print(f"Wav Shape and Type {source.shape} {source.dtype}")
+if source.shape[1] == 2: # remove channel
+    source = torch.mean(source, dim=0).unsqueeze(0)
+    print(f"Removing Channel to mono Wav Shape and Type {source.shape} {source.dtype}")
+
+display.Audio(source.squeeze().cpu(), rate=16000)
+
+
+# ===============================
+
+import datetime
+from utils_acoustic import AcousticModelScripted
+from utils_hubert import modify_model
+
+hubert_soft = torch.hub.load("bshall/hubert:main", "hubert_soft").cuda()
+hubert_soft.eval()
+
+script_acoustic = True #@param {type: "boolean"}
+print(script_acoustic)
+
+# Load the acoustic model (either hubert_soft or hubert_discrete) 
+#input_acoustic_model_file = "/content/drive/MyDrive/VCModels/Padme/model-24000.pt" #@param {type: "string"}
+input_acoustic_model_file = "/home/bt/models/voice-conversion/trump-acoustic.pt"
+acoustic_dict = torch.load(input_acoustic_model_file) # this used ddp ## REPLACE THIS! <-- !
+if script_acoustic == True:
+    acoustic = AcousticModelScripted().cuda()
+else:
+    acoustic = AcousticModel().cuda()
+
+acoustic.eval()
+
+new_state_dict = modify_model(state_dict=acoustic_dict)
+acoustic.load_state_dict(new_state_dict)
+
+import importlib.util
+import sys
+
+spec = importlib.util.spec_from_file_location("hifigan.utils", "/content/hifigan/hifigan/utils.py")
+u = importlib.util.module_from_spec(spec)
+sys.modules["hifigan.utils"] = u
+spec.loader.exec_module(u)
+
+spec = importlib.util.spec_from_file_location("hifigan.generator", "/content/hifigan/hifigan/generator.py")
+g = importlib.util.module_from_spec(spec)
+sys.modules["hifigan.generator"] = g
+spec.loader.exec_module(g)
+
+new_state_dict = modify_model(state_dict=acoustic_dict)
+acoustic.load_state_dict(new_state_dict)
+input_hifigan_checkpoint_path = "/content/drive/MyDrive/VCModels/Hifigan/Padme.pt" #@param {type: "string"}
+if input_hifigan_checkpoint_path:
+  checkpoint = torch.load(input_hifigan_checkpoint_path)["generator"]["model"]
+  hifigan = g.HifiganGenerator()
+  consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+  hifigan.load_state_dict(checkpoint)
+  hifigan.remove_weight_norm()
+else:
+  hifigan = torch.hub.load("bshall/hifigan:main", "hifigan_hubert_soft")
+
+hifigan.cuda()
+hifigan.eval()
+
+#utils.remove_weight_norm(hubert_soft.positional_embedding.conv)
+# Convert to the target speaker
+units = None
+mel = None 
+
+unit_start_time = 0.0
+unit_end_time = 0.0
+
+acoustic_start_time = 0.0
+acoustic_end_time = 0.0 
+
+hifi_start_time = 0.0
+hifi_end_time = 0.0
+
+with torch.inference_mode():
+    unit_start_time = datetime.datetime.now()
+    units = hubert_soft.units(source)
+    print(f"Units and Type {units.shape} {units.dtype}")
+    unit_end_time = datetime.datetime.now()
+
+    hubert_soft_script = torch.jit.trace_module(hubert_soft,{"units":source})
+    torch.jit.save(hubert_soft_script, 'hubert_soft.jit')
+
+    print(f"Unit Generation Time {(unit_end_time - unit_start_time).total_seconds() * 1000}")
+
+    print("Hubert Soft Exported")
+
+    acoustic_start_time = datetime.datetime.now()
+    mel = acoustic.generate(units).transpose(1, 2)
+    print(f"Mel and Type {mel.shape} {mel.dtype}")
+    acoustic_end_time = datetime.datetime.now()
+    print(f"Acoustic Time {(acoustic_end_time - acoustic_start_time).total_seconds() * 1000}")
+    if script_acoustic == True:
+        output_acoustic_scripted = '/content/drive/MyDrive/VCModels/FinishedModels/Padme.recast' #@param {type: "string"}
+        print("Exporting Scripted")
+        acoustic_model_script = torch.jit.script(acoustic)
+        torch.jit.save(acoustic_model_script, output_acoustic_scripted)
+    else:
+        print("Exporting Traced")
+        acoustic_model_script = torch.jit.trace_module(acoustic,{"generate":units})
+        output_acoustic_traced = '' #@param {type: "string"}
+        torch.jit.save(acoustic_model_script, output_acoustic_traced)
+
+    print("Acoustic Model Exported")
+    hifi_start_time = datetime.datetime.now()
+
+    target = hifigan(mel)
+    print(f"Wavform and Type {target.shape} {target.dtype}")
+    hifi_end_time = datetime.datetime.now()
+    print(f"MEL {target.shape}")
+    print(f"Hifi Gan Time {(hifi_end_time - hifi_start_time).total_seconds() * 1000}")
+
+    hifi_gan_script = torch.jit.trace_module(hifigan,{"forward":mel})
+    output_hifigan_jit = '/content/drive/MyDrive/VCModels/FinishedModels/Padme-hifigan-jit.pt' #@param {type: "string"}
+    torch.jit.save(hifi_gan_script, output_hifigan_jit)
+    print("Hifi Gan Model Exported")
+

voice_conversion/softvc/requirements.txt

@@ -1,24 +1,41 @@
+asttokens==2.2.1
+backcall==0.2.0
 certifi==2022.12.7
 cffi==1.15.1
 charset-normalizer==2.1.1
 contourpy==1.0.6
 cycler==0.11.0
+decorator==5.1.1
+executing==1.2.0
 fonttools==4.38.0
 idna==3.4
+ipython==8.7.0
+jedi==0.18.2
 kiwisolver==1.4.4
 matplotlib==3.6.2
+matplotlib-inline==0.1.6
 numpy==1.23.5
 packaging==22.0
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
 Pillow==9.3.0
+prompt-toolkit==3.0.36
+ptyprocess==0.7.0
+pure-eval==0.2.2
 pycparser==2.21
+Pygments==2.13.0
 pyparsing==3.0.9
 python-dateutil==2.8.2
 requests==2.28.1
 six==1.16.0
 sklearn==0.0.post1
 soundfile==0.11.0
+stack-data==0.6.2
 torch==1.13.0+cu116
 torchaudio==0.13.0+cu116
 torchvision==0.14.0+cu116
+traitlets==5.6.0
 typing_extensions==4.4.0
 urllib3==1.26.13
+wcwidth==0.2.5

voice_conversion/softvc/utils_acoustic.py

@@ -0,0 +1,113 @@
+import torch
+import torch.nn as nn
+
+# @torch.jit.export
+# def generate(self, x: torch.Tensor) -> torch.Tensor:
+#     x = self.encoder(x)
+#     return self.decoder.generate(x)
+class AcousticModelScripted(nn.Module):
+    def __init__(self, discrete: bool = False, upsample: bool = True):
+        super().__init__()
+        self.encoder = Encoder(discrete, upsample)
+        self.decoder = Decoder()
+
+    def forward(self, x: torch.Tensor, mels: torch.Tensor) -> torch.Tensor:
+        x = self.encoder(x)
+        return self.decoder(x, mels)
+
+    @torch.jit.export
+    def generate(self, x: torch.Tensor) -> torch.Tensor:
+        x = self.encoder(x)
+        return self.decoder.generate(x)
+
+
+class Encoder(nn.Module):
+    def __init__(self, discrete: bool = False, upsample: bool = True):
+        super().__init__()
+        self.embedding = nn.Embedding(100 + 1, 256) if discrete else None
+        self.prenet = PreNet(256, 256, 256)
+        self.convs = nn.Sequential(
+            nn.Conv1d(256, 512, 5, 1, 2),
+            nn.ReLU(),
+            nn.InstanceNorm1d(512),
+            nn.ConvTranspose1d(512, 512, 4, 2, 1) if upsample else nn.Identity(),
+            nn.Conv1d(512, 512, 5, 1, 2),
+            nn.ReLU(),
+            nn.InstanceNorm1d(512),
+            nn.Conv1d(512, 512, 5, 1, 2),
+            nn.ReLU(),
+            nn.InstanceNorm1d(512),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        if self.embedding is not None:
+            x = self.embedding(x)
+        x = self.prenet(x)
+        x = self.convs(x.transpose(1, 2))
+        return x.transpose(1, 2)
+
+
+class Decoder(nn.Module):
+    def __init__(self):
+        super().__init__()
+        self.prenet = PreNet(128, 256, 256)
+        self.lstm1 = nn.LSTM(512 + 256, 768, batch_first=True)
+        self.lstm2 = nn.LSTM(768, 768, batch_first=True)
+        self.lstm3 = nn.LSTM(768, 768, batch_first=True)
+        self.proj = nn.Linear(768, 128, bias=False)
+
+    def forward(self, x: torch.Tensor, mels: torch.Tensor) -> torch.Tensor:
+        mels = self.prenet(mels)
+        x, _ = self.lstm1(torch.cat((x, mels), dim=-1))
+        res = x
+        x, _ = self.lstm2(x)
+        x = res + x
+        res = x
+        x, _ = self.lstm3(x)
+        x = res + x
+        return self.proj(x)
+
+    @torch.inference_mode()
+    def generate(self, xs: torch.Tensor) -> torch.Tensor:
+        m = torch.zeros(xs.size(0), 128, device=xs.device)
+        h1 = torch.zeros(1, xs.size(0), 768, device=xs.device)
+        c1 = torch.zeros(1, xs.size(0), 768, device=xs.device)
+        h2 = torch.zeros(1, xs.size(0), 768, device=xs.device)
+        c2 = torch.zeros(1, xs.size(0), 768, device=xs.device)
+        h3 = torch.zeros(1, xs.size(0), 768, device=xs.device)
+        c3 = torch.zeros(1, xs.size(0), 768, device=xs.device)
+
+        mel = []
+        for x in torch.unbind(xs, dim=1):
+            m = self.prenet(m)
+            x = torch.cat((x, m), dim=1).unsqueeze(1)
+            x1, (h1, c1) = self.lstm1(x, (h1, c1))
+            x2, (h2, c2) = self.lstm2(x1, (h2, c2))
+            x = x1 + x2
+            x3, (h3, c3) = self.lstm3(x, (h3, c3))
+            x = x + x3
+            m = self.proj(x).squeeze(1)
+            mel.append(m)
+        return torch.stack(mel, dim=1)
+
+
+class PreNet(nn.Module):
+    def __init__(
+        self,
+        input_size: int,
+        hidden_size: int,
+        output_size: int,
+        dropout: float = 0.5,
+    ):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(input_size, hidden_size),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+            nn.Linear(hidden_size, output_size),
+            nn.ReLU(),
+            nn.Dropout(dropout),
+        )
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.net(x)

voice_conversion/softvc/utils_hubert.py

@@ -0,0 +1,36 @@
+# Torch Script Inference Hubert
+import copy
+from typing import Optional, Tuple
+import random
+
+from sklearn.cluster import KMeans
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn.modules.utils import consume_prefix_in_state_dict_if_present
+import torch.nn.utils as utils
+
+from torch.nn.utils.weight_norm import WeightNorm
+from typing import Dict, Any
+from collections import OrderedDict
+
+def remove_weight_norm(module):
+    module_list = [mod for mod in module.children()]
+    if len(module_list) == 0:
+        for k, hook in module._forward_pre_hooks.items():
+            if isinstance(hook, WeightNorm):
+                hook.remove(module)
+                del module._forward_pre_hooks[k]
+    else:
+        for mod in module_list:
+            remove_weight_norm(mod)
+
+
+def modify_model(state_dict:Dict[str,Any]):
+    new_state_dict = OrderedDict()
+    for k in state_dict['acoustic-model'].keys():
+        value = state_dict['acoustic-model'][k]
+        name = k[7:] # remove `module.` because they used ddp training to save
+        new_state_dict[name] = value
+    return new_state_dict