mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-29 09:18:21 +00:00
Add files via upload
This commit is contained in:
Justin John
@@ -0,0 +1,175 @@
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import parselmouth
|
||||
|
||||
now_dir = os.getcwd()
|
||||
sys.path.append(now_dir)
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pyworld
|
||||
|
||||
from infer.lib.audio import load_audio
|
||||
|
||||
logging.getLogger("numba").setLevel(logging.WARNING)
|
||||
from multiprocessing import Process
|
||||
|
||||
exp_dir = sys.argv[1]
|
||||
f = open("%s/extract_f0_feature.log" % exp_dir, "a+")
|
||||
|
||||
|
||||
def printt(strr):
|
||||
print(strr)
|
||||
f.write("%s\n" % strr)
|
||||
f.flush()
|
||||
|
||||
|
||||
n_p = int(sys.argv[2])
|
||||
f0method = sys.argv[3]
|
||||
|
||||
|
||||
class FeatureInput(object):
|
||||
def __init__(self, samplerate=16000, hop_size=160):
|
||||
self.fs = samplerate
|
||||
self.hop = hop_size
|
||||
|
||||
self.f0_bin = 256
|
||||
self.f0_max = 1100.0
|
||||
self.f0_min = 50.0
|
||||
self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
|
||||
self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)
|
||||
|
||||
def compute_f0(self, path, f0_method):
|
||||
x = load_audio(path, self.fs)
|
||||
p_len = x.shape[0] // self.hop
|
||||
if f0_method == "pm":
|
||||
time_step = 160 / 16000 * 1000
|
||||
f0_min = 50
|
||||
f0_max = 1100
|
||||
f0 = (
|
||||
parselmouth.Sound(x, self.fs)
|
||||
.to_pitch_ac(
|
||||
time_step=time_step / 1000,
|
||||
voicing_threshold=0.6,
|
||||
pitch_floor=f0_min,
|
||||
pitch_ceiling=f0_max,
|
||||
)
|
||||
.selected_array["frequency"]
|
||||
)
|
||||
pad_size = (p_len - len(f0) + 1) // 2
|
||||
if pad_size > 0 or p_len - len(f0) - pad_size > 0:
|
||||
f0 = np.pad(
|
||||
f0, [[pad_size, p_len - len(f0) - pad_size]], mode="constant"
|
||||
)
|
||||
elif f0_method == "harvest":
|
||||
f0, t = pyworld.harvest(
|
||||
x.astype(np.double),
|
||||
fs=self.fs,
|
||||
f0_ceil=self.f0_max,
|
||||
f0_floor=self.f0_min,
|
||||
frame_period=1000 * self.hop / self.fs,
|
||||
)
|
||||
f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.fs)
|
||||
elif f0_method == "dio":
|
||||
f0, t = pyworld.dio(
|
||||
x.astype(np.double),
|
||||
fs=self.fs,
|
||||
f0_ceil=self.f0_max,
|
||||
f0_floor=self.f0_min,
|
||||
frame_period=1000 * self.hop / self.fs,
|
||||
)
|
||||
f0 = pyworld.stonemask(x.astype(np.double), f0, t, self.fs)
|
||||
elif f0_method == "rmvpe":
|
||||
if hasattr(self, "model_rmvpe") == False:
|
||||
from infer.lib.rmvpe import RMVPE
|
||||
|
||||
print("Loading rmvpe model")
|
||||
self.model_rmvpe = RMVPE(
|
||||
"assets/rmvpe/rmvpe.pt", is_half=False, device="cpu"
|
||||
)
|
||||
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
|
||||
return f0
|
||||
|
||||
def coarse_f0(self, f0):
|
||||
f0_mel = 1127 * np.log(1 + f0 / 700)
|
||||
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * (
|
||||
self.f0_bin - 2
|
||||
) / (self.f0_mel_max - self.f0_mel_min) + 1
|
||||
|
||||
# use 0 or 1
|
||||
f0_mel[f0_mel <= 1] = 1
|
||||
f0_mel[f0_mel > self.f0_bin - 1] = self.f0_bin - 1
|
||||
f0_coarse = np.rint(f0_mel).astype(int)
|
||||
assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (
|
||||
f0_coarse.max(),
|
||||
f0_coarse.min(),
|
||||
)
|
||||
return f0_coarse
|
||||
|
||||
def go(self, paths, f0_method):
|
||||
if len(paths) == 0:
|
||||
printt("no-f0-todo")
|
||||
else:
|
||||
printt("todo-f0-%s" % len(paths))
|
||||
n = max(len(paths) // 5, 1) # 每个进程最多打印5条
|
||||
for idx, (inp_path, opt_path1, opt_path2) in enumerate(paths):
|
||||
try:
|
||||
if idx % n == 0:
|
||||
printt("f0ing,now-%s,all-%s,-%s" % (idx, len(paths), inp_path))
|
||||
if (
|
||||
os.path.exists(opt_path1 + ".npy") == True
|
||||
and os.path.exists(opt_path2 + ".npy") == True
|
||||
):
|
||||
continue
|
||||
featur_pit = self.compute_f0(inp_path, f0_method)
|
||||
np.save(
|
||||
opt_path2,
|
||||
featur_pit,
|
||||
allow_pickle=False,
|
||||
) # nsf
|
||||
coarse_pit = self.coarse_f0(featur_pit)
|
||||
np.save(
|
||||
opt_path1,
|
||||
coarse_pit,
|
||||
allow_pickle=False,
|
||||
) # ori
|
||||
except:
|
||||
printt("f0fail-%s-%s-%s" % (idx, inp_path, traceback.format_exc()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# exp_dir=r"E:\codes\py39\dataset\mi-test"
|
||||
# n_p=16
|
||||
# f = open("%s/log_extract_f0.log"%exp_dir, "w")
|
||||
printt(sys.argv)
|
||||
featureInput = FeatureInput()
|
||||
paths = []
|
||||
inp_root = "%s/1_16k_wavs" % (exp_dir)
|
||||
opt_root1 = "%s/2a_f0" % (exp_dir)
|
||||
opt_root2 = "%s/2b-f0nsf" % (exp_dir)
|
||||
|
||||
os.makedirs(opt_root1, exist_ok=True)
|
||||
os.makedirs(opt_root2, exist_ok=True)
|
||||
for name in sorted(list(os.listdir(inp_root))):
|
||||
inp_path = "%s/%s" % (inp_root, name)
|
||||
if "spec" in inp_path:
|
||||
continue
|
||||
opt_path1 = "%s/%s" % (opt_root1, name)
|
||||
opt_path2 = "%s/%s" % (opt_root2, name)
|
||||
paths.append([inp_path, opt_path1, opt_path2])
|
||||
|
||||
ps = []
|
||||
for i in range(n_p):
|
||||
p = Process(
|
||||
target=featureInput.go,
|
||||
args=(
|
||||
paths[i::n_p],
|
||||
f0method,
|
||||
),
|
||||
)
|
||||
ps.append(p)
|
||||
p.start()
|
||||
for i in range(n_p):
|
||||
ps[i].join()
|
||||
@@ -0,0 +1,141 @@
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import parselmouth
|
||||
|
||||
now_dir = os.getcwd()
|
||||
sys.path.append(now_dir)
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pyworld
|
||||
|
||||
from infer.lib.audio import load_audio
|
||||
|
||||
logging.getLogger("numba").setLevel(logging.WARNING)
|
||||
|
||||
n_part = int(sys.argv[1])
|
||||
i_part = int(sys.argv[2])
|
||||
i_gpu = sys.argv[3]
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = str(i_gpu)
|
||||
exp_dir = sys.argv[4]
|
||||
is_half = sys.argv[5]
|
||||
f = open("%s/extract_f0_feature.log" % exp_dir, "a+")
|
||||
|
||||
|
||||
def printt(strr):
|
||||
print(strr)
|
||||
f.write("%s\n" % strr)
|
||||
f.flush()
|
||||
|
||||
|
||||
class FeatureInput(object):
|
||||
def __init__(self, samplerate=16000, hop_size=160):
|
||||
self.fs = samplerate
|
||||
self.hop = hop_size
|
||||
|
||||
self.f0_bin = 256
|
||||
self.f0_max = 1100.0
|
||||
self.f0_min = 50.0
|
||||
self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
|
||||
self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)
|
||||
|
||||
def compute_f0(self, path, f0_method):
|
||||
x = load_audio(path, self.fs)
|
||||
# p_len = x.shape[0] // self.hop
|
||||
if f0_method == "rmvpe":
|
||||
if hasattr(self, "model_rmvpe") == False:
|
||||
from infer.lib.rmvpe import RMVPE
|
||||
|
||||
print("Loading rmvpe model")
|
||||
self.model_rmvpe = RMVPE(
|
||||
"assets/rmvpe/rmvpe.pt", is_half=is_half, device="cuda"
|
||||
)
|
||||
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
|
||||
return f0
|
||||
|
||||
def coarse_f0(self, f0):
|
||||
f0_mel = 1127 * np.log(1 + f0 / 700)
|
||||
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * (
|
||||
self.f0_bin - 2
|
||||
) / (self.f0_mel_max - self.f0_mel_min) + 1
|
||||
|
||||
# use 0 or 1
|
||||
f0_mel[f0_mel <= 1] = 1
|
||||
f0_mel[f0_mel > self.f0_bin - 1] = self.f0_bin - 1
|
||||
f0_coarse = np.rint(f0_mel).astype(int)
|
||||
assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (
|
||||
f0_coarse.max(),
|
||||
f0_coarse.min(),
|
||||
)
|
||||
return f0_coarse
|
||||
|
||||
def go(self, paths, f0_method):
|
||||
if len(paths) == 0:
|
||||
printt("no-f0-todo")
|
||||
else:
|
||||
printt("todo-f0-%s" % len(paths))
|
||||
n = max(len(paths) // 5, 1) # 每个进程最多打印5条
|
||||
for idx, (inp_path, opt_path1, opt_path2) in enumerate(paths):
|
||||
try:
|
||||
if idx % n == 0:
|
||||
printt("f0ing,now-%s,all-%s,-%s" % (idx, len(paths), inp_path))
|
||||
if (
|
||||
os.path.exists(opt_path1 + ".npy") == True
|
||||
and os.path.exists(opt_path2 + ".npy") == True
|
||||
):
|
||||
continue
|
||||
featur_pit = self.compute_f0(inp_path, f0_method)
|
||||
np.save(
|
||||
opt_path2,
|
||||
featur_pit,
|
||||
allow_pickle=False,
|
||||
) # nsf
|
||||
coarse_pit = self.coarse_f0(featur_pit)
|
||||
np.save(
|
||||
opt_path1,
|
||||
coarse_pit,
|
||||
allow_pickle=False,
|
||||
) # ori
|
||||
except:
|
||||
printt("f0fail-%s-%s-%s" % (idx, inp_path, traceback.format_exc()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# exp_dir=r"E:\codes\py39\dataset\mi-test"
|
||||
# n_p=16
|
||||
# f = open("%s/log_extract_f0.log"%exp_dir, "w")
|
||||
printt(sys.argv)
|
||||
featureInput = FeatureInput()
|
||||
paths = []
|
||||
inp_root = "%s/1_16k_wavs" % (exp_dir)
|
||||
opt_root1 = "%s/2a_f0" % (exp_dir)
|
||||
opt_root2 = "%s/2b-f0nsf" % (exp_dir)
|
||||
|
||||
os.makedirs(opt_root1, exist_ok=True)
|
||||
os.makedirs(opt_root2, exist_ok=True)
|
||||
for name in sorted(list(os.listdir(inp_root))):
|
||||
inp_path = "%s/%s" % (inp_root, name)
|
||||
if "spec" in inp_path:
|
||||
continue
|
||||
opt_path1 = "%s/%s" % (opt_root1, name)
|
||||
opt_path2 = "%s/%s" % (opt_root2, name)
|
||||
paths.append([inp_path, opt_path1, opt_path2])
|
||||
try:
|
||||
featureInput.go(paths[i_part::n_part], "rmvpe")
|
||||
except:
|
||||
printt("f0_all_fail-%s" % (traceback.format_exc()))
|
||||
# ps = []
|
||||
# for i in range(n_p):
|
||||
# p = Process(
|
||||
# target=featureInput.go,
|
||||
# args=(
|
||||
# paths[i::n_p],
|
||||
# f0method,
|
||||
# ),
|
||||
# )
|
||||
# ps.append(p)
|
||||
# p.start()
|
||||
# for i in range(n_p):
|
||||
# ps[i].join()
|
||||
@@ -0,0 +1,139 @@
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
import parselmouth
|
||||
|
||||
now_dir = os.getcwd()
|
||||
sys.path.append(now_dir)
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import pyworld
|
||||
|
||||
from infer.lib.audio import load_audio
|
||||
|
||||
logging.getLogger("numba").setLevel(logging.WARNING)
|
||||
|
||||
exp_dir = sys.argv[1]
|
||||
import torch_directml
|
||||
|
||||
device = torch_directml.device(torch_directml.default_device())
|
||||
f = open("%s/extract_f0_feature.log" % exp_dir, "a+")
|
||||
|
||||
|
||||
def printt(strr):
|
||||
print(strr)
|
||||
f.write("%s\n" % strr)
|
||||
f.flush()
|
||||
|
||||
|
||||
class FeatureInput(object):
|
||||
def __init__(self, samplerate=16000, hop_size=160):
|
||||
self.fs = samplerate
|
||||
self.hop = hop_size
|
||||
|
||||
self.f0_bin = 256
|
||||
self.f0_max = 1100.0
|
||||
self.f0_min = 50.0
|
||||
self.f0_mel_min = 1127 * np.log(1 + self.f0_min / 700)
|
||||
self.f0_mel_max = 1127 * np.log(1 + self.f0_max / 700)
|
||||
|
||||
def compute_f0(self, path, f0_method):
|
||||
x = load_audio(path, self.fs)
|
||||
# p_len = x.shape[0] // self.hop
|
||||
if f0_method == "rmvpe":
|
||||
if hasattr(self, "model_rmvpe") == False:
|
||||
from infer.lib.rmvpe import RMVPE
|
||||
|
||||
print("Loading rmvpe model")
|
||||
self.model_rmvpe = RMVPE(
|
||||
"assets/rmvpe/rmvpe.pt", is_half=False, device=device
|
||||
)
|
||||
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
|
||||
return f0
|
||||
|
||||
def coarse_f0(self, f0):
|
||||
f0_mel = 1127 * np.log(1 + f0 / 700)
|
||||
f0_mel[f0_mel > 0] = (f0_mel[f0_mel > 0] - self.f0_mel_min) * (
|
||||
self.f0_bin - 2
|
||||
) / (self.f0_mel_max - self.f0_mel_min) + 1
|
||||
|
||||
# use 0 or 1
|
||||
f0_mel[f0_mel <= 1] = 1
|
||||
f0_mel[f0_mel > self.f0_bin - 1] = self.f0_bin - 1
|
||||
f0_coarse = np.rint(f0_mel).astype(int)
|
||||
assert f0_coarse.max() <= 255 and f0_coarse.min() >= 1, (
|
||||
f0_coarse.max(),
|
||||
f0_coarse.min(),
|
||||
)
|
||||
return f0_coarse
|
||||
|
||||
def go(self, paths, f0_method):
|
||||
if len(paths) == 0:
|
||||
printt("no-f0-todo")
|
||||
else:
|
||||
printt("todo-f0-%s" % len(paths))
|
||||
n = max(len(paths) // 5, 1) # 每个进程最多打印5条
|
||||
for idx, (inp_path, opt_path1, opt_path2) in enumerate(paths):
|
||||
try:
|
||||
if idx % n == 0:
|
||||
printt("f0ing,now-%s,all-%s,-%s" % (idx, len(paths), inp_path))
|
||||
if (
|
||||
os.path.exists(opt_path1 + ".npy") == True
|
||||
and os.path.exists(opt_path2 + ".npy") == True
|
||||
):
|
||||
continue
|
||||
featur_pit = self.compute_f0(inp_path, f0_method)
|
||||
np.save(
|
||||
opt_path2,
|
||||
featur_pit,
|
||||
allow_pickle=False,
|
||||
) # nsf
|
||||
coarse_pit = self.coarse_f0(featur_pit)
|
||||
np.save(
|
||||
opt_path1,
|
||||
coarse_pit,
|
||||
allow_pickle=False,
|
||||
) # ori
|
||||
except:
|
||||
printt("f0fail-%s-%s-%s" % (idx, inp_path, traceback.format_exc()))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# exp_dir=r"E:\codes\py39\dataset\mi-test"
|
||||
# n_p=16
|
||||
# f = open("%s/log_extract_f0.log"%exp_dir, "w")
|
||||
printt(sys.argv)
|
||||
featureInput = FeatureInput()
|
||||
paths = []
|
||||
inp_root = "%s/1_16k_wavs" % (exp_dir)
|
||||
opt_root1 = "%s/2a_f0" % (exp_dir)
|
||||
opt_root2 = "%s/2b-f0nsf" % (exp_dir)
|
||||
|
||||
os.makedirs(opt_root1, exist_ok=True)
|
||||
os.makedirs(opt_root2, exist_ok=True)
|
||||
for name in sorted(list(os.listdir(inp_root))):
|
||||
inp_path = "%s/%s" % (inp_root, name)
|
||||
if "spec" in inp_path:
|
||||
continue
|
||||
opt_path1 = "%s/%s" % (opt_root1, name)
|
||||
opt_path2 = "%s/%s" % (opt_root2, name)
|
||||
paths.append([inp_path, opt_path1, opt_path2])
|
||||
try:
|
||||
featureInput.go(paths, "rmvpe")
|
||||
except:
|
||||
printt("f0_all_fail-%s" % (traceback.format_exc()))
|
||||
# ps = []
|
||||
# for i in range(n_p):
|
||||
# p = Process(
|
||||
# target=featureInput.go,
|
||||
# args=(
|
||||
# paths[i::n_p],
|
||||
# f0method,
|
||||
# ),
|
||||
# )
|
||||
# ps.append(p)
|
||||
# p.start()
|
||||
# for i in range(n_p):
|
||||
# ps[i].join()
|
||||
@@ -0,0 +1,137 @@
|
||||
import os
|
||||
import sys
|
||||
import traceback
|
||||
|
||||
os.environ["PYTORCH_ENABLE_MPS_FALLBACK"] = "1"
|
||||
os.environ["PYTORCH_MPS_HIGH_WATERMARK_RATIO"] = "0.0"
|
||||
|
||||
device = sys.argv[1]
|
||||
n_part = int(sys.argv[2])
|
||||
i_part = int(sys.argv[3])
|
||||
if len(sys.argv) == 6:
|
||||
exp_dir = sys.argv[4]
|
||||
version = sys.argv[5]
|
||||
else:
|
||||
i_gpu = sys.argv[4]
|
||||
exp_dir = sys.argv[5]
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = str(i_gpu)
|
||||
version = sys.argv[6]
|
||||
import fairseq
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
if "privateuseone" not in device:
|
||||
device = "cpu"
|
||||
if torch.cuda.is_available():
|
||||
device = "cuda"
|
||||
elif torch.backends.mps.is_available():
|
||||
device = "mps"
|
||||
else:
|
||||
import torch_directml
|
||||
|
||||
device = torch_directml.device(torch_directml.default_device())
|
||||
|
||||
def forward_dml(ctx, x, scale):
|
||||
ctx.scale = scale
|
||||
res = x.clone().detach()
|
||||
return res
|
||||
|
||||
fairseq.modules.grad_multiply.GradMultiply.forward = forward_dml
|
||||
|
||||
f = open("%s/extract_f0_feature.log" % exp_dir, "a+")
|
||||
|
||||
|
||||
def printt(strr):
|
||||
print(strr)
|
||||
f.write("%s\n" % strr)
|
||||
f.flush()
|
||||
|
||||
|
||||
printt(sys.argv)
|
||||
model_path = "assets/hubert/hubert_base.pt"
|
||||
|
||||
printt(exp_dir)
|
||||
wavPath = "%s/1_16k_wavs" % exp_dir
|
||||
outPath = (
|
||||
"%s/3_feature256" % exp_dir if version == "v1" else "%s/3_feature768" % exp_dir
|
||||
)
|
||||
os.makedirs(outPath, exist_ok=True)
|
||||
|
||||
|
||||
# wave must be 16k, hop_size=320
|
||||
def readwave(wav_path, normalize=False):
|
||||
wav, sr = sf.read(wav_path)
|
||||
assert sr == 16000
|
||||
feats = torch.from_numpy(wav).float()
|
||||
if feats.dim() == 2: # double channels
|
||||
feats = feats.mean(-1)
|
||||
assert feats.dim() == 1, feats.dim()
|
||||
if normalize:
|
||||
with torch.no_grad():
|
||||
feats = F.layer_norm(feats, feats.shape)
|
||||
feats = feats.view(1, -1)
|
||||
return feats
|
||||
|
||||
|
||||
# HuBERT model
|
||||
printt("load model(s) from {}".format(model_path))
|
||||
# if hubert model is exist
|
||||
if os.access(model_path, os.F_OK) == False:
|
||||
printt(
|
||||
"Error: Extracting is shut down because %s does not exist, you may download it from https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main"
|
||||
% model_path
|
||||
)
|
||||
exit(0)
|
||||
models, saved_cfg, task = fairseq.checkpoint_utils.load_model_ensemble_and_task(
|
||||
[model_path],
|
||||
suffix="",
|
||||
)
|
||||
model = models[0]
|
||||
model = model.to(device)
|
||||
printt("move model to %s" % device)
|
||||
if device not in ["mps", "cpu"]:
|
||||
model = model.half()
|
||||
model.eval()
|
||||
|
||||
todo = sorted(list(os.listdir(wavPath)))[i_part::n_part]
|
||||
n = max(1, len(todo) // 10) # 最多打印十条
|
||||
if len(todo) == 0:
|
||||
printt("no-feature-todo")
|
||||
else:
|
||||
printt("all-feature-%s" % len(todo))
|
||||
for idx, file in enumerate(todo):
|
||||
try:
|
||||
if file.endswith(".wav"):
|
||||
wav_path = "%s/%s" % (wavPath, file)
|
||||
out_path = "%s/%s" % (outPath, file.replace("wav", "npy"))
|
||||
|
||||
if os.path.exists(out_path):
|
||||
continue
|
||||
|
||||
feats = readwave(wav_path, normalize=saved_cfg.task.normalize)
|
||||
padding_mask = torch.BoolTensor(feats.shape).fill_(False)
|
||||
inputs = {
|
||||
"source": feats.half().to(device)
|
||||
if device not in ["mps", "cpu"]
|
||||
else feats.to(device),
|
||||
"padding_mask": padding_mask.to(device),
|
||||
"output_layer": 9 if version == "v1" else 12, # layer 9
|
||||
}
|
||||
with torch.no_grad():
|
||||
logits = model.extract_features(**inputs)
|
||||
feats = (
|
||||
model.final_proj(logits[0]) if version == "v1" else logits[0]
|
||||
)
|
||||
|
||||
feats = feats.squeeze(0).float().cpu().numpy()
|
||||
if np.isnan(feats).sum() == 0:
|
||||
np.save(out_path, feats, allow_pickle=False)
|
||||
else:
|
||||
printt("%s-contains nan" % file)
|
||||
if idx % n == 0:
|
||||
printt("now-%s,all-%s,%s,%s" % (len(todo), idx, file, feats.shape))
|
||||
except:
|
||||
printt(traceback.format_exc())
|
||||
printt("all-feature-done")
|
||||
@@ -0,0 +1,147 @@
|
||||
import multiprocessing
|
||||
import os
|
||||
import sys
|
||||
|
||||
from scipy import signal
|
||||
|
||||
now_dir = os.getcwd()
|
||||
sys.path.append(now_dir)
|
||||
print(sys.argv)
|
||||
inp_root = sys.argv[1]
|
||||
sr = int(sys.argv[2])
|
||||
n_p = int(sys.argv[3])
|
||||
exp_dir = sys.argv[4]
|
||||
noparallel = sys.argv[5] == "True"
|
||||
per = float(sys.argv[6])
|
||||
import multiprocessing
|
||||
import os
|
||||
import traceback
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
from scipy.io import wavfile
|
||||
|
||||
from infer.lib.audio import load_audio
|
||||
from infer.lib.slicer2 import Slicer
|
||||
|
||||
mutex = multiprocessing.Lock()
|
||||
f = open("%s/preprocess.log" % exp_dir, "a+")
|
||||
|
||||
|
||||
def println(strr):
|
||||
mutex.acquire()
|
||||
print(strr)
|
||||
f.write("%s\n" % strr)
|
||||
f.flush()
|
||||
mutex.release()
|
||||
|
||||
|
||||
class PreProcess:
|
||||
def __init__(self, sr, exp_dir, per=3.7):
|
||||
self.slicer = Slicer(
|
||||
sr=sr,
|
||||
threshold=-42,
|
||||
min_length=1500,
|
||||
min_interval=400,
|
||||
hop_size=15,
|
||||
max_sil_kept=500,
|
||||
)
|
||||
self.sr = sr
|
||||
self.bh, self.ah = signal.butter(N=5, Wn=48, btype="high", fs=self.sr)
|
||||
self.per = per
|
||||
self.overlap = 0.3
|
||||
self.tail = self.per + self.overlap
|
||||
self.max = 0.9
|
||||
self.alpha = 0.75
|
||||
self.exp_dir = exp_dir
|
||||
self.gt_wavs_dir = "%s/0_gt_wavs" % exp_dir
|
||||
self.wavs16k_dir = "%s/1_16k_wavs" % exp_dir
|
||||
os.makedirs(self.exp_dir, exist_ok=True)
|
||||
os.makedirs(self.gt_wavs_dir, exist_ok=True)
|
||||
os.makedirs(self.wavs16k_dir, exist_ok=True)
|
||||
|
||||
def norm_write(self, tmp_audio, idx0, idx1):
|
||||
tmp_max = np.abs(tmp_audio).max()
|
||||
if tmp_max > 2.5:
|
||||
print("%s-%s-%s-filtered" % (idx0, idx1, tmp_max))
|
||||
return
|
||||
tmp_audio = (tmp_audio / tmp_max * (self.max * self.alpha)) + (
|
||||
1 - self.alpha
|
||||
) * tmp_audio
|
||||
wavfile.write(
|
||||
"%s/%s_%s.wav" % (self.gt_wavs_dir, idx0, idx1),
|
||||
self.sr,
|
||||
tmp_audio.astype(np.float32),
|
||||
)
|
||||
tmp_audio = librosa.resample(
|
||||
tmp_audio, orig_sr=self.sr, target_sr=16000
|
||||
) # , res_type="soxr_vhq"
|
||||
wavfile.write(
|
||||
"%s/%s_%s.wav" % (self.wavs16k_dir, idx0, idx1),
|
||||
16000,
|
||||
tmp_audio.astype(np.float32),
|
||||
)
|
||||
|
||||
def pipeline(self, path, idx0):
|
||||
try:
|
||||
audio = load_audio(path, self.sr)
|
||||
# zero phased digital filter cause pre-ringing noise...
|
||||
# audio = signal.filtfilt(self.bh, self.ah, audio)
|
||||
audio = signal.lfilter(self.bh, self.ah, audio)
|
||||
|
||||
idx1 = 0
|
||||
for audio in self.slicer.slice(audio):
|
||||
i = 0
|
||||
while 1:
|
||||
start = int(self.sr * (self.per - self.overlap) * i)
|
||||
i += 1
|
||||
if len(audio[start:]) > self.tail * self.sr:
|
||||
tmp_audio = audio[start : start + int(self.per * self.sr)]
|
||||
self.norm_write(tmp_audio, idx0, idx1)
|
||||
idx1 += 1
|
||||
else:
|
||||
tmp_audio = audio[start:]
|
||||
idx1 += 1
|
||||
break
|
||||
self.norm_write(tmp_audio, idx0, idx1)
|
||||
println("%s->Suc." % path)
|
||||
except:
|
||||
println("%s->%s" % (path, traceback.format_exc()))
|
||||
|
||||
def pipeline_mp(self, infos):
|
||||
for path, idx0 in infos:
|
||||
self.pipeline(path, idx0)
|
||||
|
||||
def pipeline_mp_inp_dir(self, inp_root, n_p):
|
||||
try:
|
||||
infos = [
|
||||
("%s/%s" % (inp_root, name), idx)
|
||||
for idx, name in enumerate(sorted(list(os.listdir(inp_root))))
|
||||
]
|
||||
if noparallel:
|
||||
for i in range(n_p):
|
||||
self.pipeline_mp(infos[i::n_p])
|
||||
else:
|
||||
ps = []
|
||||
for i in range(n_p):
|
||||
p = multiprocessing.Process(
|
||||
target=self.pipeline_mp, args=(infos[i::n_p],)
|
||||
)
|
||||
ps.append(p)
|
||||
p.start()
|
||||
for i in range(n_p):
|
||||
ps[i].join()
|
||||
except:
|
||||
println("Fail. %s" % traceback.format_exc())
|
||||
|
||||
|
||||
def preprocess_trainset(inp_root, sr, n_p, exp_dir, per):
|
||||
pp = PreProcess(sr, exp_dir, per)
|
||||
println("start preprocess")
|
||||
println(sys.argv)
|
||||
pp.pipeline_mp_inp_dir(inp_root, n_p)
|
||||
println("end preprocess")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
preprocess_trainset(inp_root, sr, n_p, exp_dir, per)
|
||||
@@ -0,0 +1,643 @@
|
||||
import os
|
||||
import sys
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
now_dir = os.getcwd()
|
||||
sys.path.append(os.path.join(now_dir))
|
||||
|
||||
import datetime
|
||||
|
||||
from infer.lib.train import utils
|
||||
|
||||
hps = utils.get_hparams()
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = hps.gpus.replace("-", ",")
|
||||
n_gpus = len(hps.gpus.split("-"))
|
||||
from random import randint, shuffle
|
||||
|
||||
import torch
|
||||
|
||||
try:
|
||||
import intel_extension_for_pytorch as ipex # pylint: disable=import-error, unused-import
|
||||
|
||||
if torch.xpu.is_available():
|
||||
from infer.modules.ipex import ipex_init
|
||||
from infer.modules.ipex.gradscaler import gradscaler_init
|
||||
from torch.xpu.amp import autocast
|
||||
|
||||
GradScaler = gradscaler_init()
|
||||
ipex_init()
|
||||
else:
|
||||
from torch.cuda.amp import GradScaler, autocast
|
||||
except Exception:
|
||||
from torch.cuda.amp import GradScaler, autocast
|
||||
|
||||
torch.backends.cudnn.deterministic = False
|
||||
torch.backends.cudnn.benchmark = False
|
||||
from time import sleep
|
||||
from time import time as ttime
|
||||
|
||||
import torch.distributed as dist
|
||||
import torch.multiprocessing as mp
|
||||
from torch.nn import functional as F
|
||||
from torch.nn.parallel import DistributedDataParallel as DDP
|
||||
from torch.utils.data import DataLoader
|
||||
from torch.utils.tensorboard import SummaryWriter
|
||||
|
||||
from infer.lib.infer_pack import commons
|
||||
from infer.lib.train.data_utils import (
|
||||
DistributedBucketSampler,
|
||||
TextAudioCollate,
|
||||
TextAudioCollateMultiNSFsid,
|
||||
TextAudioLoader,
|
||||
TextAudioLoaderMultiNSFsid,
|
||||
)
|
||||
|
||||
if hps.version == "v1":
|
||||
from infer.lib.infer_pack.models import MultiPeriodDiscriminator
|
||||
from infer.lib.infer_pack.models import SynthesizerTrnMs256NSFsid as RVC_Model_f0
|
||||
from infer.lib.infer_pack.models import (
|
||||
SynthesizerTrnMs256NSFsid_nono as RVC_Model_nof0,
|
||||
)
|
||||
else:
|
||||
from infer.lib.infer_pack.models import (
|
||||
SynthesizerTrnMs768NSFsid as RVC_Model_f0,
|
||||
SynthesizerTrnMs768NSFsid_nono as RVC_Model_nof0,
|
||||
MultiPeriodDiscriminatorV2 as MultiPeriodDiscriminator,
|
||||
)
|
||||
|
||||
from infer.lib.train.losses import (
|
||||
discriminator_loss,
|
||||
feature_loss,
|
||||
generator_loss,
|
||||
kl_loss,
|
||||
)
|
||||
from infer.lib.train.mel_processing import mel_spectrogram_torch, spec_to_mel_torch
|
||||
from infer.lib.train.process_ckpt import savee
|
||||
|
||||
global_step = 0
|
||||
|
||||
|
||||
class EpochRecorder:
|
||||
def __init__(self):
|
||||
self.last_time = ttime()
|
||||
|
||||
def record(self):
|
||||
now_time = ttime()
|
||||
elapsed_time = now_time - self.last_time
|
||||
self.last_time = now_time
|
||||
elapsed_time_str = str(datetime.timedelta(seconds=elapsed_time))
|
||||
current_time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
return f"[{current_time}] | ({elapsed_time_str})"
|
||||
|
||||
|
||||
def main():
|
||||
n_gpus = torch.cuda.device_count()
|
||||
|
||||
if torch.cuda.is_available() == False and torch.backends.mps.is_available() == True:
|
||||
n_gpus = 1
|
||||
if n_gpus < 1:
|
||||
# patch to unblock people without gpus. there is probably a better way.
|
||||
logger.warn("NO GPU DETECTED: falling back to CPU - this may take a while")
|
||||
n_gpus = 1
|
||||
os.environ["MASTER_ADDR"] = "localhost"
|
||||
os.environ["MASTER_PORT"] = str(randint(20000, 55555))
|
||||
children = []
|
||||
for i in range(n_gpus):
|
||||
subproc = mp.Process(
|
||||
target=run,
|
||||
args=(
|
||||
i,
|
||||
n_gpus,
|
||||
hps,
|
||||
),
|
||||
)
|
||||
children.append(subproc)
|
||||
subproc.start()
|
||||
|
||||
for i in range(n_gpus):
|
||||
children[i].join()
|
||||
|
||||
|
||||
def run(rank, n_gpus, hps):
|
||||
global global_step
|
||||
if rank == 0:
|
||||
logger = utils.get_logger(hps.model_dir)
|
||||
logger.info(hps)
|
||||
# utils.check_git_hash(hps.model_dir)
|
||||
writer = SummaryWriter(log_dir=hps.model_dir)
|
||||
writer_eval = SummaryWriter(log_dir=os.path.join(hps.model_dir, "eval"))
|
||||
|
||||
dist.init_process_group(
|
||||
backend="gloo", init_method="env://", world_size=n_gpus, rank=rank
|
||||
)
|
||||
torch.manual_seed(hps.train.seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.set_device(rank)
|
||||
|
||||
if hps.if_f0 == 1:
|
||||
train_dataset = TextAudioLoaderMultiNSFsid(hps.data.training_files, hps.data)
|
||||
else:
|
||||
train_dataset = TextAudioLoader(hps.data.training_files, hps.data)
|
||||
train_sampler = DistributedBucketSampler(
|
||||
train_dataset,
|
||||
hps.train.batch_size * n_gpus,
|
||||
# [100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, 1200,1400], # 16s
|
||||
[100, 200, 300, 400, 500, 600, 700, 800, 900], # 16s
|
||||
num_replicas=n_gpus,
|
||||
rank=rank,
|
||||
shuffle=True,
|
||||
)
|
||||
# It is possible that dataloader's workers are out of shared memory. Please try to raise your shared memory limit.
|
||||
# num_workers=8 -> num_workers=4
|
||||
if hps.if_f0 == 1:
|
||||
collate_fn = TextAudioCollateMultiNSFsid()
|
||||
else:
|
||||
collate_fn = TextAudioCollate()
|
||||
train_loader = DataLoader(
|
||||
train_dataset,
|
||||
num_workers=4,
|
||||
shuffle=False,
|
||||
pin_memory=True,
|
||||
collate_fn=collate_fn,
|
||||
batch_sampler=train_sampler,
|
||||
persistent_workers=True,
|
||||
prefetch_factor=8,
|
||||
)
|
||||
if hps.if_f0 == 1:
|
||||
net_g = RVC_Model_f0(
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
**hps.model,
|
||||
is_half=hps.train.fp16_run,
|
||||
sr=hps.sample_rate,
|
||||
)
|
||||
else:
|
||||
net_g = RVC_Model_nof0(
|
||||
hps.data.filter_length // 2 + 1,
|
||||
hps.train.segment_size // hps.data.hop_length,
|
||||
**hps.model,
|
||||
is_half=hps.train.fp16_run,
|
||||
)
|
||||
if torch.cuda.is_available():
|
||||
net_g = net_g.cuda(rank)
|
||||
net_d = MultiPeriodDiscriminator(hps.model.use_spectral_norm)
|
||||
if torch.cuda.is_available():
|
||||
net_d = net_d.cuda(rank)
|
||||
optim_g = torch.optim.AdamW(
|
||||
net_g.parameters(),
|
||||
hps.train.learning_rate,
|
||||
betas=hps.train.betas,
|
||||
eps=hps.train.eps,
|
||||
)
|
||||
optim_d = torch.optim.AdamW(
|
||||
net_d.parameters(),
|
||||
hps.train.learning_rate,
|
||||
betas=hps.train.betas,
|
||||
eps=hps.train.eps,
|
||||
)
|
||||
# net_g = DDP(net_g, device_ids=[rank], find_unused_parameters=True)
|
||||
# net_d = DDP(net_d, device_ids=[rank], find_unused_parameters=True)
|
||||
if hasattr(torch, "xpu") and torch.xpu.is_available():
|
||||
pass
|
||||
elif torch.cuda.is_available():
|
||||
net_g = DDP(net_g, device_ids=[rank])
|
||||
net_d = DDP(net_d, device_ids=[rank])
|
||||
else:
|
||||
net_g = DDP(net_g)
|
||||
net_d = DDP(net_d)
|
||||
|
||||
try: # 如果能加载自动resume
|
||||
_, _, _, epoch_str = utils.load_checkpoint(
|
||||
utils.latest_checkpoint_path(hps.model_dir, "D_*.pth"), net_d, optim_d
|
||||
) # D多半加载没事
|
||||
if rank == 0:
|
||||
logger.info("loaded D")
|
||||
# _, _, _, epoch_str = utils.load_checkpoint(utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g,load_opt=0)
|
||||
_, _, _, epoch_str = utils.load_checkpoint(
|
||||
utils.latest_checkpoint_path(hps.model_dir, "G_*.pth"), net_g, optim_g
|
||||
)
|
||||
global_step = (epoch_str - 1) * len(train_loader)
|
||||
# epoch_str = 1
|
||||
# global_step = 0
|
||||
except: # 如果首次不能加载，加载pretrain
|
||||
# traceback.print_exc()
|
||||
epoch_str = 1
|
||||
global_step = 0
|
||||
if hps.pretrainG != "":
|
||||
if rank == 0:
|
||||
logger.info("loaded pretrained %s" % (hps.pretrainG))
|
||||
if hasattr(net_g, "module"):
|
||||
logger.info(
|
||||
net_g.module.load_state_dict(
|
||||
torch.load(hps.pretrainG, map_location="cpu")["model"]
|
||||
)
|
||||
) ##测试不加载优化器
|
||||
else:
|
||||
logger.info(
|
||||
net_g.load_state_dict(
|
||||
torch.load(hps.pretrainG, map_location="cpu")["model"]
|
||||
)
|
||||
) ##测试不加载优化器
|
||||
if hps.pretrainD != "":
|
||||
if rank == 0:
|
||||
logger.info("loaded pretrained %s" % (hps.pretrainD))
|
||||
if hasattr(net_d, "module"):
|
||||
logger.info(
|
||||
net_d.module.load_state_dict(
|
||||
torch.load(hps.pretrainD, map_location="cpu")["model"]
|
||||
)
|
||||
)
|
||||
else:
|
||||
logger.info(
|
||||
net_d.load_state_dict(
|
||||
torch.load(hps.pretrainD, map_location="cpu")["model"]
|
||||
)
|
||||
)
|
||||
|
||||
scheduler_g = torch.optim.lr_scheduler.ExponentialLR(
|
||||
optim_g, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
|
||||
)
|
||||
scheduler_d = torch.optim.lr_scheduler.ExponentialLR(
|
||||
optim_d, gamma=hps.train.lr_decay, last_epoch=epoch_str - 2
|
||||
)
|
||||
|
||||
scaler = GradScaler(enabled=hps.train.fp16_run)
|
||||
|
||||
cache = []
|
||||
for epoch in range(epoch_str, hps.train.epochs + 1):
|
||||
if rank == 0:
|
||||
train_and_evaluate(
|
||||
rank,
|
||||
epoch,
|
||||
hps,
|
||||
[net_g, net_d],
|
||||
[optim_g, optim_d],
|
||||
[scheduler_g, scheduler_d],
|
||||
scaler,
|
||||
[train_loader, None],
|
||||
logger,
|
||||
[writer, writer_eval],
|
||||
cache,
|
||||
)
|
||||
else:
|
||||
train_and_evaluate(
|
||||
rank,
|
||||
epoch,
|
||||
hps,
|
||||
[net_g, net_d],
|
||||
[optim_g, optim_d],
|
||||
[scheduler_g, scheduler_d],
|
||||
scaler,
|
||||
[train_loader, None],
|
||||
None,
|
||||
None,
|
||||
cache,
|
||||
)
|
||||
scheduler_g.step()
|
||||
scheduler_d.step()
|
||||
|
||||
|
||||
def train_and_evaluate(
|
||||
rank, epoch, hps, nets, optims, schedulers, scaler, loaders, logger, writers, cache
|
||||
):
|
||||
net_g, net_d = nets
|
||||
optim_g, optim_d = optims
|
||||
train_loader, eval_loader = loaders
|
||||
if writers is not None:
|
||||
writer, writer_eval = writers
|
||||
|
||||
train_loader.batch_sampler.set_epoch(epoch)
|
||||
global global_step
|
||||
|
||||
net_g.train()
|
||||
net_d.train()
|
||||
|
||||
# Prepare data iterator
|
||||
if hps.if_cache_data_in_gpu == True:
|
||||
# Use Cache
|
||||
data_iterator = cache
|
||||
if cache == []:
|
||||
# Make new cache
|
||||
for batch_idx, info in enumerate(train_loader):
|
||||
# Unpack
|
||||
if hps.if_f0 == 1:
|
||||
(
|
||||
phone,
|
||||
phone_lengths,
|
||||
pitch,
|
||||
pitchf,
|
||||
spec,
|
||||
spec_lengths,
|
||||
wave,
|
||||
wave_lengths,
|
||||
sid,
|
||||
) = info
|
||||
else:
|
||||
(
|
||||
phone,
|
||||
phone_lengths,
|
||||
spec,
|
||||
spec_lengths,
|
||||
wave,
|
||||
wave_lengths,
|
||||
sid,
|
||||
) = info
|
||||
# Load on CUDA
|
||||
if torch.cuda.is_available():
|
||||
phone = phone.cuda(rank, non_blocking=True)
|
||||
phone_lengths = phone_lengths.cuda(rank, non_blocking=True)
|
||||
if hps.if_f0 == 1:
|
||||
pitch = pitch.cuda(rank, non_blocking=True)
|
||||
pitchf = pitchf.cuda(rank, non_blocking=True)
|
||||
sid = sid.cuda(rank, non_blocking=True)
|
||||
spec = spec.cuda(rank, non_blocking=True)
|
||||
spec_lengths = spec_lengths.cuda(rank, non_blocking=True)
|
||||
wave = wave.cuda(rank, non_blocking=True)
|
||||
wave_lengths = wave_lengths.cuda(rank, non_blocking=True)
|
||||
# Cache on list
|
||||
if hps.if_f0 == 1:
|
||||
cache.append(
|
||||
(
|
||||
batch_idx,
|
||||
(
|
||||
phone,
|
||||
phone_lengths,
|
||||
pitch,
|
||||
pitchf,
|
||||
spec,
|
||||
spec_lengths,
|
||||
wave,
|
||||
wave_lengths,
|
||||
sid,
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
cache.append(
|
||||
(
|
||||
batch_idx,
|
||||
(
|
||||
phone,
|
||||
phone_lengths,
|
||||
spec,
|
||||
spec_lengths,
|
||||
wave,
|
||||
wave_lengths,
|
||||
sid,
|
||||
),
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Load shuffled cache
|
||||
shuffle(cache)
|
||||
else:
|
||||
# Loader
|
||||
data_iterator = enumerate(train_loader)
|
||||
|
||||
# Run steps
|
||||
epoch_recorder = EpochRecorder()
|
||||
for batch_idx, info in data_iterator:
|
||||
# Data
|
||||
## Unpack
|
||||
if hps.if_f0 == 1:
|
||||
(
|
||||
phone,
|
||||
phone_lengths,
|
||||
pitch,
|
||||
pitchf,
|
||||
spec,
|
||||
spec_lengths,
|
||||
wave,
|
||||
wave_lengths,
|
||||
sid,
|
||||
) = info
|
||||
else:
|
||||
phone, phone_lengths, spec, spec_lengths, wave, wave_lengths, sid = info
|
||||
## Load on CUDA
|
||||
if (hps.if_cache_data_in_gpu == False) and torch.cuda.is_available():
|
||||
phone = phone.cuda(rank, non_blocking=True)
|
||||
phone_lengths = phone_lengths.cuda(rank, non_blocking=True)
|
||||
if hps.if_f0 == 1:
|
||||
pitch = pitch.cuda(rank, non_blocking=True)
|
||||
pitchf = pitchf.cuda(rank, non_blocking=True)
|
||||
sid = sid.cuda(rank, non_blocking=True)
|
||||
spec = spec.cuda(rank, non_blocking=True)
|
||||
spec_lengths = spec_lengths.cuda(rank, non_blocking=True)
|
||||
wave = wave.cuda(rank, non_blocking=True)
|
||||
# wave_lengths = wave_lengths.cuda(rank, non_blocking=True)
|
||||
|
||||
# Calculate
|
||||
with autocast(enabled=hps.train.fp16_run):
|
||||
if hps.if_f0 == 1:
|
||||
(
|
||||
y_hat,
|
||||
ids_slice,
|
||||
x_mask,
|
||||
z_mask,
|
||||
(z, z_p, m_p, logs_p, m_q, logs_q),
|
||||
) = net_g(phone, phone_lengths, pitch, pitchf, spec, spec_lengths, sid)
|
||||
else:
|
||||
(
|
||||
y_hat,
|
||||
ids_slice,
|
||||
x_mask,
|
||||
z_mask,
|
||||
(z, z_p, m_p, logs_p, m_q, logs_q),
|
||||
) = net_g(phone, phone_lengths, spec, spec_lengths, sid)
|
||||
mel = spec_to_mel_torch(
|
||||
spec,
|
||||
hps.data.filter_length,
|
||||
hps.data.n_mel_channels,
|
||||
hps.data.sampling_rate,
|
||||
hps.data.mel_fmin,
|
||||
hps.data.mel_fmax,
|
||||
)
|
||||
y_mel = commons.slice_segments(
|
||||
mel, ids_slice, hps.train.segment_size // hps.data.hop_length
|
||||
)
|
||||
with autocast(enabled=False):
|
||||
y_hat_mel = mel_spectrogram_torch(
|
||||
y_hat.float().squeeze(1),
|
||||
hps.data.filter_length,
|
||||
hps.data.n_mel_channels,
|
||||
hps.data.sampling_rate,
|
||||
hps.data.hop_length,
|
||||
hps.data.win_length,
|
||||
hps.data.mel_fmin,
|
||||
hps.data.mel_fmax,
|
||||
)
|
||||
if hps.train.fp16_run == True:
|
||||
y_hat_mel = y_hat_mel.half()
|
||||
wave = commons.slice_segments(
|
||||
wave, ids_slice * hps.data.hop_length, hps.train.segment_size
|
||||
) # slice
|
||||
|
||||
# Discriminator
|
||||
y_d_hat_r, y_d_hat_g, _, _ = net_d(wave, y_hat.detach())
|
||||
with autocast(enabled=False):
|
||||
loss_disc, losses_disc_r, losses_disc_g = discriminator_loss(
|
||||
y_d_hat_r, y_d_hat_g
|
||||
)
|
||||
optim_d.zero_grad()
|
||||
scaler.scale(loss_disc).backward()
|
||||
scaler.unscale_(optim_d)
|
||||
grad_norm_d = commons.clip_grad_value_(net_d.parameters(), None)
|
||||
scaler.step(optim_d)
|
||||
|
||||
with autocast(enabled=hps.train.fp16_run):
|
||||
# Generator
|
||||
y_d_hat_r, y_d_hat_g, fmap_r, fmap_g = net_d(wave, y_hat)
|
||||
with autocast(enabled=False):
|
||||
loss_mel = F.l1_loss(y_mel, y_hat_mel) * hps.train.c_mel
|
||||
loss_kl = kl_loss(z_p, logs_q, m_p, logs_p, z_mask) * hps.train.c_kl
|
||||
loss_fm = feature_loss(fmap_r, fmap_g)
|
||||
loss_gen, losses_gen = generator_loss(y_d_hat_g)
|
||||
loss_gen_all = loss_gen + loss_fm + loss_mel + loss_kl
|
||||
optim_g.zero_grad()
|
||||
scaler.scale(loss_gen_all).backward()
|
||||
scaler.unscale_(optim_g)
|
||||
grad_norm_g = commons.clip_grad_value_(net_g.parameters(), None)
|
||||
scaler.step(optim_g)
|
||||
scaler.update()
|
||||
|
||||
if rank == 0:
|
||||
if global_step % hps.train.log_interval == 0:
|
||||
lr = optim_g.param_groups[0]["lr"]
|
||||
logger.info(
|
||||
"Train Epoch: {} [{:.0f}%]".format(
|
||||
epoch, 100.0 * batch_idx / len(train_loader)
|
||||
)
|
||||
)
|
||||
# Amor For Tensorboard display
|
||||
if loss_mel > 75:
|
||||
loss_mel = 75
|
||||
if loss_kl > 9:
|
||||
loss_kl = 9
|
||||
|
||||
logger.info([global_step, lr])
|
||||
logger.info(
|
||||
f"loss_disc={loss_disc:.3f}, loss_gen={loss_gen:.3f}, loss_fm={loss_fm:.3f},loss_mel={loss_mel:.3f}, loss_kl={loss_kl:.3f}"
|
||||
)
|
||||
scalar_dict = {
|
||||
"loss/g/total": loss_gen_all,
|
||||
"loss/d/total": loss_disc,
|
||||
"learning_rate": lr,
|
||||
"grad_norm_d": grad_norm_d,
|
||||
"grad_norm_g": grad_norm_g,
|
||||
}
|
||||
scalar_dict.update(
|
||||
{
|
||||
"loss/g/fm": loss_fm,
|
||||
"loss/g/mel": loss_mel,
|
||||
"loss/g/kl": loss_kl,
|
||||
}
|
||||
)
|
||||
|
||||
scalar_dict.update(
|
||||
{"loss/g/{}".format(i): v for i, v in enumerate(losses_gen)}
|
||||
)
|
||||
scalar_dict.update(
|
||||
{"loss/d_r/{}".format(i): v for i, v in enumerate(losses_disc_r)}
|
||||
)
|
||||
scalar_dict.update(
|
||||
{"loss/d_g/{}".format(i): v for i, v in enumerate(losses_disc_g)}
|
||||
)
|
||||
image_dict = {
|
||||
"slice/mel_org": utils.plot_spectrogram_to_numpy(
|
||||
y_mel[0].data.cpu().numpy()
|
||||
),
|
||||
"slice/mel_gen": utils.plot_spectrogram_to_numpy(
|
||||
y_hat_mel[0].data.cpu().numpy()
|
||||
),
|
||||
"all/mel": utils.plot_spectrogram_to_numpy(
|
||||
mel[0].data.cpu().numpy()
|
||||
),
|
||||
}
|
||||
utils.summarize(
|
||||
writer=writer,
|
||||
global_step=global_step,
|
||||
images=image_dict,
|
||||
scalars=scalar_dict,
|
||||
)
|
||||
global_step += 1
|
||||
# /Run steps
|
||||
|
||||
if epoch % hps.save_every_epoch == 0 and rank == 0:
|
||||
if hps.if_latest == 0:
|
||||
utils.save_checkpoint(
|
||||
net_g,
|
||||
optim_g,
|
||||
hps.train.learning_rate,
|
||||
epoch,
|
||||
os.path.join(hps.model_dir, "G_{}.pth".format(global_step)),
|
||||
)
|
||||
utils.save_checkpoint(
|
||||
net_d,
|
||||
optim_d,
|
||||
hps.train.learning_rate,
|
||||
epoch,
|
||||
os.path.join(hps.model_dir, "D_{}.pth".format(global_step)),
|
||||
)
|
||||
else:
|
||||
utils.save_checkpoint(
|
||||
net_g,
|
||||
optim_g,
|
||||
hps.train.learning_rate,
|
||||
epoch,
|
||||
os.path.join(hps.model_dir, "G_{}.pth".format(2333333)),
|
||||
)
|
||||
utils.save_checkpoint(
|
||||
net_d,
|
||||
optim_d,
|
||||
hps.train.learning_rate,
|
||||
epoch,
|
||||
os.path.join(hps.model_dir, "D_{}.pth".format(2333333)),
|
||||
)
|
||||
if rank == 0 and hps.save_every_weights == "1":
|
||||
if hasattr(net_g, "module"):
|
||||
ckpt = net_g.module.state_dict()
|
||||
else:
|
||||
ckpt = net_g.state_dict()
|
||||
logger.info(
|
||||
"saving ckpt %s_e%s:%s"
|
||||
% (
|
||||
hps.name,
|
||||
epoch,
|
||||
savee(
|
||||
ckpt,
|
||||
hps.sample_rate,
|
||||
hps.if_f0,
|
||||
hps.name + "_e%s_s%s" % (epoch, global_step),
|
||||
epoch,
|
||||
hps.version,
|
||||
hps,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
if rank == 0:
|
||||
logger.info("====> Epoch: {} {}".format(epoch, epoch_recorder.record()))
|
||||
if epoch >= hps.total_epoch and rank == 0:
|
||||
logger.info("Training is done. The program is closed.")
|
||||
|
||||
if hasattr(net_g, "module"):
|
||||
ckpt = net_g.module.state_dict()
|
||||
else:
|
||||
ckpt = net_g.state_dict()
|
||||
logger.info(
|
||||
"saving final ckpt:%s"
|
||||
% (
|
||||
savee(
|
||||
ckpt, hps.sample_rate, hps.if_f0, hps.name, epoch, hps.version, hps
|
||||
)
|
||||
)
|
||||
)
|
||||
sleep(1)
|
||||
os._exit(2333333)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
torch.multiprocessing.set_start_method("spawn")
|
||||
main()
|
||||
Reference in New Issue
Block a user