storytold/storyteller-ml
2
0
Fork 0
mirror of https://github.com/storytold/storyteller-ml.git synced 2026-10-29 09:18:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #20 from storytold/brandon-rmvpe-support

Browse Source 
Rmvpe support
This commit is contained in:
Brandon Thomas
2023-10-10 01:10:04 -04:00
committed by GitHub
parent 3f47928b66 b41538fa61
commit 9e6633eaa6
3 changed files with 168 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
voice_conversion/rvc-v2-brandon/fakeyou_infer.py
+7 -3
View File
@@ -119,6 +119,8 @@ parser.add_argument('--model_path', type=str, help='path to the .pth file', requ
parser.add_argument('--model_index_path', type=str, help='path to the .index file (empty string for none)', required=False, default='') parser.add_argument('--model_index_path', type=str, help='path to the .index file (empty string for none)', required=False, default='')
parser.add_argument('--hubert_model_path', type=str, help='path to the hubert model .pth file (default hubert_base.pt)', parser.add_argument('--hubert_model_path', type=str, help='path to the hubert model .pth file (default hubert_base.pt)',
required=False, default='hubert_base.pt') required=False, default='hubert_base.pt')
parser.add_argument('--rmvpe_model_path', type=str, help='path to the RMVPE model .pth file (default rmvpe.pt)',
required=False, default='rmvpe.pt')
parser.add_argument('--input_audio_filename', type=str, help='source audio file', required=True) parser.add_argument('--input_audio_filename', type=str, help='source audio file', required=True)
parser.add_argument('--output_audio_filename', type=str, help='result audio file', required=True) parser.add_argument('--output_audio_filename', type=str, help='result audio file', required=True)
@@ -130,7 +132,7 @@ parser.add_argument('--f0_up_key', type=str,
help='the f0 key for the input audio file(default "0")', help='the f0 key for the input audio file(default "0")',
required=False, default='0') required=False, default='0')
parser.add_argument('--f0_method', type=str, parser.add_argument('--f0_method', type=str,
help='f0 estimation method to use: harvest (default), crepe, or pm', help='f0 estimation method to use: rmvpe (best), harvest (default), crepe, or pm',
required=False, default='harvest') required=False, default='harvest')
parser.add_argument('--index_rate', type=float, parser.add_argument('--index_rate', type=float,
help='The rate for the index (search feature ratio) (default 0.75)', help='The rate for the index (search feature ratio) (default 0.75)',
@@ -168,6 +170,7 @@ resample_sr = args.resample_sr
rms_mix_rate = args.rms_mix_rate rms_mix_rate = args.rms_mix_rate
protect = args.protect protect = args.protect
hubert_path = args.hubert_model_path hubert_path = args.hubert_model_path
rmvpe_path = args.rmvpe_model_path
config = Config(device, is_half) config = Config(device, is_half)
now_dir = os.getcwd() now_dir = os.getcwd()
@@ -201,7 +204,7 @@ def load_hubert(hubert_path="hubert_base.pt"):
hubert_model.eval() hubert_model.eval()
def vc_single(sid, input_audio, f0_up_key, f0_file, f0_method, file_index, index_rate, hubert_path="hubert_base.pt"): def vc_single(sid, input_audio, f0_up_key, f0_file, f0_method, file_index, index_rate, rmvpe_path, hubert_path="hubert_base.pt"):
global tgt_sr, net_g, vc, hubert_model, version global tgt_sr, net_g, vc, hubert_model, version
if input_audio is None: if input_audio is None:
return "You need to upload an audio", None return "You need to upload an audio", None
@@ -218,6 +221,7 @@ def vc_single(sid, input_audio, f0_up_key, f0_file, f0_method, file_index, index
sid, sid,
audio, audio,
input_audio, input_audio,
rmvpe_path,
times, times,
f0_up_key, f0_up_key,
f0_method, f0_method,
@@ -270,7 +274,7 @@ get_vc(model_path)
file_path = input_path file_path = input_path
wav_opt = vc_single( wav_opt = vc_single(
0, file_path, f0up_key, None, f0method, index_path, index_rate, hubert_path 0, file_path, f0up_key, None, f0method, index_path, index_rate, rmvpe_path, hubert_path
) )
out_path = opt_path out_path = opt_path
wavfile.write(out_path, tgt_sr, wav_opt) wavfile.write(out_path, tgt_sr, wav_opt)
voice_conversion/rvc-v2-brandon/trainset_preprocess_pipeline_print.py
+139 -139
View File
@@ -1,139 +1,139 @@
import sys, os, multiprocessing import sys, os, multiprocessing
from scipy import signal from scipy import signal
now_dir = os.getcwd() now_dir = os.getcwd()
sys.path.append(now_dir) sys.path.append(now_dir)
print(sys.argv) print(sys.argv)
inp_root = sys.argv[1] inp_root = sys.argv[1]
sr = int(sys.argv[2]) sr = int(sys.argv[2])
n_p = int(sys.argv[3]) n_p = int(sys.argv[3])
exp_dir = sys.argv[4] exp_dir = sys.argv[4]
noparallel = sys.argv[5] == "True" noparallel = sys.argv[5] == "True"
import numpy as np, os, traceback import numpy as np, os, traceback
from lib.slicer2 import Slicer from lib.slicer2 import Slicer
import librosa, traceback import librosa, traceback
from scipy.io import wavfile from scipy.io import wavfile
import multiprocessing import multiprocessing
from lib.audio import load_audio from lib.audio import load_audio
mutex = multiprocessing.Lock() mutex = multiprocessing.Lock()
f = open("%s/preprocess.log" % exp_dir, "a+") f = open("%s/preprocess.log" % exp_dir, "a+")
def println(strr): def println(strr):
mutex.acquire() mutex.acquire()
print(strr) print(strr)
f.write("%s\n" % strr) f.write("%s\n" % strr)
f.flush() f.flush()
mutex.release() mutex.release()
class PreProcess: class PreProcess:
def __init__(self, sr, exp_dir): def __init__(self, sr, exp_dir):
self.slicer = Slicer( self.slicer = Slicer(
sr=sr, sr=sr,
threshold=-42, threshold=-42,
min_length=1500, min_length=1500,
min_interval=400, min_interval=400,
hop_size=15, hop_size=15,
max_sil_kept=500, max_sil_kept=500,
) )
self.sr = sr self.sr = sr
self.bh, self.ah = signal.butter(N=5, Wn=48, btype="high", fs=self.sr) self.bh, self.ah = signal.butter(N=5, Wn=48, btype="high", fs=self.sr)
self.per = 3.0 self.per = 3.0
self.overlap = 0.3 self.overlap = 0.3
self.tail = self.per + self.overlap self.tail = self.per + self.overlap
self.max = 0.9 self.max = 0.9
self.alpha = 0.75 self.alpha = 0.75
self.exp_dir = exp_dir self.exp_dir = exp_dir
self.gt_wavs_dir = "%s/0_gt_wavs" % exp_dir self.gt_wavs_dir = "%s/0_gt_wavs" % exp_dir
self.wavs16k_dir = "%s/1_16k_wavs" % exp_dir self.wavs16k_dir = "%s/1_16k_wavs" % exp_dir
os.makedirs(self.exp_dir, exist_ok=True) os.makedirs(self.exp_dir, exist_ok=True)
os.makedirs(self.gt_wavs_dir, exist_ok=True) os.makedirs(self.gt_wavs_dir, exist_ok=True)
os.makedirs(self.wavs16k_dir, exist_ok=True) os.makedirs(self.wavs16k_dir, exist_ok=True)
def norm_write(self, tmp_audio, idx0, idx1): def norm_write(self, tmp_audio, idx0, idx1):
tmp_max = np.abs(tmp_audio).max() tmp_max = np.abs(tmp_audio).max()
if tmp_max > 2.5: if tmp_max > 2.5:
print("%s-%s-%s-filtered" % (idx0, idx1, tmp_max)) print("%s-%s-%s-filtered" % (idx0, idx1, tmp_max))
return return
tmp_audio = (tmp_audio / tmp_max * (self.max * self.alpha)) + ( tmp_audio = (tmp_audio / tmp_max * (self.max * self.alpha)) + (
1 - self.alpha 1 - self.alpha
) * tmp_audio ) * tmp_audio
wavfile.write( wavfile.write(
"%s/%s_%s.wav" % (self.gt_wavs_dir, idx0, idx1), "%s/%s_%s.wav" % (self.gt_wavs_dir, idx0, idx1),
self.sr, self.sr,
tmp_audio.astype(np.float32), tmp_audio.astype(np.float32),
) )
tmp_audio = librosa.resample( tmp_audio = librosa.resample(
tmp_audio, orig_sr=self.sr, target_sr=16000 tmp_audio, orig_sr=self.sr, target_sr=16000
) # , res_type="soxr_vhq" ) # , res_type="soxr_vhq"
wavfile.write( wavfile.write(
"%s/%s_%s.wav" % (self.wavs16k_dir, idx0, idx1), "%s/%s_%s.wav" % (self.wavs16k_dir, idx0, idx1),
16000, 16000,
tmp_audio.astype(np.float32), tmp_audio.astype(np.float32),
) )
def pipeline(self, path, idx0): def pipeline(self, path, idx0):
try: try:
audio = load_audio(path, self.sr) audio = load_audio(path, self.sr)
# zero phased digital filter cause pre-ringing noise... # zero phased digital filter cause pre-ringing noise...
# audio = signal.filtfilt(self.bh, self.ah, audio) # audio = signal.filtfilt(self.bh, self.ah, audio)
audio = signal.lfilter(self.bh, self.ah, audio) audio = signal.lfilter(self.bh, self.ah, audio)
idx1 = 0 idx1 = 0
for audio in self.slicer.slice(audio): for audio in self.slicer.slice(audio):
i = 0 i = 0
while 1: while 1:
start = int(self.sr * (self.per - self.overlap) * i) start = int(self.sr * (self.per - self.overlap) * i)
i += 1 i += 1
if len(audio[start:]) > self.tail * self.sr: if len(audio[start:]) > self.tail * self.sr:
tmp_audio = audio[start : start + int(self.per * self.sr)] tmp_audio = audio[start : start + int(self.per * self.sr)]
self.norm_write(tmp_audio, idx0, idx1) self.norm_write(tmp_audio, idx0, idx1)
idx1 += 1 idx1 += 1
else: else:
tmp_audio = audio[start:] tmp_audio = audio[start:]
idx1 += 1 idx1 += 1
break break
self.norm_write(tmp_audio, idx0, idx1) self.norm_write(tmp_audio, idx0, idx1)
println("%s->Suc." % path) println("%s->Suc." % path)
except: except:
println("%s->%s" % (path, traceback.format_exc())) println("%s->%s" % (path, traceback.format_exc()))
def pipeline_mp(self, infos): def pipeline_mp(self, infos):
for path, idx0 in infos: for path, idx0 in infos:
self.pipeline(path, idx0) self.pipeline(path, idx0)
def pipeline_mp_inp_dir(self, inp_root, n_p): def pipeline_mp_inp_dir(self, inp_root, n_p):
try: try:
infos = [ infos = [
("%s/%s" % (inp_root, name), idx) ("%s/%s" % (inp_root, name), idx)
for idx, name in enumerate(sorted(list(os.listdir(inp_root)))) for idx, name in enumerate(sorted(list(os.listdir(inp_root))))
] ]
if noparallel: if noparallel:
for i in range(n_p): for i in range(n_p):
self.pipeline_mp(infos[i::n_p]) self.pipeline_mp(infos[i::n_p])
else: else:
ps = [] ps = []
for i in range(n_p): for i in range(n_p):
p = multiprocessing.Process( p = multiprocessing.Process(
target=self.pipeline_mp, args=(infos[i::n_p],) target=self.pipeline_mp, args=(infos[i::n_p],)
) )
ps.append(p) ps.append(p)
p.start() p.start()
for i in range(n_p): for i in range(n_p):
ps[i].join() ps[i].join()
except: except:
println("Fail. %s" % traceback.format_exc()) println("Fail. %s" % traceback.format_exc())
def preprocess_trainset(inp_root, sr, n_p, exp_dir): def preprocess_trainset(inp_root, sr, n_p, exp_dir):
pp = PreProcess(sr, exp_dir) pp = PreProcess(sr, exp_dir)
println("start preprocess") println("start preprocess")
println(sys.argv) println(sys.argv)
pp.pipeline_mp_inp_dir(inp_root, n_p) pp.pipeline_mp_inp_dir(inp_root, n_p)
println("end preprocess") println("end preprocess")
if __name__ == "__main__": if __name__ == "__main__":
preprocess_trainset(inp_root, sr, n_p, exp_dir) preprocess_trainset(inp_root, sr, n_p, exp_dir)
voice_conversion/rvc-v2-brandon/vc_infer_pipeline.py
+22 -1
View File
@@ -27,6 +27,18 @@ def cache_harvest_f0(input_audio_path, fs, f0max, f0min, frame_period):
f0 = pyworld.stonemask(audio, f0, t, fs) f0 = pyworld.stonemask(audio, f0, t, fs)
return f0 return f0
@lru_cache
def cache_dio_f0(input_audio_path, fs, f0max, f0min, frame_period):
audio = input_audio_path2wav[input_audio_path]
f0, t = pyworld.dio(
audio,
fs=fs,
f0_ceil=f0max,
f0_floor=f0min,
frame_period=frame_period,
)
f0 = pyworld.stonemask(audio, f0, t, fs)
return f0
def change_rms(data1, sr1, data2, sr2, rate): # 1是输入音频，2是输出音频,rate是2的占比 def change_rms(data1, sr1, data2, sr2, rate): # 1是输入音频，2是输出音频,rate是2的占比
# print(data1.max(),data2.max()) # print(data1.max(),data2.max())
@@ -72,6 +84,7 @@ class VC(object):
def get_f0( def get_f0(
self, self,
input_audio_path, input_audio_path,
rmvpe_model_path,
x, x,
p_len, p_len,
f0_up_key, f0_up_key,
@@ -106,6 +119,11 @@ class VC(object):
f0 = cache_harvest_f0(input_audio_path, self.sr, f0_max, f0_min, 10) f0 = cache_harvest_f0(input_audio_path, self.sr, f0_max, f0_min, 10)
if filter_radius > 2: if filter_radius > 2:
f0 = signal.medfilt(f0, 3) f0 = signal.medfilt(f0, 3)
elif f0_method == "dio":
input_audio_path2wav[input_audio_path] = x.astype(np.double)
f0 = cache_dio_f0(input_audio_path, self.sr, f0_max, f0_min, 10)
if filter_radius > 2:
f0 = signal.medfilt(f0, 3)
elif f0_method == "crepe": elif f0_method == "crepe":
model = "full" model = "full"
# Pick a batch size that doesn't cause memory errors on your gpu # Pick a batch size that doesn't cause memory errors on your gpu
@@ -132,7 +150,8 @@ class VC(object):
from lib.rmvpe import RMVPE from lib.rmvpe import RMVPE
print("loading rmvpe model") print("loading rmvpe model")
self.model_rmvpe = RMVPE( self.model_rmvpe = RMVPE(
"rmvpe.pt", is_half=self.is_half, device=self.device #"rmvpe.pt", is_half=self.is_half, device=self.device
rmvpe_model_path, is_half=self.is_half, device=self.device
) )
f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03) f0 = self.model_rmvpe.infer_from_audio(x, thred=0.03)
@@ -275,6 +294,7 @@ class VC(object):
sid, sid,
audio, audio,
input_audio_path, input_audio_path,
rmvpe_model_path,
times, times,
f0_up_key, f0_up_key,
f0_method, f0_method,
@@ -344,6 +364,7 @@ class VC(object):
if if_f0 == 1: if if_f0 == 1:
pitch, pitchf = self.get_f0( pitch, pitchf = self.get_f0(
input_audio_path, input_audio_path,
rmvpe_model_path,
audio_pad, audio_pad,
p_len, p_len,
f0_up_key, f0_up_key,