diff --git a/tts/GPT-SoVITS/.gitattributes b/tts/GPT-SoVITS/.gitattributes new file mode 100644 index 0000000..dfe0770 --- /dev/null +++ b/tts/GPT-SoVITS/.gitattributes @@ -0,0 +1,2 @@ +# Auto detect text files and perform LF normalization +* text=auto diff --git a/tts/GPT-SoVITS/.gitignore b/tts/GPT-SoVITS/.gitignore index 754b06b..c484cf2 100644 --- a/tts/GPT-SoVITS/.gitignore +++ b/tts/GPT-SoVITS/.gitignore @@ -9,4 +9,4 @@ logs reference GPT_weights SoVITS_weights -TEMP +TEMP \ No newline at end of file diff --git a/tts/GPT-SoVITS/Docker/damo.sha256 b/tts/GPT-SoVITS/Docker/damo.sha256 deleted file mode 100644 index 6e9804d..0000000 --- a/tts/GPT-SoVITS/Docker/damo.sha256 +++ /dev/null @@ -1,3 +0,0 @@ -5bba782a5e9196166233b9ab12ba04cadff9ef9212b4ff6153ed9290ff679025 /workspace/tools/damo_asr/models/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/model.pb -b3be75be477f0780277f3bae0fe489f48718f585f3a6e45d7dd1fbb1a4255fc5 /workspace/tools/damo_asr/models/speech_fsmn_vad_zh-cn-16k-common-pytorch/model.pb -a5818bb9d933805a916eebe41eb41648f7f9caad30b4bd59d56f3ca135421916 /workspace/tools/damo_asr/models/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/model.pb \ No newline at end of file diff --git a/tts/GPT-SoVITS/Docker/download.py b/tts/GPT-SoVITS/Docker/download.py deleted file mode 100644 index 234fd06..0000000 --- a/tts/GPT-SoVITS/Docker/download.py +++ /dev/null @@ -1,5 +0,0 @@ -# Download moda ASR related models -from modelscope import snapshot_download -model_dir = snapshot_download('damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch',revision="v2.0.4") -model_dir = snapshot_download('damo/speech_fsmn_vad_zh-cn-16k-common-pytorch',revision="v2.0.4") -model_dir = snapshot_download('damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch',revision="v2.0.4") diff --git a/tts/GPT-SoVITS/Docker/download.sh b/tts/GPT-SoVITS/Docker/download.sh deleted file mode 100644 index 447e018..0000000 --- a/tts/GPT-SoVITS/Docker/download.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -set -Eeuo pipefail - -echo "Downloading models..." - -aria2c --disable-ipv6 --input-file /workspace/Docker/links.txt --dir /workspace --continue - -echo "Checking SHA256..." - -parallel --will-cite -a /workspace/Docker/links.sha256 "echo -n {} | sha256sum -c" diff --git a/tts/GPT-SoVITS/Docker/links.sha256 b/tts/GPT-SoVITS/Docker/links.sha256 deleted file mode 100644 index 37e74f6..0000000 --- a/tts/GPT-SoVITS/Docker/links.sha256 +++ /dev/null @@ -1,12 +0,0 @@ -b1c1e17e9c99547a89388f72048cd6e1b41b5a18b170e86a46dfde0324d63eb1 /python_install/GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt -fc579c1db3c1e21b721001cf99d7a584214280df19b002e200b630a34fa06eb8 /python_install/GPT_SoVITS/pretrained_models/s2D488k.pth -020a014e1e01e550e510f2f61fae5e5f5b6aab40f15c22f1f12f724df507e835 /python_install/GPT_SoVITS/pretrained_models/s2G488k.pth -24164f129c66499d1346e2aa55f183250c223161ec2770c0da3d3b08cf432d3c /python_install/GPT_SoVITS/pretrained_models/chinese-hubert-base/pytorch_model.bin -e53a693acc59ace251d143d068096ae0d7b79e4b1b503fa84c9dcf576448c1d8 /python_install/GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large/pytorch_model.bin -39796caa5db18d7f9382d8ac997ac967bfd85f7761014bb807d2543cc844ef05 /python_install/tools/uvr5/uvr5_weights/HP2_all_vocals.pth -45e6b65199e781b4a6542002699be9f19cd3d1cb7d1558bc2bfbcd84674dfe28 /python_install/tools/uvr5/uvr5_weights/HP3_all_vocals.pth -5908891829634926119720241e8573d97cbeb8277110a7512bdb0bd7563258ee /python_install/tools/uvr5/uvr5_weights/HP5_only_main_vocal.pth -8c8fd1582f9aabc363e47af62ddb88df6cae7e064cae75bbf041a067a5e0aee2 /python_install/tools/uvr5/uvr5_weights/VR-DeEchoAggressive.pth -01376dd2a571bf3cb9cced680732726d2d732609d09216a610b0d110f133febe /python_install/tools/uvr5/uvr5_weights/VR-DeEchoDeReverb.pth -56aba59db3bcdd14a14464e62f3129698ecdea62eee0f003b9360923eb3ac79e /python_install/tools/uvr5/uvr5_weights/VR-DeEchoNormal.pth -233bb5c6aaa365e568659a0a81211746fa881f8f47f82d9e864fce1f7692db80 /python_install/tools/uvr5/uvr5_weights/onnx_dereverb_By_FoxJoy/vocals.onnx diff --git a/tts/GPT-SoVITS/Docker/links.txt b/tts/GPT-SoVITS/Docker/links.txt deleted file mode 100644 index e6603db..0000000 --- a/tts/GPT-SoVITS/Docker/links.txt +++ /dev/null @@ -1,34 +0,0 @@ -# GPT-SoVITS models -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/s1bert25hz-2kh-longer-epoch%3D68e-step%3D50232.ckpt - out=GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/s2D488k.pth - out=GPT_SoVITS/pretrained_models/s2D488k.pth -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/s2G488k.pth - out=GPT_SoVITS/pretrained_models/s2G488k.pth -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/chinese-hubert-base/config.json - out=GPT_SoVITS/pretrained_models/chinese-hubert-base/config.json -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/chinese-hubert-base/preprocessor_config.json - out=GPT_SoVITS/pretrained_models/chinese-hubert-base/preprocessor_config.json -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/chinese-hubert-base/pytorch_model.bin - out=GPT_SoVITS/pretrained_models/chinese-hubert-base/pytorch_model.bin -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/chinese-roberta-wwm-ext-large/config.json - out=GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large/config.json -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/chinese-roberta-wwm-ext-large/pytorch_model.bin - out=GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large/pytorch_model.bin -https://huggingface.co/lj1995/GPT-SoVITS/resolve/main/chinese-roberta-wwm-ext-large/tokenizer.json - out=GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large/tokenizer.json -# UVR5 -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP2_all_vocals.pth - out=tools/uvr5/uvr5_weights/HP2_all_vocals.pth -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP3_all_vocals.pth - out=tools/uvr5/uvr5_weights/HP3_all_vocals.pth -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/HP5_only_main_vocal.pth - out=tools/uvr5/uvr5_weights/HP5_only_main_vocal.pth -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/VR-DeEchoAggressive.pth - out=tools/uvr5/uvr5_weights/VR-DeEchoAggressive.pth -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/VR-DeEchoDeReverb.pth - out=tools/uvr5/uvr5_weights/VR-DeEchoDeReverb.pth -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/VR-DeEchoNormal.pth - out=tools/uvr5/uvr5_weights/VR-DeEchoNormal.pth -https://huggingface.co/lj1995/VoiceConversionWebUI/resolve/main/uvr5_weights/onnx_dereverb_By_FoxJoy/vocals.onnx - out=tools/uvr5/uvr5_weights/onnx_dereverb_By_FoxJoy/vocals.onnx \ No newline at end of file diff --git a/tts/GPT-SoVITS/Dockerfile b/tts/GPT-SoVITS/Dockerfile index b93d8dd..47a57e9 100644 --- a/tts/GPT-SoVITS/Dockerfile +++ b/tts/GPT-SoVITS/Dockerfile @@ -6,13 +6,12 @@ ENV DEBIAN_FRONTEND=noninteractive ENV TZ=Etc/UTC # Create directories for Python installation and model code -RUN mkdir /python_install -RUN mkdir -p /model_code/GPT-SoVITS +RUN mkdir -p /python_install /model_code/GPT-SoVITS/GPT_SoVITS/pretrained_models # Set the working directory for Python installation WORKDIR /python_install -# Install necessary packages +# Install necessary packages and git-lfs RUN apt-get update && \ apt-get install -y --no-install-recommends \ tzdata \ @@ -21,10 +20,17 @@ RUN apt-get update && \ parallel \ aria2 \ git \ - git-lfs \ dos2unix \ - nano && \ + nano \ + curl \ + software-properties-common \ + espeak-ng \ + sudo \ + rsync && \ + curl -s https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh | bash && \ + apt-get install -y git-lfs && \ git lfs install && \ + add-apt-repository ppa:deadsnakes/ppa && \ rm -rf /var/lib/apt/lists/* # Copy the requirements.txt and install Python dependencies @@ -34,32 +40,17 @@ RUN python3 -m venv python && \ python/bin/pip install --upgrade pip wheel setuptools && \ python/bin/pip install --no-cache-dir -r requirements.txt -# Copy links.txt and links.sha256 to the appropriate directory -COPY Docker/links.txt /python_install/Docker/links.txt -COPY Docker/links.sha256 /python_install/Docker/links.sha256 - -# Download models directly using aria2c -RUN aria2c --disable-ipv6 --input-file /python_install/Docker/links.txt --dir /python_install --continue - -# Verify the downloaded files using checksums -RUN cat /python_install/Docker/links.sha256 && ls -la /python_install/GPT_SoVITS/pretrained_models && \ - parallel --will-cite -a /python_install/Docker/links.sha256 "echo -n {} | sha256sum -c" +# Clone Hugging Face repository and move models +RUN git clone https://huggingface.co/lj1995/GPT-SoVITS /python_install/GPT-SoVITS && \ + mkdir -p /model_code/GPT-SoVITS/GPT_SoVITS/pretrained_models && \ + mv /python_install/GPT-SoVITS/* /model_code/GPT-SoVITS/GPT_SoVITS/pretrained_models/ # Move Docker, GPT_SoVITS, requirements.txt, and tools to /model_code/GPT-SoVITS -RUN mv /python_install/Docker /model_code/GPT-SoVITS/ && \ - mv /python_install/GPT_SoVITS /model_code/GPT-SoVITS/ && \ - mv /python_install/requirements.txt /model_code/GPT-SoVITS/ && \ - mv /python_install/tools /model_code/GPT-SoVITS/ +RUN mv /python_install/requirements.txt /model_code/GPT-SoVITS/ # Set the working directory for model code WORKDIR /model_code/GPT-SoVITS -# Download additional models using modelscope -RUN /python_install/python/bin/pip install modelscope && \ - /python_install/python/bin/python -c "from modelscope import snapshot_download; snapshot_download('damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch', revision='v2.0.4');" && \ - /python_install/python/bin/python -c "from modelscope import snapshot_download; snapshot_download('damo/speech_fsmn_vad_zh-cn-16k-common-pytorch', revision='v2.0.4');" && \ - /python_install/python/bin/python -c "from modelscope import snapshot_download; snapshot_download('damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch', revision='v2.0.4');" - # Copy the remaining contents to the model_code directory COPY . /model_code/GPT-SoVITS diff --git a/tts/GPT-SoVITS/GPT_SoVITS/AR/data/dataset.py b/tts/GPT-SoVITS/GPT_SoVITS/AR/data/dataset.py index 54b9278..1a2ffef 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/AR/data/dataset.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/AR/data/dataset.py @@ -64,7 +64,7 @@ class Text2SemanticDataset(Dataset): # get dict self.path2 = phoneme_path # "%s/2-name2text.txt"%exp_dir#phoneme_path self.path3 = "%s/3-bert" % ( - os.path.dirname(phoneme_path) + os.path.basename(phoneme_path) ) # "%s/3-bert"%exp_dir#bert_dir self.path6 = semantic_path # "%s/6-name2semantic.tsv"%exp_dir#semantic_path assert os.path.exists(self.path2) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model.py b/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model.py index aef7825..c8ad3d8 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model.py @@ -1,9 +1,5 @@ # modified from https://github.com/yangdongchao/SoundStorm/blob/master/soundstorm/s1/AR/models/t2s_model.py # reference: https://github.com/lifeiteng/vall-e -import os, sys -now_dir = os.getcwd() -sys.path.append(now_dir) -from typing import List import torch from tqdm import tqdm @@ -39,155 +35,6 @@ default_config = { } -@torch.jit.script -class T2SMLP: - def __init__(self, w1, b1, w2, b2): - self.w1 = w1 - self.b1 = b1 - self.w2 = w2 - self.b2 = b2 - - def forward(self, x): - x = F.relu(F.linear(x, self.w1, self.b1)) - x = F.linear(x, self.w2, self.b2) - return x - - -@torch.jit.script -class T2SBlock: - def __init__( - self, - num_heads, - hidden_dim: int, - mlp: T2SMLP, - qkv_w, - qkv_b, - out_w, - out_b, - norm_w1, - norm_b1, - norm_eps1, - norm_w2, - norm_b2, - norm_eps2, - ): - self.num_heads = num_heads - self.mlp = mlp - self.hidden_dim: int = hidden_dim - self.qkv_w = qkv_w - self.qkv_b = qkv_b - self.out_w = out_w - self.out_b = out_b - self.norm_w1 = norm_w1 - self.norm_b1 = norm_b1 - self.norm_eps1 = norm_eps1 - self.norm_w2 = norm_w2 - self.norm_b2 = norm_b2 - self.norm_eps2 = norm_eps2 - - @torch.jit.ignore - def to_mask(self, x, padding_mask): - return x*padding_mask if padding_mask is not None else x - - def process_prompt(self, x, attn_mask : torch.Tensor, padding_mask:torch.Tensor=None): - - - q, k, v = F.linear(self.to_mask(x, padding_mask), self.qkv_w, self.qkv_b).chunk(3, dim=-1) - - batch_size = q.shape[0] - q_len = q.shape[1] - kv_len = k.shape[1] - - q = self.to_mask(q, padding_mask) - k_cache = self.to_mask(k, padding_mask) - v_cache = self.to_mask(v, padding_mask) - - q = q.view(batch_size, q_len, self.num_heads, -1).transpose(1, 2) - k = k_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2) - v = v_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2) - - attn = F.scaled_dot_product_attention(q, k, v, attn_mask) - - attn = attn.permute(2, 0, 1, 3).reshape(batch_size*q_len, self.hidden_dim) - attn = attn.view(q_len, batch_size, self.hidden_dim).transpose(1, 0) - attn = F.linear(self.to_mask(attn, padding_mask), self.out_w, self.out_b) - - x = self.to_mask(x + attn, padding_mask) - x = F.layer_norm( - x, [self.hidden_dim], self.norm_w1, self.norm_b1, self.norm_eps1 - ) - x = self.to_mask(x + self.mlp.forward(self.to_mask(x, padding_mask)), padding_mask) - x = F.layer_norm( - x, - [self.hidden_dim], - self.norm_w2, - self.norm_b2, - self.norm_eps2, - ) - return x, k_cache, v_cache - - def decode_next_token(self, x, k_cache, v_cache): - q, k, v = F.linear(x, self.qkv_w, self.qkv_b).chunk(3, dim=-1) - - k_cache = torch.cat([k_cache, k], dim=1) - v_cache = torch.cat([v_cache, v], dim=1) - - batch_size = q.shape[0] - q_len = q.shape[1] - kv_len = k_cache.shape[1] - - q = q.view(batch_size, q_len, self.num_heads, -1).transpose(1, 2) - k = k_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2) - v = v_cache.view(batch_size, kv_len, self.num_heads, -1).transpose(1, 2) - - - attn = F.scaled_dot_product_attention(q, k, v) - - attn = attn.permute(2, 0, 1, 3).reshape(batch_size*q_len, self.hidden_dim) - attn = attn.view(q_len, batch_size, self.hidden_dim).transpose(1, 0) - attn = F.linear(attn, self.out_w, self.out_b) - - x = x + attn - x = F.layer_norm( - x, [self.hidden_dim], self.norm_w1, self.norm_b1, self.norm_eps1 - ) - x = x + self.mlp.forward(x) - x = F.layer_norm( - x, - [self.hidden_dim], - self.norm_w2, - self.norm_b2, - self.norm_eps2, - ) - return x, k_cache, v_cache - - -@torch.jit.script -class T2STransformer: - def __init__(self, num_blocks : int, blocks: List[T2SBlock]): - self.num_blocks : int = num_blocks - self.blocks = blocks - - def process_prompt( - self, x, attn_mask : torch.Tensor, - padding_mask : torch.Tensor=None, - ): - k_cache : List[torch.Tensor] = [] - v_cache : List[torch.Tensor] = [] - for i in range(self.num_blocks): - x, k_cache_, v_cache_ = self.blocks[i].process_prompt(x, attn_mask, padding_mask) - k_cache.append(k_cache_) - v_cache.append(v_cache_) - return x, k_cache, v_cache - - def decode_next_token( - self, x, k_cache: List[torch.Tensor], v_cache: List[torch.Tensor] - ): - for i in range(self.num_blocks): - x, k_cache[i], v_cache[i] = self.blocks[i].decode_next_token(x, k_cache[i], v_cache[i]) - return x, k_cache, v_cache - - class Text2SemanticDecoder(nn.Module): def __init__(self, config, norm_first=False, top_k=3): super(Text2SemanticDecoder, self).__init__() @@ -242,37 +89,6 @@ class Text2SemanticDecoder(nn.Module): ignore_index=self.EOS, ) - blocks = [] - - for i in range(self.num_layers): - layer = self.h.layers[i] - t2smlp = T2SMLP( - layer.linear1.weight, - layer.linear1.bias, - layer.linear2.weight, - layer.linear2.bias - ) - - block = T2SBlock( - self.num_head, - self.model_dim, - t2smlp, - layer.self_attn.in_proj_weight, - layer.self_attn.in_proj_bias, - layer.self_attn.out_proj.weight, - layer.self_attn.out_proj.bias, - layer.norm1.weight, - layer.norm1.bias, - layer.norm1.eps, - layer.norm2.weight, - layer.norm2.bias, - layer.norm2.eps - ) - - blocks.append(block) - - self.t2s_transformer = T2STransformer(self.num_layers, blocks) - def make_input_data(self, x, x_lens, y, y_lens, bert_feature): x = self.ar_text_embedding(x) x = x + self.bert_proj(bert_feature.transpose(1, 2)) @@ -300,7 +116,7 @@ class Text2SemanticDecoder(nn.Module): (0, y_len), value=True, ) - # x_attn_mask[:, x_len]=False + y_attn_mask = F.pad( torch.triu( torch.ones(y_len, y_len, dtype=torch.bool, device=x.device), @@ -396,7 +212,6 @@ class Text2SemanticDecoder(nn.Module): (0, y_len), value=True, ) - # x_attn_mask[:, x_len]=False y_attn_mask = F.pad( torch.triu( torch.ones(y_len, y_len, dtype=torch.bool, device=x.device), @@ -462,7 +277,7 @@ class Text2SemanticDecoder(nn.Module): value=True, ) y_attn_mask = F.pad( - torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=0), + torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1), (x_len, 0), value=False, ) @@ -506,226 +321,16 @@ class Text2SemanticDecoder(nn.Module): # 错位 return targets[:, :-1], targets[:, 1:] - def infer_panel_batch_infer_with_flash_attn( + def infer_panel( self, - x:torch.LongTensor, #####全部文本token - x_lens:torch.LongTensor, - prompts:torch.LongTensor, ####参考音频token - bert_feature:torch.LongTensor, + x, #####全部文本token + x_lens, + prompts, ####参考音频token + bert_feature, top_k: int = -100, top_p: int = 100, early_stop_num: int = -1, temperature: float = 1.0, - repetition_penalty: float = 1.35, - **kwargs, - ): - # # fp16 会对结果产生影响（和没pad相比） - # bert_feature_dtype = bert_feature[0].dtype - # if not hasattr(self.bert_proj, "dtype"): - # self.bert_proj.dtype = torch.float32 - # self.bert_proj=self.bert_proj.float() - - ## 先对phones进行embedding、对bert_features进行project，再pad到相同长度（padding策略会影响T2S模型生成的结果。） - ## pad之后再进行Linear会有误差（和没pad相比），就离谱。。。 - max_len = kwargs.get("max_len",x_lens.max()) - # for x_item, bert_item in zip(x, bert_feature): - # max_len = max(max_len, x_item.shape[0], bert_item.shape[1]) - x_list = [self.ar_text_embedding(item) for item in x] - x_list = [F.pad(item,(0,0,0,max_len-item.shape[0]),value=0) if item.shape[0] early_stop_num) or idx==1499: - print("use early stop num:", early_stop_num) - stop = True - for i, batch_index in enumerate(batch_idx_map): - batch_index = batch_idx_map[i] - idx_list[batch_index] = idx - y_list[batch_index] = y[i, :-1] - - if not (None in idx_list): - stop = True - - if stop: - if y.shape[1]==0: - y = torch.concat([y, torch.zeros_like(samples)], dim=1) - print("bad zero prediction") - print(f"T2S Decoding EOS [{prefix_len} -> {y.shape[1]}]") - break - - ####################### update next step ################################### - y_emb = self.ar_audio_embedding(y[:, -1:]) - xy_pos = y_emb * self.ar_audio_position.x_scale + self.ar_audio_position.alpha * self.ar_audio_position.pe[:, y_len + idx].to( dtype= y_emb.dtype,device=y_emb.device) - - if (None in idx_list): - for i in range(x.shape[0]): - if idx_list[i] is None: - idx_list[i] = 1500-1 ###如果没有生成到EOS，就用最大长度代替 - - if ref_free: - return y_list, [0]*x.shape[0] - return y_list, idx_list - - def infer_panel_0307(self, - x:List[torch.LongTensor], #####全部文本token - x_lens:torch.LongTensor, - prompts:torch.LongTensor, ####参考音频token - bert_feature:torch.LongTensor, - top_k: int = -100, - top_p: int = 100, - early_stop_num: int = -1, - temperature: float = 1.0, - repetition_penalty: float = 1.35, - **kwargs - ): - y_list = [] - idx_list = [] - for i in range(len(x)): - y, idx = self.infer_panel_with_flash_attn_only(x[i].unsqueeze(0), - x_lens[i], - prompts[i].unsqueeze(0), - bert_feature[i].unsqueeze(0), - top_k, - top_p, - early_stop_num, - temperature, - repetition_penalty, - **kwargs) - y_list.append(y[0]) - idx_list.append(idx) - - return y_list, idx_list - - def infer_panel_with_flash_attn_only( - self, - x:torch.LongTensor, #####全部文本token - x_lens:torch.LongTensor, - prompts:torch.LongTensor, ####参考音频token - bert_feature:torch.LongTensor, - top_k: int = -100, - top_p: int = 100, - early_stop_num: int = -1, - temperature: float = 1.0, - repetition_penalty: float = 1.35, - **kwargs ): x = self.ar_text_embedding(x) x = x + self.bert_proj(bert_feature.transpose(1, 2)) @@ -738,9 +343,17 @@ class Text2SemanticDecoder(nn.Module): x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool) stop = False # print(1111111,self.num_layers) - - k_cache = None - v_cache = None + cache = { + "all_stage": self.num_layers, + "k": [None] * self.num_layers, ###根据配置自己手写 + "v": [None] * self.num_layers, + # "xy_pos":None,##y_pos位置编码每次都不一样的没法缓存，每次都要重新拼xy_pos.主要还是写法原因，其实是可以历史统一一样的，但也没啥计算量就不管了 + "y_emb": None, ##只需要对最新的samples求emb，再拼历史的就行 + # "logits":None,###原版就已经只对结尾求再拼接了，不用管 + # "xy_dec":None,###不需要，本来只需要最后一个做logits + "first_infer": 1, + "stage": 0, + } ################### first step ########################## if y is not None: y_emb = self.ar_audio_embedding(y) @@ -748,6 +361,7 @@ class Text2SemanticDecoder(nn.Module): prefix_len = y.shape[1] y_pos = self.ar_audio_position(y_emb) xy_pos = torch.concat([x, y_pos], dim=1) + cache["y_emb"] = y_emb ref_free = False else: y_emb = None @@ -758,58 +372,77 @@ class Text2SemanticDecoder(nn.Module): y = torch.zeros(x.shape[0], 0, dtype=torch.int, device=x.device) ref_free = True - bsz = x.shape[0] - src_len = x_len + y_len x_attn_mask_pad = F.pad( - x_attn_mask, - (0, y_len), ###xx的纯0扩展到xx纯0+xy纯1，(x,x+y) - value=True, - ) + x_attn_mask, + (0, y_len), ###xx的纯0扩展到xx纯0+xy纯1，(x,x+y) + value=True, + ) y_attn_mask = F.pad( ###yy的右上1扩展到左边xy的0,(y,x+y) torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1), (x_len, 0), value=False, ) - xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).unsqueeze(0).expand(bsz*self.num_head, -1, -1).view(bsz, self.num_head, src_len, src_len).to(x.device) - new_attn_mask = torch.zeros_like(xy_attn_mask, dtype=x.dtype) - xy_attn_mask = new_attn_mask.masked_fill(xy_attn_mask, float("-inf")) - for idx in tqdm(range(1500)): - if xy_attn_mask is not None: - xy_dec, k_cache, v_cache = self.t2s_transformer.process_prompt(xy_pos, xy_attn_mask, None) - else: - xy_dec, k_cache, v_cache = self.t2s_transformer.decode_next_token(xy_pos, k_cache, v_cache) + xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to( + x.device + ) + + for idx in tqdm(range(1500)): + + xy_dec, _ = self.h((xy_pos, None), mask=xy_attn_mask, cache=cache) logits = self.ar_predict_layer( xy_dec[:, -1] - ) - - if idx == 0: - xy_attn_mask = None - logits = logits[:, :-1] - + ) ##不用改，如果用了cache的默认就是只有一帧，取最后一帧一样的 + # samples = topk_sampling(logits, top_k=top_k, top_p=1.0, temperature=temperature) + if(idx==0):###第一次跑不能EOS否则没有了 + logits = logits[:, :-1] ###刨除1024终止符号的概率 samples = sample( - logits, y, top_k=top_k, top_p=top_p, repetition_penalty=repetition_penalty, temperature=temperature - )[0] - - y = torch.concat([y, samples], dim=1) + logits[0], y, top_k=top_k, top_p=top_p, repetition_penalty=1.35, temperature=temperature + )[0].unsqueeze(0) + # 本次生成的 semantic_ids 和之前的 y 构成新的 y + # print(samples.shape)#[1,1]#第一个1是bs + y = torch.concat([y, samples], dim=1) if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num: print("use early stop num:", early_stop_num) stop = True if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS: + # print(torch.argmax(logits, dim=-1)[0] == self.EOS, samples[0, 0] == self.EOS) stop = True if stop: + # if prompts.shape[1] == y.shape[1]: + # y = torch.concat([y, torch.zeros_like(samples)], dim=1) + # print("bad zero prediction") if y.shape[1]==0: y = torch.concat([y, torch.zeros_like(samples)], dim=1) print("bad zero prediction") print(f"T2S Decoding EOS [{prefix_len} -> {y.shape[1]}]") break - + ####################### update next step ################################### - y_emb = self.ar_audio_embedding(y[:, -1:]) - xy_pos = y_emb * self.ar_audio_position.x_scale + self.ar_audio_position.alpha * self.ar_audio_position.pe[:, y_len + idx].to(dtype=y_emb.dtype,device=y_emb.device) + cache["first_infer"] = 0 + if cache["y_emb"] is not None: + y_emb = torch.cat( + [cache["y_emb"], self.ar_audio_embedding(y[:, -1:])], dim = 1 + ) + cache["y_emb"] = y_emb + y_pos = self.ar_audio_position(y_emb) + xy_pos = y_pos[:, -1:] + else: + y_emb = self.ar_audio_embedding(y[:, -1:]) + cache["y_emb"] = y_emb + y_pos = self.ar_audio_position(y_emb) + xy_pos = y_pos + y_len = y_pos.shape[1] + ###最右边一列（是错的） + # xy_attn_mask=torch.ones((1, x_len+y_len), dtype=torch.bool,device=xy_pos.device) + # xy_attn_mask[:,-1]=False + ###最下面一行（是对的） + xy_attn_mask = torch.zeros( + (1, x_len + y_len), dtype=torch.bool, device=xy_pos.device + ) if ref_free: return y[:, :-1], 0 - return y[:, :-1], idx - 1 + return y[:, :-1], idx-1 diff --git a/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model_batch_only.py b/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model_batch_only.py deleted file mode 100644 index 8c31f12..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/AR/models/t2s_model_batch_only.py +++ /dev/null @@ -1,483 +0,0 @@ -# modified from https://github.com/feng-yufei/shared_debugging_code/blob/main/model/t2s_model.py -import torch -from tqdm import tqdm - -from AR.models.utils import make_pad_mask -from AR.models.utils import ( - topk_sampling, - sample, - logits_to_probs, - multinomial_sample_one_no_sync, - dpo_loss, - make_reject_y, - get_batch_logps -) -from AR.modules.embedding import SinePositionalEmbedding -from AR.modules.embedding import TokenEmbedding -from AR.modules.transformer import LayerNorm -from AR.modules.transformer import TransformerEncoder -from AR.modules.transformer import TransformerEncoderLayer -from torch import nn -from torch.nn import functional as F -from torchmetrics.classification import MulticlassAccuracy - -default_config = { - "embedding_dim": 512, - "hidden_dim": 512, - "num_head": 8, - "num_layers": 12, - "num_codebook": 8, - "p_dropout": 0.0, - "vocab_size": 1024 + 1, - "phoneme_vocab_size": 512, - "EOS": 1024, -} - - -class Text2SemanticDecoder(nn.Module): - def __init__(self, config, norm_first=False, top_k=3): - super(Text2SemanticDecoder, self).__init__() - self.model_dim = config["model"]["hidden_dim"] - self.embedding_dim = config["model"]["embedding_dim"] - self.num_head = config["model"]["head"] - self.num_layers = config["model"]["n_layer"] - self.norm_first = norm_first - self.vocab_size = config["model"]["vocab_size"] - self.phoneme_vocab_size = config["model"]["phoneme_vocab_size"] - self.p_dropout = config["model"]["dropout"] - self.EOS = config["model"]["EOS"] - self.norm_first = norm_first - assert self.EOS == self.vocab_size - 1 - # should be same as num of kmeans bin - # assert self.EOS == 1024 - self.bert_proj = nn.Linear(1024, self.embedding_dim) - self.ar_text_embedding = TokenEmbedding( - self.embedding_dim, self.phoneme_vocab_size, self.p_dropout - ) - self.ar_text_position = SinePositionalEmbedding( - self.embedding_dim, dropout=0.1, scale=False, alpha=True - ) - self.ar_audio_embedding = TokenEmbedding( - self.embedding_dim, self.vocab_size, self.p_dropout - ) - self.ar_audio_position = SinePositionalEmbedding( - self.embedding_dim, dropout=0.1, scale=False, alpha=True - ) - - self.h = TransformerEncoder( - TransformerEncoderLayer( - d_model=self.model_dim, - nhead=self.num_head, - dim_feedforward=self.model_dim * 4, - dropout=0.1, - batch_first=True, - norm_first=norm_first, - ), - num_layers=self.num_layers, - norm=LayerNorm(self.model_dim) if norm_first else None, - ) - - self.ar_predict_layer = nn.Linear(self.model_dim, self.vocab_size, bias=False) - self.loss_fct = nn.CrossEntropyLoss(reduction="sum") - - self.ar_accuracy_metric = MulticlassAccuracy( - self.vocab_size, - top_k=top_k, - average="micro", - multidim_average="global", - ignore_index=self.EOS, - ) - - def make_input_data(self, x, x_lens, y, y_lens, bert_feature): - x = self.ar_text_embedding(x) - x = x + self.bert_proj(bert_feature.transpose(1, 2)) - x = self.ar_text_position(x) - x_mask = make_pad_mask(x_lens) - - y_mask = make_pad_mask(y_lens) - y_mask_int = y_mask.type(torch.int64) - codes = y.type(torch.int64) * (1 - y_mask_int) - - # Training - # AR Decoder - y, targets = self.pad_y_eos(codes, y_mask_int, eos_id=self.EOS) - x_len = x_lens.max() - y_len = y_lens.max() - y_emb = self.ar_audio_embedding(y) - y_pos = self.ar_audio_position(y_emb) - - xy_padding_mask = torch.concat([x_mask, y_mask], dim=1) - - ar_xy_padding_mask = xy_padding_mask - - x_attn_mask = F.pad( - torch.zeros((x_len, x_len), dtype=torch.bool, device=x.device), - (0, y_len), - value=True, - ) - - y_attn_mask = F.pad( - torch.triu( - torch.ones(y_len, y_len, dtype=torch.bool, device=x.device), - diagonal=1, - ), - (x_len, 0), - value=False, - ) - - xy_attn_mask = torch.concat([x_attn_mask, y_attn_mask], dim=0) - bsz, src_len = x.shape[0], x_len + y_len - _xy_padding_mask = ( - ar_xy_padding_mask.view(bsz, 1, 1, src_len) - .expand(-1, self.num_head, -1, -1) - .reshape(bsz * self.num_head, 1, src_len) - ) - xy_attn_mask = xy_attn_mask.logical_or(_xy_padding_mask) - new_attn_mask = torch.zeros_like(xy_attn_mask, dtype=x.dtype) - new_attn_mask.masked_fill_(xy_attn_mask, float("-inf")) - xy_attn_mask = new_attn_mask - # x 和完整的 y 一次性输入模型 - xy_pos = torch.concat([x, y_pos], dim=1) - - return xy_pos, xy_attn_mask, targets - - def forward(self, x, x_lens, y, y_lens, bert_feature): - """ - x: phoneme_ids - y: semantic_ids - """ - - reject_y, reject_y_lens = make_reject_y(y, y_lens) - - xy_pos, xy_attn_mask, targets = self.make_input_data(x, x_lens, y, y_lens, bert_feature) - - xy_dec, _ = self.h( - (xy_pos, None), - mask=xy_attn_mask, - ) - x_len = x_lens.max() - logits = self.ar_predict_layer(xy_dec[:, x_len:]) - - ###### DPO ############# - reject_xy_pos, reject_xy_attn_mask, reject_targets = self.make_input_data(x, x_lens, reject_y, reject_y_lens, bert_feature) - - reject_xy_dec, _ = self.h( - (reject_xy_pos, None), - mask=reject_xy_attn_mask, - ) - x_len = x_lens.max() - reject_logits = self.ar_predict_layer(reject_xy_dec[:, x_len:]) - - # loss - # from feiteng: 每次 duration 越多, 梯度更新也应该更多, 所以用 sum - - loss_1 = F.cross_entropy(logits.permute(0, 2, 1), targets, reduction="sum") - acc = self.ar_accuracy_metric(logits.permute(0, 2, 1).detach(), targets).item() - - A_logits, R_logits = get_batch_logps(logits, reject_logits, targets, reject_targets) - loss_2, _, _ = dpo_loss(A_logits, R_logits, 0, 0, 0.2, reference_free=True) - - loss = loss_1 + loss_2 - - return loss, acc - - def forward_old(self, x, x_lens, y, y_lens, bert_feature): - """ - x: phoneme_ids - y: semantic_ids - """ - x = self.ar_text_embedding(x) - x = x + self.bert_proj(bert_feature.transpose(1, 2)) - x = self.ar_text_position(x) - x_mask = make_pad_mask(x_lens) - - y_mask = make_pad_mask(y_lens) - y_mask_int = y_mask.type(torch.int64) - codes = y.type(torch.int64) * (1 - y_mask_int) - - # Training - # AR Decoder - y, targets = self.pad_y_eos(codes, y_mask_int, eos_id=self.EOS) - x_len = x_lens.max() - y_len = y_lens.max() - y_emb = self.ar_audio_embedding(y) - y_pos = self.ar_audio_position(y_emb) - - xy_padding_mask = torch.concat([x_mask, y_mask], dim=1) - ar_xy_padding_mask = xy_padding_mask - - x_attn_mask = F.pad( - torch.zeros((x_len, x_len), dtype=torch.bool, device=x.device), - (0, y_len), - value=True, - ) - y_attn_mask = F.pad( - torch.triu( - torch.ones(y_len, y_len, dtype=torch.bool, device=x.device), - diagonal=1, - ), - (x_len, 0), - value=False, - ) - xy_attn_mask = torch.concat([x_attn_mask, y_attn_mask], dim=0) - bsz, src_len = x.shape[0], x_len + y_len - _xy_padding_mask = ( - ar_xy_padding_mask.view(bsz, 1, 1, src_len) - .expand(-1, self.num_head, -1, -1) - .reshape(bsz * self.num_head, 1, src_len) - ) - xy_attn_mask = xy_attn_mask.logical_or(_xy_padding_mask) - new_attn_mask = torch.zeros_like(xy_attn_mask, dtype=x.dtype) - new_attn_mask.masked_fill_(xy_attn_mask, float("-inf")) - xy_attn_mask = new_attn_mask - # x 和完整的 y 一次性输入模型 - xy_pos = torch.concat([x, y_pos], dim=1) - xy_dec, _ = self.h( - (xy_pos, None), - mask=xy_attn_mask, - ) - logits = self.ar_predict_layer(xy_dec[:, x_len:]).permute(0, 2, 1) - # loss - # from feiteng: 每次 duration 越多, 梯度更新也应该更多, 所以用 sum - loss = F.cross_entropy(logits, targets, reduction="sum") - acc = self.ar_accuracy_metric(logits.detach(), targets).item() - return loss, acc - - # 需要看下这个函数和 forward 的区别以及没有 semantic 的时候 prompts 输入什么 - def infer( - self, - x, - x_lens, - prompts, - bert_feature, - top_k: int = -100, - early_stop_num: int = -1, - temperature: float = 1.0, - ): - x = self.ar_text_embedding(x) - x = x + self.bert_proj(bert_feature.transpose(1, 2)) - x = self.ar_text_position(x) - - # AR Decoder - y = prompts - prefix_len = y.shape[1] - x_len = x.shape[1] - x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool) - stop = False - for _ in tqdm(range(1500)): - y_emb = self.ar_audio_embedding(y) - y_pos = self.ar_audio_position(y_emb) - # x 和逐渐增长的 y 一起输入给模型 - xy_pos = torch.concat([x, y_pos], dim=1) - y_len = y.shape[1] - x_attn_mask_pad = F.pad( - x_attn_mask, - (0, y_len), - value=True, - ) - y_attn_mask = F.pad( - torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1), - (x_len, 0), - value=False, - ) - xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to( - y.device - ) - - xy_dec, _ = self.h( - (xy_pos, None), - mask=xy_attn_mask, - ) - logits = self.ar_predict_layer(xy_dec[:, -1]) - samples = topk_sampling( - logits, top_k=top_k, top_p=1.0, temperature=temperature - ) - - if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num: - print("use early stop num:", early_stop_num) - stop = True - - if torch.argmax(logits, dim=-1)[0] == self.EOS or samples[0, 0] == self.EOS: - # print(torch.argmax(logits, dim=-1)[0] == self.EOS, samples[0, 0] == self.EOS) - stop = True - if stop: - if prompts.shape[1] == y.shape[1]: - y = torch.concat([y, torch.zeros_like(samples)], dim=1) - print("bad zero prediction") - print(f"T2S Decoding EOS [{prefix_len} -> {y.shape[1]}]") - break - # 本次生成的 semantic_ids 和之前的 y 构成新的 y - # print(samples.shape)#[1,1]#第一个1是bs - # import os - # os._exit(2333) - y = torch.concat([y, samples], dim=1) - return y - - def pad_y_eos(self, y, y_mask_int, eos_id): - targets = F.pad(y, (0, 1), value=0) + eos_id * F.pad( - y_mask_int, (0, 1), value=1 - ) - # 错位 - return targets[:, :-1], targets[:, 1:] - - def infer_panel( - self, - x, #####全部文本token - x_lens, - prompts, ####参考音频token - bert_feature, - top_k: int = -100, - top_p: int = 100, - early_stop_num: int = -1, - temperature: float = 1.0, - ): - x = self.ar_text_embedding(x) - x = x + self.bert_proj(bert_feature.transpose(1, 2)) - x = self.ar_text_position(x) - - # AR Decoder - y = prompts - - x_len = x.shape[1] - x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool) - stop = False - # print(1111111,self.num_layers) - cache = { - "all_stage": self.num_layers, - "k": [None] * self.num_layers, ###根据配置自己手写 - "v": [None] * self.num_layers, - # "xy_pos":None,##y_pos位置编码每次都不一样的没法缓存，每次都要重新拼xy_pos.主要还是写法原因，其实是可以历史统一一样的，但也没啥计算量就不管了 - "y_emb": None, ##只需要对最新的samples求emb，再拼历史的就行 - # "logits":None,###原版就已经只对结尾求再拼接了，不用管 - # "xy_dec":None,###不需要，本来只需要最后一个做logits - "first_infer": 1, - "stage": 0, - } - ################### first step ########################## - if y is not None: - y_emb = self.ar_audio_embedding(y) - y_len = y_emb.shape[1] - prefix_len = y.shape[1] - y_pos = self.ar_audio_position(y_emb) - xy_pos = torch.concat([x, y_pos], dim=1) - cache["y_emb"] = y_emb - ref_free = False - else: - y_emb = None - y_len = 0 - prefix_len = 0 - y_pos = None - xy_pos = x - y = torch.zeros(x.shape[0], 0, dtype=torch.int, device=x.device) - ref_free = True - - x_attn_mask_pad = F.pad( - x_attn_mask, - (0, y_len), ###xx的纯0扩展到xx纯0+xy纯1，(x,x+y) - value=True, - ) - y_attn_mask = F.pad( ###yy的右上1扩展到左边xy的0,(y,x+y) - torch.triu(torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1), - (x_len, 0), - value=False, - ) - xy_attn_mask = torch.concat([x_attn_mask_pad, y_attn_mask], dim=0).to( - x.device - ) - - y_list = [None]*y.shape[0] - batch_idx_map = list(range(y.shape[0])) - idx_list = [None]*y.shape[0] - for idx in tqdm(range(1500)): - - xy_dec, _ = self.h((xy_pos, None), mask=xy_attn_mask, cache=cache) - logits = self.ar_predict_layer( - xy_dec[:, -1] - ) ##不用改，如果用了cache的默认就是只有一帧，取最后一帧一样的 - # samples = topk_sampling(logits, top_k=top_k, top_p=1.0, temperature=temperature) - if(idx==0):###第一次跑不能EOS否则没有了 - logits = logits[:, :-1] ###刨除1024终止符号的概率 - samples = sample( - logits, y, top_k=top_k, top_p=top_p, repetition_penalty=1.35, temperature=temperature - )[0] - # 本次生成的 semantic_ids 和之前的 y 构成新的 y - # print(samples.shape)#[1,1]#第一个1是bs - y = torch.concat([y, samples], dim=1) - - # 移除已经生成完毕的序列 - reserved_idx_of_batch_for_y = None - if (self.EOS in torch.argmax(logits, dim=-1)) or \ - (self.EOS in samples[:, 0]): ###如果生成到EOS，则停止 - l = samples[:, 0]==self.EOS - removed_idx_of_batch_for_y = torch.where(l==True)[0].tolist() - reserved_idx_of_batch_for_y = torch.where(l==False)[0] - # batch_indexs = torch.tensor(batch_idx_map, device=y.device)[removed_idx_of_batch_for_y] - for i in removed_idx_of_batch_for_y: - batch_index = batch_idx_map[i] - idx_list[batch_index] = idx - 1 - y_list[batch_index] = y[i, :-1] - - batch_idx_map = [batch_idx_map[i] for i in reserved_idx_of_batch_for_y.tolist()] - - # 只保留未生成完毕的序列 - if reserved_idx_of_batch_for_y is not None: - # index = torch.LongTensor(batch_idx_map).to(y.device) - y = torch.index_select(y, dim=0, index=reserved_idx_of_batch_for_y) - if cache["y_emb"] is not None: - cache["y_emb"] = torch.index_select(cache["y_emb"], dim=0, index=reserved_idx_of_batch_for_y) - if cache["k"] is not None: - for i in range(self.num_layers): - # 因为kv转置了，所以batch dim是1 - cache["k"][i] = torch.index_select(cache["k"][i], dim=1, index=reserved_idx_of_batch_for_y) - cache["v"][i] = torch.index_select(cache["v"][i], dim=1, index=reserved_idx_of_batch_for_y) - - - if early_stop_num != -1 and (y.shape[1] - prefix_len) > early_stop_num: - print("use early stop num:", early_stop_num) - stop = True - - if not (None in idx_list): - # print(torch.argmax(logits, dim=-1)[0] == self.EOS, samples[0, 0] == self.EOS) - stop = True - if stop: - # if prompts.shape[1] == y.shape[1]: - # y = torch.concat([y, torch.zeros_like(samples)], dim=1) - # print("bad zero prediction") - if y.shape[1]==0: - y = torch.concat([y, torch.zeros_like(samples)], dim=1) - print("bad zero prediction") - print(f"T2S Decoding EOS [{prefix_len} -> {y.shape[1]}]") - break - - ####################### update next step ################################### - cache["first_infer"] = 0 - if cache["y_emb"] is not None: - y_emb = torch.cat( - [cache["y_emb"], self.ar_audio_embedding(y[:, -1:])], dim = 1 - ) - cache["y_emb"] = y_emb - y_pos = self.ar_audio_position(y_emb) - xy_pos = y_pos[:, -1:] - else: - y_emb = self.ar_audio_embedding(y[:, -1:]) - cache["y_emb"] = y_emb - y_pos = self.ar_audio_position(y_emb) - xy_pos = y_pos - y_len = y_pos.shape[1] - - ###最右边一列（是错的） - # xy_attn_mask=torch.ones((1, x_len+y_len), dtype=torch.bool,device=xy_pos.device) - # xy_attn_mask[:,-1]=False - ###最下面一行（是对的） - xy_attn_mask = torch.zeros( - (1, x_len + y_len), dtype=torch.bool, device=xy_pos.device - ) - - if (None in idx_list): - for i in range(x.shape[0]): - if idx_list[i] is None: - idx_list[i] = 1500-1 ###如果没有生成到EOS，就用最大长度代替 - - if ref_free: - return y_list, [0]*x.shape[0] - return y_list, idx_list diff --git a/tts/GPT-SoVITS/GPT_SoVITS/AR/models/utils.py b/tts/GPT-SoVITS/GPT_SoVITS/AR/models/utils.py index ce0a98b..9678c7e 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/AR/models/utils.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/AR/models/utils.py @@ -115,17 +115,17 @@ def logits_to_probs( top_p: Optional[int] = None, repetition_penalty: float = 1.0, ): - # if previous_tokens is not None: - # previous_tokens = previous_tokens.squeeze() + if previous_tokens is not None: + previous_tokens = previous_tokens.squeeze() # print(logits.shape,previous_tokens.shape) # pdb.set_trace() if previous_tokens is not None and repetition_penalty != 1.0: previous_tokens = previous_tokens.long() - score = torch.gather(logits, dim=1, index=previous_tokens) + score = torch.gather(logits, dim=0, index=previous_tokens) score = torch.where( score < 0, score * repetition_penalty, score / repetition_penalty ) - logits.scatter_(dim=1, index=previous_tokens, src=score) + logits.scatter_(dim=0, index=previous_tokens, src=score) if top_p is not None and top_p < 1.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True) @@ -133,9 +133,9 @@ def logits_to_probs( torch.nn.functional.softmax(sorted_logits, dim=-1), dim=-1 ) sorted_indices_to_remove = cum_probs > top_p - sorted_indices_to_remove[:, 0] = False # keep at least one option + sorted_indices_to_remove[0] = False # keep at least one option indices_to_remove = sorted_indices_to_remove.scatter( - dim=1, index=sorted_indices, src=sorted_indices_to_remove + dim=0, index=sorted_indices, src=sorted_indices_to_remove ) logits = logits.masked_fill(indices_to_remove, -float("Inf")) @@ -143,7 +143,7 @@ def logits_to_probs( if top_k is not None: v, _ = torch.topk(logits, min(top_k, logits.size(-1))) - pivot = v[: , -1].unsqueeze(-1) + pivot = v.select(-1, -1).unsqueeze(-1) logits = torch.where(logits < pivot, -float("Inf"), logits) probs = torch.nn.functional.softmax(logits, dim=-1) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/TTS.py b/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/TTS.py deleted file mode 100644 index 10dacca..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/TTS.py +++ /dev/null @@ -1,945 +0,0 @@ -from copy import deepcopy -import math -import os, sys, gc -import random -import traceback - -from tqdm import tqdm -now_dir = os.getcwd() -sys.path.append(now_dir) -import ffmpeg -import os -from typing import Generator, List, Tuple, Union -import numpy as np -import torch -import torch.nn.functional as F -import yaml -from transformers import AutoModelForMaskedLM, AutoTokenizer - -from AR.models.t2s_lightning_module import Text2SemanticLightningModule -from feature_extractor.cnhubert import CNHubert -from module.models import SynthesizerTrn -import librosa -from time import time as ttime -from tools.i18n.i18n import I18nAuto -from my_utils import load_audio -from module.mel_processing import spectrogram_torch -from TTS_infer_pack.text_segmentation_method import splits -from TTS_infer_pack.TextPreprocessor import TextPreprocessor -i18n = I18nAuto() - -# configs/tts_infer.yaml -""" -default: - device: cpu - is_half: false - bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large - cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base - t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt - vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth - -custom: - device: cuda - is_half: true - bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large - cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base - t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt - vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth - - -""" - -def set_seed(seed:int): - seed = int(seed) - seed = seed if seed != -1 else random.randrange(1 << 32) - print(f"Set seed to {seed}") - os.environ['PYTHONHASHSEED'] = str(seed) - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - try: - if torch.cuda.is_available(): - torch.cuda.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - # torch.backends.cudnn.deterministic = True - # torch.backends.cudnn.benchmark = False - # torch.backends.cudnn.enabled = True - # 开启后会影响精度 - torch.backends.cuda.matmul.allow_tf32 = False - torch.backends.cudnn.allow_tf32 = False - except: - pass - return seed - -class TTS_Config: - default_configs={ - "device": "cpu", - "is_half": False, - "t2s_weights_path": "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt", - "vits_weights_path": "GPT_SoVITS/pretrained_models/s2G488k.pth", - "cnhuhbert_base_path": "GPT_SoVITS/pretrained_models/chinese-hubert-base", - "bert_base_path": "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large", - } - configs:dict = None - def __init__(self, configs: Union[dict, str]=None): - - # 设置默认配置文件路径 - configs_base_path:str = "GPT_SoVITS/configs/" - os.makedirs(configs_base_path, exist_ok=True) - self.configs_path:str = os.path.join(configs_base_path, "tts_infer.yaml") - - if configs in ["", None]: - if not os.path.exists(self.configs_path): - self.save_configs() - print(f"Create default config file at {self.configs_path}") - configs:dict = {"default": deepcopy(self.default_configs)} - - if isinstance(configs, str): - self.configs_path = configs - configs:dict = self._load_configs(self.configs_path) - - assert isinstance(configs, dict) - default_configs:dict = configs.get("default", None) - if default_configs is not None: - self.default_configs = default_configs - - self.configs:dict = configs.get("custom", deepcopy(self.default_configs)) - - - self.device = self.configs.get("device", torch.device("cpu")) - self.is_half = self.configs.get("is_half", False) - self.t2s_weights_path = self.configs.get("t2s_weights_path", None) - self.vits_weights_path = self.configs.get("vits_weights_path", None) - self.bert_base_path = self.configs.get("bert_base_path", None) - self.cnhuhbert_base_path = self.configs.get("cnhuhbert_base_path", None) - - - if (self.t2s_weights_path in [None, ""]) or (not os.path.exists(self.t2s_weights_path)): - self.t2s_weights_path = self.default_configs['t2s_weights_path'] - print(f"fall back to default t2s_weights_path: {self.t2s_weights_path}") - if (self.vits_weights_path in [None, ""]) or (not os.path.exists(self.vits_weights_path)): - self.vits_weights_path = self.default_configs['vits_weights_path'] - print(f"fall back to default vits_weights_path: {self.vits_weights_path}") - if (self.bert_base_path in [None, ""]) or (not os.path.exists(self.bert_base_path)): - self.bert_base_path = self.default_configs['bert_base_path'] - print(f"fall back to default bert_base_path: {self.bert_base_path}") - if (self.cnhuhbert_base_path in [None, ""]) or (not os.path.exists(self.cnhuhbert_base_path)): - self.cnhuhbert_base_path = self.default_configs['cnhuhbert_base_path'] - print(f"fall back to default cnhuhbert_base_path: {self.cnhuhbert_base_path}") - self.update_configs() - - - self.max_sec = None - self.hz:int = 50 - self.semantic_frame_rate:str = "25hz" - self.segment_size:int = 20480 - self.filter_length:int = 2048 - self.sampling_rate:int = 32000 - self.hop_length:int = 640 - self.win_length:int = 2048 - self.n_speakers:int = 300 - - self.languages:list = ["auto", "en", "zh", "ja", "all_zh", "all_ja"] - - - def _load_configs(self, configs_path: str)->dict: - with open(configs_path, 'r') as f: - configs = yaml.load(f, Loader=yaml.FullLoader) - - return configs - - def save_configs(self, configs_path:str=None)->None: - configs={ - "default":self.default_configs, - } - if self.configs is not None: - configs["custom"] = self.update_configs() - - if configs_path is None: - configs_path = self.configs_path - with open(configs_path, 'w') as f: - yaml.dump(configs, f) - - def update_configs(self): - self.config = { - "device" : str(self.device), - "is_half" : self.is_half, - "t2s_weights_path" : self.t2s_weights_path, - "vits_weights_path" : self.vits_weights_path, - "bert_base_path" : self.bert_base_path, - "cnhuhbert_base_path": self.cnhuhbert_base_path, - } - return self.config - - def __str__(self): - self.configs = self.update_configs() - string = "TTS Config".center(100, '-') + '\n' - for k, v in self.configs.items(): - string += f"{str(k).ljust(20)}: {str(v)}\n" - string += "-" * 100 + '\n' - return string - - def __repr__(self): - return self.__str__() - - def __hash__(self): - return hash(self.configs_path) - - def __eq__(self, other): - return isinstance(other, TTS_Config) and self.configs_path == other.configs_path - - -class TTS: - def __init__(self, configs: Union[dict, str, TTS_Config]): - if isinstance(configs, TTS_Config): - self.configs = configs - else: - self.configs:TTS_Config = TTS_Config(configs) - - self.t2s_model:Text2SemanticLightningModule = None - self.vits_model:SynthesizerTrn = None - self.bert_tokenizer:AutoTokenizer = None - self.bert_model:AutoModelForMaskedLM = None - self.cnhuhbert_model:CNHubert = None - - self._init_models() - - self.text_preprocessor:TextPreprocessor = \ - TextPreprocessor(self.bert_model, - self.bert_tokenizer, - self.configs.device) - - - self.prompt_cache:dict = { - "ref_audio_path" : None, - "prompt_semantic": None, - "refer_spec" : None, - "prompt_text" : None, - "prompt_lang" : None, - "phones" : None, - "bert_features" : None, - "norm_text" : None, - } - - - self.stop_flag:bool = False - self.precision:torch.dtype = torch.float16 if self.configs.is_half else torch.float32 - - def _init_models(self,): - self.init_t2s_weights(self.configs.t2s_weights_path) - self.init_vits_weights(self.configs.vits_weights_path) - self.init_bert_weights(self.configs.bert_base_path) - self.init_cnhuhbert_weights(self.configs.cnhuhbert_base_path) - # self.enable_half_precision(self.configs.is_half) - - - - def init_cnhuhbert_weights(self, base_path: str): - print(f"Loading CNHuBERT weights from {base_path}") - self.cnhuhbert_model = CNHubert(base_path) - self.cnhuhbert_model=self.cnhuhbert_model.eval() - self.cnhuhbert_model = self.cnhuhbert_model.to(self.configs.device) - if self.configs.is_half and str(self.configs.device)!="cpu": - self.cnhuhbert_model = self.cnhuhbert_model.half() - - - - def init_bert_weights(self, base_path: str): - print(f"Loading BERT weights from {base_path}") - self.bert_tokenizer = AutoTokenizer.from_pretrained(base_path) - self.bert_model = AutoModelForMaskedLM.from_pretrained(base_path) - self.bert_model=self.bert_model.eval() - self.bert_model = self.bert_model.to(self.configs.device) - if self.configs.is_half and str(self.configs.device)!="cpu": - self.bert_model = self.bert_model.half() - - def init_vits_weights(self, weights_path: str): - print(f"Loading VITS weights from {weights_path}") - self.configs.vits_weights_path = weights_path - self.configs.save_configs() - dict_s2 = torch.load(weights_path, map_location=self.configs.device) - hps = dict_s2["config"] - self.configs.filter_length = hps["data"]["filter_length"] - self.configs.segment_size = hps["train"]["segment_size"] - self.configs.sampling_rate = hps["data"]["sampling_rate"] - self.configs.hop_length = hps["data"]["hop_length"] - self.configs.win_length = hps["data"]["win_length"] - self.configs.n_speakers = hps["data"]["n_speakers"] - self.configs.semantic_frame_rate = "25hz" - kwargs = hps["model"] - vits_model = SynthesizerTrn( - self.configs.filter_length // 2 + 1, - self.configs.segment_size // self.configs.hop_length, - n_speakers=self.configs.n_speakers, - **kwargs - ) - # if ("pretrained" not in weights_path): - if hasattr(vits_model, "enc_q"): - del vits_model.enc_q - - vits_model = vits_model.to(self.configs.device) - vits_model = vits_model.eval() - vits_model.load_state_dict(dict_s2["weight"], strict=False) - self.vits_model = vits_model - if self.configs.is_half and str(self.configs.device)!="cpu": - self.vits_model = self.vits_model.half() - - - def init_t2s_weights(self, weights_path: str): - print(f"Loading Text2Semantic weights from {weights_path}") - self.configs.t2s_weights_path = weights_path - self.configs.save_configs() - self.configs.hz = 50 - dict_s1 = torch.load(weights_path, map_location=self.configs.device) - config = dict_s1["config"] - self.configs.max_sec = config["data"]["max_sec"] - t2s_model = Text2SemanticLightningModule(config, "****", is_train=False) - t2s_model.load_state_dict(dict_s1["weight"]) - t2s_model = t2s_model.to(self.configs.device) - t2s_model = t2s_model.eval() - self.t2s_model = t2s_model - if self.configs.is_half and str(self.configs.device)!="cpu": - self.t2s_model = self.t2s_model.half() - - def enable_half_precision(self, enable: bool = True, save: bool = True): - ''' - To enable half precision for the TTS model. - Args: - enable: bool, whether to enable half precision. - - ''' - if str(self.configs.device) == "cpu" and enable: - print("Half precision is not supported on CPU.") - return - - self.configs.is_half = enable - self.precision = torch.float16 if enable else torch.float32 - if save: - self.configs.save_configs() - if enable: - if self.t2s_model is not None: - self.t2s_model =self.t2s_model.half() - if self.vits_model is not None: - self.vits_model = self.vits_model.half() - if self.bert_model is not None: - self.bert_model =self.bert_model.half() - if self.cnhuhbert_model is not None: - self.cnhuhbert_model = self.cnhuhbert_model.half() - else: - if self.t2s_model is not None: - self.t2s_model = self.t2s_model.float() - if self.vits_model is not None: - self.vits_model = self.vits_model.float() - if self.bert_model is not None: - self.bert_model = self.bert_model.float() - if self.cnhuhbert_model is not None: - self.cnhuhbert_model = self.cnhuhbert_model.float() - - def set_device(self, device: torch.device, save: bool = True): - ''' - To set the device for all models. - Args: - device: torch.device, the device to use for all models. - ''' - self.configs.device = device - if save: - self.configs.save_configs() - if self.t2s_model is not None: - self.t2s_model = self.t2s_model.to(device) - if self.vits_model is not None: - self.vits_model = self.vits_model.to(device) - if self.bert_model is not None: - self.bert_model = self.bert_model.to(device) - if self.cnhuhbert_model is not None: - self.cnhuhbert_model = self.cnhuhbert_model.to(device) - - def set_ref_audio(self, ref_audio_path:str): - ''' - To set the reference audio for the TTS model, - including the prompt_semantic and refer_spepc. - Args: - ref_audio_path: str, the path of the reference audio. - ''' - self._set_prompt_semantic(ref_audio_path) - self._set_ref_spec(ref_audio_path) - self._set_ref_audio_path(ref_audio_path) - - def _set_ref_audio_path(self, ref_audio_path): - self.prompt_cache["ref_audio_path"] = ref_audio_path - - def _set_ref_spec(self, ref_audio_path): - audio = load_audio(ref_audio_path, int(self.configs.sampling_rate)) - audio = torch.FloatTensor(audio) - audio_norm = audio - audio_norm = audio_norm.unsqueeze(0) - spec = spectrogram_torch( - audio_norm, - self.configs.filter_length, - self.configs.sampling_rate, - self.configs.hop_length, - self.configs.win_length, - center=False, - ) - spec = spec.to(self.configs.device) - if self.configs.is_half: - spec = spec.half() - # self.refer_spec = spec - self.prompt_cache["refer_spec"] = spec - - - def _set_prompt_semantic(self, ref_wav_path:str): - zero_wav = np.zeros( - int(self.configs.sampling_rate * 0.3), - dtype=np.float16 if self.configs.is_half else np.float32, - ) - with torch.no_grad(): - wav16k, sr = librosa.load(ref_wav_path, sr=16000) - if (wav16k.shape[0] > 160000 or wav16k.shape[0] < 48000): - raise OSError(i18n("参考音频在3~10秒范围外，请更换！")) - wav16k = torch.from_numpy(wav16k) - zero_wav_torch = torch.from_numpy(zero_wav) - wav16k = wav16k.to(self.configs.device) - zero_wav_torch = zero_wav_torch.to(self.configs.device) - if self.configs.is_half: - wav16k = wav16k.half() - zero_wav_torch = zero_wav_torch.half() - - wav16k = torch.cat([wav16k, zero_wav_torch]) - hubert_feature = self.cnhuhbert_model.model(wav16k.unsqueeze(0))[ - "last_hidden_state" - ].transpose( - 1, 2 - ) # .float() - codes = self.vits_model.extract_latent(hubert_feature) - - prompt_semantic = codes[0, 0].to(self.configs.device) - self.prompt_cache["prompt_semantic"] = prompt_semantic - - def batch_sequences(self, sequences: List[torch.Tensor], axis: int = 0, pad_value: int = 0, max_length:int=None): - seq = sequences[0] - ndim = seq.dim() - if axis < 0: - axis += ndim - dtype:torch.dtype = seq.dtype - pad_value = torch.tensor(pad_value, dtype=dtype) - seq_lengths = [seq.shape[axis] for seq in sequences] - if max_length is None: - max_length = max(seq_lengths) - else: - max_length = max(seq_lengths) if max_length < max(seq_lengths) else max_length - - padded_sequences = [] - for seq, length in zip(sequences, seq_lengths): - padding = [0] * axis + [0, max_length - length] + [0] * (ndim - axis - 1) - padded_seq = torch.nn.functional.pad(seq, padding, value=pad_value) - padded_sequences.append(padded_seq) - batch = torch.stack(padded_sequences) - return batch - - def to_batch(self, data:list, - prompt_data:dict=None, - batch_size:int=5, - threshold:float=0.75, - split_bucket:bool=True, - device:torch.device=torch.device("cpu"), - precision:torch.dtype=torch.float32, - ): - _data:list = [] - index_and_len_list = [] - for idx, item in enumerate(data): - norm_text_len = len(item["norm_text"]) - index_and_len_list.append([idx, norm_text_len]) - - batch_index_list = [] - if split_bucket: - index_and_len_list.sort(key=lambda x: x[1]) - index_and_len_list = np.array(index_and_len_list, dtype=np.int64) - - batch_index_list_len = 0 - pos = 0 - while pos =threshold) or (pos_end-pos==1): - batch_index=index_and_len_list[pos:pos_end, 0].tolist() - batch_index_list_len += len(batch_index) - batch_index_list.append(batch_index) - pos = pos_end - break - pos_end=pos_end-1 - - assert batch_index_list_len == len(data) - - else: - for i in range(len(data)): - if i%batch_size == 0: - batch_index_list.append([]) - batch_index_list[-1].append(i) - - - for batch_idx, index_list in enumerate(batch_index_list): - item_list = [data[idx] for idx in index_list] - phones_list = [] - phones_len_list = [] - # bert_features_list = [] - all_phones_list = [] - all_phones_len_list = [] - all_bert_features_list = [] - norm_text_batch = [] - bert_max_len = 0 - phones_max_len = 0 - for item in item_list: - if prompt_data is not None: - all_bert_features = torch.cat([prompt_data["bert_features"], item["bert_features"]], 1)\ - .to(dtype=precision, device=device) - all_phones = torch.LongTensor(prompt_data["phones"]+item["phones"]).to(device) - phones = torch.LongTensor(item["phones"]).to(device) - # norm_text = prompt_data["norm_text"]+item["norm_text"] - else: - all_bert_features = item["bert_features"]\ - .to(dtype=precision, device=device) - phones = torch.LongTensor(item["phones"]).to(device) - all_phones = phones - # norm_text = item["norm_text"] - - bert_max_len = max(bert_max_len, all_bert_features.shape[-1]) - phones_max_len = max(phones_max_len, phones.shape[-1]) - - phones_list.append(phones) - phones_len_list.append(phones.shape[-1]) - all_phones_list.append(all_phones) - all_phones_len_list.append(all_phones.shape[-1]) - all_bert_features_list.append(all_bert_features) - norm_text_batch.append(item["norm_text"]) - - phones_batch = phones_list - all_phones_batch = all_phones_list - all_bert_features_batch = all_bert_features_list - - - max_len = max(bert_max_len, phones_max_len) - # phones_batch = self.batch_sequences(phones_list, axis=0, pad_value=0, max_length=max_len) - #### 直接对phones和bert_features进行pad。（padding策略会影响T2S模型生成的结果，但不直接影响复读概率。影响复读概率的主要因素是mask的策略） - # all_phones_batch = self.batch_sequences(all_phones_list, axis=0, pad_value=0, max_length=max_len) - # all_bert_features_batch = all_bert_features_list - # all_bert_features_batch = torch.zeros((len(all_bert_features_list), 1024, max_len), dtype=precision, device=device) - # for idx, item in enumerate(all_bert_features_list): - # all_bert_features_batch[idx, :, : item.shape[-1]] = item - - # #### 先对phones进行embedding、对bert_features进行project，再pad到相同长度，（padding策略会影响T2S模型生成的结果，但不直接影响复读概率。影响复读概率的主要因素是mask的策略） - # all_phones_list = [self.t2s_model.model.ar_text_embedding(item.to(self.t2s_model.device)) for item in all_phones_list] - # all_phones_list = [F.pad(item,(0,0,0,max_len-item.shape[0]),value=0) for item in all_phones_list] - # all_phones_batch = torch.stack(all_phones_list, dim=0) - - # all_bert_features_list = [self.t2s_model.model.bert_proj(item.to(self.t2s_model.device).transpose(0, 1)) for item in all_bert_features_list] - # all_bert_features_list = [F.pad(item,(0,0,0,max_len-item.shape[0]), value=0) for item in all_bert_features_list] - # all_bert_features_batch = torch.stack(all_bert_features_list, dim=0) - - batch = { - "phones": phones_batch, - "phones_len": torch.LongTensor(phones_len_list).to(device), - "all_phones": all_phones_batch, - "all_phones_len": torch.LongTensor(all_phones_len_list).to(device), - "all_bert_features": all_bert_features_batch, - "norm_text": norm_text_batch, - "max_len": max_len, - } - _data.append(batch) - - return _data, batch_index_list - - def recovery_order(self, data:list, batch_index_list:list)->list: - ''' - Recovery the order of the audio according to the batch_index_list. - - Args: - data (List[list(np.ndarray)]): the out of order audio . - batch_index_list (List[list[int]]): the batch index list. - - Returns: - list (List[np.ndarray]): the data in the original order. - ''' - length = len(sum(batch_index_list, [])) - _data = [None]*length - for i, index_list in enumerate(batch_index_list): - for j, index in enumerate(index_list): - _data[index] = data[i][j] - return _data - - def stop(self,): - ''' - Stop the inference process. - ''' - self.stop_flag = True - - @torch.no_grad() - def run(self, inputs:dict): - """ - Text to speech inference. - - Args: - inputs (dict): - { - "text": "", # str.(required) text to be synthesized - "text_lang: "", # str.(required) language of the text to be synthesized - "ref_audio_path": "", # str.(required) reference audio path - "prompt_text": "", # str.(optional) prompt text for the reference audio - "prompt_lang": "", # str.(required) language of the prompt text for the reference audio - "top_k": 5, # int. top k sampling - "top_p": 1, # float. top p sampling - "temperature": 1, # float. temperature for sampling - "text_split_method": "cut0", # str. text split method, see text_segmentation_method.py for details. - "batch_size": 1, # int. batch size for inference - "batch_threshold": 0.75, # float. threshold for batch splitting. - "split_bucket: True, # bool. whether to split the batch into multiple buckets. - "return_fragment": False, # bool. step by step return the audio fragment. - "speed_factor":1.0, # float. control the speed of the synthesized audio. - "fragment_interval":0.3, # float. to control the interval of the audio fragment. - "seed": -1, # int. random seed for reproducibility. - "parallel_infer": True, # bool. whether to use parallel inference. - "repetition_penalty": 1.35 # float. repetition penalty for T2S model. - } - returns: - Tuple[int, np.ndarray]: sampling rate and audio data. - """ - ########## variables initialization ########### - self.stop_flag:bool = False - text:str = inputs.get("text", "") - text_lang:str = inputs.get("text_lang", "") - ref_audio_path:str = inputs.get("ref_audio_path", "") - prompt_text:str = inputs.get("prompt_text", "") - prompt_lang:str = inputs.get("prompt_lang", "") - top_k:int = inputs.get("top_k", 5) - top_p:float = inputs.get("top_p", 1) - temperature:float = inputs.get("temperature", 1) - text_split_method:str = inputs.get("text_split_method", "cut0") - batch_size = inputs.get("batch_size", 1) - batch_threshold = inputs.get("batch_threshold", 0.75) - speed_factor = inputs.get("speed_factor", 1.0) - split_bucket = inputs.get("split_bucket", True) - return_fragment = inputs.get("return_fragment", False) - fragment_interval = inputs.get("fragment_interval", 0.3) - seed = inputs.get("seed", -1) - seed = -1 if seed in ["", None] else seed - actual_seed = set_seed(seed) - parallel_infer = inputs.get("parallel_infer", True) - repetition_penalty = inputs.get("repetition_penalty", 1.35) - - if parallel_infer: - print(i18n("并行推理模式已开启")) - self.t2s_model.model.infer_panel = self.t2s_model.model.infer_panel_batch_infer_with_flash_attn - else: - print(i18n("并行推理模式已关闭")) - self.t2s_model.model.infer_panel = self.t2s_model.model.infer_panel_0307 - - if return_fragment: - print(i18n("分段返回模式已开启")) - if split_bucket: - split_bucket = False - print(i18n("分段返回模式不支持分桶处理，已自动关闭分桶处理")) - - if split_bucket: - print(i18n("分桶处理模式已开启")) - - if fragment_interval<0.01: - fragment_interval = 0.01 - print(i18n("分段间隔过小，已自动设置为0.01")) - - no_prompt_text = False - if prompt_text in [None, ""]: - no_prompt_text = True - - assert text_lang in self.configs.languages - if not no_prompt_text: - assert prompt_lang in self.configs.languages - - if ref_audio_path in [None, ""] and \ - ((self.prompt_cache["prompt_semantic"] is None) or (self.prompt_cache["refer_spec"] is None)): - raise ValueError("ref_audio_path cannot be empty, when the reference audio is not set using set_ref_audio()") - - ###### setting reference audio and prompt text preprocessing ######## - t0 = ttime() - if (ref_audio_path is not None) and (ref_audio_path != self.prompt_cache["ref_audio_path"]): - self.set_ref_audio(ref_audio_path) - - if not no_prompt_text: - prompt_text = prompt_text.strip("\n") - if (prompt_text[-1] not in splits): prompt_text += "。" if prompt_lang != "en" else "." - print(i18n("实际输入的参考文本:"), prompt_text) - if self.prompt_cache["prompt_text"] != prompt_text: - self.prompt_cache["prompt_text"] = prompt_text - self.prompt_cache["prompt_lang"] = prompt_lang - phones, bert_features, norm_text = \ - self.text_preprocessor.segment_and_extract_feature_for_text( - prompt_text, - prompt_lang) - self.prompt_cache["phones"] = phones - self.prompt_cache["bert_features"] = bert_features - self.prompt_cache["norm_text"] = norm_text - - ###### text preprocessing ######## - t1 = ttime() - data:list = None - if not return_fragment: - data = self.text_preprocessor.preprocess(text, text_lang, text_split_method) - if len(data) == 0: - yield self.configs.sampling_rate, np.zeros(int(self.configs.sampling_rate), - dtype=np.int16) - return - - batch_index_list:list = None - data, batch_index_list = self.to_batch(data, - prompt_data=self.prompt_cache if not no_prompt_text else None, - batch_size=batch_size, - threshold=batch_threshold, - split_bucket=split_bucket, - device=self.configs.device, - precision=self.precision - ) - else: - print(i18n("############ 切分文本 ############")) - texts = self.text_preprocessor.pre_seg_text(text, text_lang, text_split_method) - data = [] - for i in range(len(texts)): - if i%batch_size == 0: - data.append([]) - data[-1].append(texts[i]) - - def make_batch(batch_texts): - batch_data = [] - print(i18n("############ 提取文本Bert特征 ############")) - for text in tqdm(batch_texts): - phones, bert_features, norm_text = self.text_preprocessor.segment_and_extract_feature_for_text(text, text_lang) - if phones is None: - continue - res={ - "phones": phones, - "bert_features": bert_features, - "norm_text": norm_text, - } - batch_data.append(res) - if len(batch_data) == 0: - return None - batch, _ = self.to_batch(batch_data, - prompt_data=self.prompt_cache if not no_prompt_text else None, - batch_size=batch_size, - threshold=batch_threshold, - split_bucket=False, - device=self.configs.device, - precision=self.precision - ) - return batch[0] - - - t2 = ttime() - try: - print("############ 推理 ############") - ###### inference ###### - t_34 = 0.0 - t_45 = 0.0 - audio = [] - for item in data: - t3 = ttime() - if return_fragment: - item = make_batch(item) - if item is None: - continue - - batch_phones:List[torch.LongTensor] = item["phones"] - # batch_phones:torch.LongTensor = item["phones"] - batch_phones_len:torch.LongTensor = item["phones_len"] - all_phoneme_ids:torch.LongTensor = item["all_phones"] - all_phoneme_lens:torch.LongTensor = item["all_phones_len"] - all_bert_features:torch.LongTensor = item["all_bert_features"] - norm_text:str = item["norm_text"] - max_len = item["max_len"] - - print(i18n("前端处理后的文本(每句):"), norm_text) - if no_prompt_text : - prompt = None - else: - prompt = self.prompt_cache["prompt_semantic"].expand(len(all_phoneme_ids), -1).to(self.configs.device) - - - pred_semantic_list, idx_list = self.t2s_model.model.infer_panel( - all_phoneme_ids, - all_phoneme_lens, - prompt, - all_bert_features, - # prompt_phone_len=ph_offset, - top_k=top_k, - top_p=top_p, - temperature=temperature, - early_stop_num=self.configs.hz * self.configs.max_sec, - max_len=max_len, - repetition_penalty=repetition_penalty, - ) - t4 = ttime() - t_34 += t4 - t3 - - refer_audio_spec:torch.Tensor = self.prompt_cache["refer_spec"]\ - .to(dtype=self.precision, device=self.configs.device) - - batch_audio_fragment = [] - - # 这里要记得加 torch.no_grad() 不然速度慢一大截 - # with torch.no_grad(): - - # ## vits并行推理 method 1 - # pred_semantic_list = [item[-idx:] for item, idx in zip(pred_semantic_list, idx_list)] - # pred_semantic_len = torch.LongTensor([item.shape[0] for item in pred_semantic_list]).to(self.configs.device) - # pred_semantic = self.batch_sequences(pred_semantic_list, axis=0, pad_value=0).unsqueeze(0) - # max_len = 0 - # for i in range(0, len(batch_phones)): - # max_len = max(max_len, batch_phones[i].shape[-1]) - # batch_phones = self.batch_sequences(batch_phones, axis=0, pad_value=0, max_length=max_len) - # batch_phones = batch_phones.to(self.configs.device) - # batch_audio_fragment = (self.vits_model.batched_decode( - # pred_semantic, pred_semantic_len, batch_phones, batch_phones_len,refer_audio_spec - # )) - - # ## vits并行推理 method 2 - pred_semantic_list = [item[-idx:] for item, idx in zip(pred_semantic_list, idx_list)] - upsample_rate = math.prod(self.vits_model.upsample_rates) - audio_frag_idx = [pred_semantic_list[i].shape[0]*2*upsample_rate for i in range(0, len(pred_semantic_list))] - audio_frag_end_idx = [ sum(audio_frag_idx[:i+1]) for i in range(0, len(audio_frag_idx))] - all_pred_semantic = torch.cat(pred_semantic_list).unsqueeze(0).unsqueeze(0).to(self.configs.device) - _batch_phones = torch.cat(batch_phones).unsqueeze(0).to(self.configs.device) - _batch_audio_fragment = (self.vits_model.decode( - all_pred_semantic, _batch_phones, refer_audio_spec - ).detach()[0, 0, :]) - audio_frag_end_idx.insert(0, 0) - batch_audio_fragment= [_batch_audio_fragment[audio_frag_end_idx[i-1]:audio_frag_end_idx[i]] for i in range(1, len(audio_frag_end_idx))] - - # ## vits串行推理 - # for i, idx in enumerate(idx_list): - # phones = batch_phones[i].unsqueeze(0).to(self.configs.device) - # _pred_semantic = (pred_semantic_list[i][-idx:].unsqueeze(0).unsqueeze(0)) # .unsqueeze(0)#mq要多unsqueeze一次 - # audio_fragment =(self.vits_model.decode( - # _pred_semantic, phones, refer_audio_spec - # ).detach()[0, 0, :]) - # batch_audio_fragment.append( - # audio_fragment - # ) ###试试重建不带上prompt部分 - - t5 = ttime() - t_45 += t5 - t4 - if return_fragment: - print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t4 - t3, t5 - t4)) - yield self.audio_postprocess([batch_audio_fragment], - self.configs.sampling_rate, - None, - speed_factor, - False, - fragment_interval - ) - else: - audio.append(batch_audio_fragment) - - if self.stop_flag: - yield self.configs.sampling_rate, np.zeros(int(self.configs.sampling_rate), - dtype=np.int16) - return - - if not return_fragment: - print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t_34, t_45)) - yield self.audio_postprocess(audio, - self.configs.sampling_rate, - batch_index_list, - speed_factor, - split_bucket, - fragment_interval - ) - - except Exception as e: - traceback.print_exc() - # 必须返回一个空音频, 否则会导致显存不释放。 - yield self.configs.sampling_rate, np.zeros(int(self.configs.sampling_rate), - dtype=np.int16) - # 重置模型, 否则会导致显存释放不完全。 - del self.t2s_model - del self.vits_model - self.t2s_model = None - self.vits_model = None - self.init_t2s_weights(self.configs.t2s_weights_path) - self.init_vits_weights(self.configs.vits_weights_path) - raise e - finally: - self.empty_cache() - - def empty_cache(self): - try: - gc.collect() # 触发gc的垃圾回收。避免内存一直增长。 - if "cuda" in str(self.configs.device): - torch.cuda.empty_cache() - elif str(self.configs.device) == "mps": - torch.mps.empty_cache() - except: - pass - - def audio_postprocess(self, - audio:List[torch.Tensor], - sr:int, - batch_index_list:list=None, - speed_factor:float=1.0, - split_bucket:bool=True, - fragment_interval:float=0.3 - )->Tuple[int, np.ndarray]: - zero_wav = torch.zeros( - int(self.configs.sampling_rate * fragment_interval), - dtype=self.precision, - device=self.configs.device - ) - - for i, batch in enumerate(audio): - for j, audio_fragment in enumerate(batch): - max_audio=torch.abs(audio_fragment).max()#简单防止16bit爆音 - if max_audio>1: audio_fragment/=max_audio - audio_fragment:torch.Tensor = torch.cat([audio_fragment, zero_wav], dim=0) - audio[i][j] = audio_fragment.cpu().numpy() - - - if split_bucket: - audio = self.recovery_order(audio, batch_index_list) - else: - # audio = [item for batch in audio for item in batch] - audio = sum(audio, []) - - - audio = np.concatenate(audio, 0) - audio = (audio * 32768).astype(np.int16) - - try: - if speed_factor != 1.0: - audio = speed_change(audio, speed=speed_factor, sr=int(sr)) - except Exception as e: - print(f"Failed to change speed of audio: \n{e}") - - return sr, audio - - - - -def speed_change(input_audio:np.ndarray, speed:float, sr:int): - # 将 NumPy 数组转换为原始 PCM 流 - raw_audio = input_audio.astype(np.int16).tobytes() - - # 设置 ffmpeg 输入流 - input_stream = ffmpeg.input('pipe:', format='s16le', acodec='pcm_s16le', ar=str(sr), ac=1) - - # 变速处理 - output_stream = input_stream.filter('atempo', speed) - - # 输出流到管道 - out, _ = ( - output_stream.output('pipe:', format='s16le', acodec='pcm_s16le') - .run(input=raw_audio, capture_stdout=True, capture_stderr=True) - ) - - # 将管道输出解码为 NumPy 数组 - processed_audio = np.frombuffer(out, np.int16) - - return processed_audio diff --git a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/TextPreprocessor.py b/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/TextPreprocessor.py deleted file mode 100644 index 0891227..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/TextPreprocessor.py +++ /dev/null @@ -1,234 +0,0 @@ - -import os, sys - -from tqdm import tqdm -now_dir = os.getcwd() -sys.path.append(now_dir) - -import re -import torch -import LangSegment - -from typing import Dict, List, Tuple -from text.cleaner import clean_text -from text import cleaned_text_to_sequence -from transformers import AutoModelForMaskedLM, AutoTokenizer -from TTS_infer_pack.text_segmentation_method import split_big_text, splits, get_method as get_seg_method - -from tools.i18n.i18n import I18nAuto - -i18n = I18nAuto() -punctuation = set(['!', '?', '…', ',', '.', '-'," "]) - -def get_first(text:str) -> str: - pattern = "[" + "".join(re.escape(sep) for sep in splits) + "]" - text = re.split(pattern, text)[0].strip() - return text - -def merge_short_text_in_array(texts:str, threshold:int) -> list: - if (len(texts)) < 2: - return texts - result = [] - text = "" - for ele in texts: - text += ele - if len(text) >= threshold: - result.append(text) - text = "" - if (len(text) > 0): - if len(result) == 0: - result.append(text) - else: - result[len(result) - 1] += text - return result - - - - - - -class TextPreprocessor: - def __init__(self, bert_model:AutoModelForMaskedLM, - tokenizer:AutoTokenizer, device:torch.device): - self.bert_model = bert_model - self.tokenizer = tokenizer - self.device = device - - def preprocess(self, text:str, lang:str, text_split_method:str)->List[Dict]: - print(i18n("############ 切分文本 ############")) - text = self.replace_consecutive_punctuation(text) # 变量命名应该是写错了 - texts = self.pre_seg_text(text, lang, text_split_method) - result = [] - print(i18n("############ 提取文本Bert特征 ############")) - for text in tqdm(texts): - phones, bert_features, norm_text = self.segment_and_extract_feature_for_text(text, lang) - if phones is None: - continue - res={ - "phones": phones, - "bert_features": bert_features, - "norm_text": norm_text, - } - result.append(res) - return result - - def pre_seg_text(self, text:str, lang:str, text_split_method:str): - text = text.strip("\n") - if (text[0] not in splits and len(get_first(text)) < 4): - text = "。" + text if lang != "en" else "." + text - print(i18n("实际输入的目标文本:")) - print(text) - - seg_method = get_seg_method(text_split_method) - text = seg_method(text) - - while "\n\n" in text: - text = text.replace("\n\n", "\n") - - _texts = text.split("\n") - _texts = self.process_text(_texts) - _texts = merge_short_text_in_array(_texts, 5) - texts = [] - - - for text in _texts: - # 解决输入目标文本的空行导致报错的问题 - if (len(text.strip()) == 0): - continue - if not re.sub("\W+", "", text): - # 检测一下，如果是纯符号，就跳过。 - continue - if (text[-1] not in splits): text += "。" if lang != "en" else "." - - # 解决句子过长导致Bert报错的问题 - if (len(text) > 510): - texts.extend(split_big_text(text)) - else: - texts.append(text) - - print(i18n("实际输入的目标文本(切句后):")) - print(texts) - return texts - - def segment_and_extract_feature_for_text(self, texts:list, language:str)->Tuple[list, torch.Tensor, str]: - textlist, langlist = self.seg_text(texts, language) - if len(textlist) == 0: - return None, None, None - - phones, bert_features, norm_text = self.extract_bert_feature(textlist, langlist) - return phones, bert_features, norm_text - - - def seg_text(self, text:str, language:str)->Tuple[list, list]: - - textlist=[] - langlist=[] - if language in ["auto", "zh", "ja"]: - LangSegment.setfilters(["zh","ja","en","ko"]) - for tmp in LangSegment.getTexts(text): - if tmp["text"] == "": - continue - if tmp["lang"] == "ko": - langlist.append("zh") - elif tmp["lang"] == "en": - langlist.append("en") - else: - # 因无法区别中日文汉字,以用户输入为准 - langlist.append(language if language!="auto" else tmp["lang"]) - textlist.append(tmp["text"]) - elif language == "en": - LangSegment.setfilters(["en"]) - formattext = " ".join(tmp["text"] for tmp in LangSegment.getTexts(text)) - while " " in formattext: - formattext = formattext.replace(" ", " ") - if formattext != "": - textlist.append(formattext) - langlist.append("en") - - elif language in ["all_zh","all_ja"]: - - formattext = text - while " " in formattext: - formattext = formattext.replace(" ", " ") - language = language.replace("all_","") - if text == "": - return [],[] - textlist.append(formattext) - langlist.append(language) - - else: - raise ValueError(f"language {language} not supported") - - return textlist, langlist - - - def extract_bert_feature(self, textlist:list, langlist:list): - phones_list = [] - bert_feature_list = [] - norm_text_list = [] - for i in range(len(textlist)): - lang = langlist[i] - phones, word2ph, norm_text = self.clean_text_inf(textlist[i], lang) - _bert_feature = self.get_bert_inf(phones, word2ph, norm_text, lang) - # phones_list.append(phones) - phones_list.extend(phones) - norm_text_list.append(norm_text) - bert_feature_list.append(_bert_feature) - bert_feature = torch.cat(bert_feature_list, dim=1) - # phones = sum(phones_list, []) - norm_text = ''.join(norm_text_list) - return phones_list, bert_feature, norm_text - - - def get_bert_feature(self, text:str, word2ph:list)->torch.Tensor: - with torch.no_grad(): - inputs = self.tokenizer(text, return_tensors="pt") - for i in inputs: - inputs[i] = inputs[i].to(self.device) - res = self.bert_model(**inputs, output_hidden_states=True) - res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] - assert len(word2ph) == len(text) - phone_level_feature = [] - for i in range(len(word2ph)): - repeat_feature = res[i].repeat(word2ph[i], 1) - phone_level_feature.append(repeat_feature) - phone_level_feature = torch.cat(phone_level_feature, dim=0) - return phone_level_feature.T - - def clean_text_inf(self, text:str, language:str): - phones, word2ph, norm_text = clean_text(text, language) - phones = cleaned_text_to_sequence(phones) - return phones, word2ph, norm_text - - def get_bert_inf(self, phones:list, word2ph:list, norm_text:str, language:str): - language=language.replace("all_","") - if language == "zh": - feature = self.get_bert_feature(norm_text, word2ph).to(self.device) - else: - feature = torch.zeros( - (1024, len(phones)), - dtype=torch.float32, - ).to(self.device) - - return feature - - def process_text(self,texts): - _text=[] - if all(text in [None, " ", "\n",""] for text in texts): - raise ValueError(i18n("请输入有效文本")) - for text in texts: - if text in [None, " ", ""]: - pass - else: - _text.append(text) - return _text - - - def replace_consecutive_punctuation(self,text): - punctuations = ''.join(re.escape(p) for p in punctuation) - pattern = f'([{punctuations}])([{punctuations}])+' - result = re.sub(pattern, r'\1', text) - return result - - - diff --git a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/__init__.py b/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/__init__.py deleted file mode 100644 index 7438198..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import TTS, text_segmentation_method \ No newline at end of file diff --git a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/text_segmentation_method.py b/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/text_segmentation_method.py deleted file mode 100644 index 33c6e5d..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/TTS_infer_pack/text_segmentation_method.py +++ /dev/null @@ -1,167 +0,0 @@ - - - - -import re -from typing import Callable -from tools.i18n.i18n import I18nAuto - -i18n = I18nAuto() -punctuation = set(['!', '?', '…', ',', '.', '-'," "]) -METHODS = dict() - -def get_method(name:str)->Callable: - method = METHODS.get(name, None) - if method is None: - raise ValueError(f"Method {name} not found") - return method - -def get_method_names()->list: - return list(METHODS.keys()) - -def register_method(name): - def decorator(func): - METHODS[name] = func - return func - return decorator - -splits = {"，", "。", "？", "！", ",", ".", "?", "!", "~", ":", "：", "—", "…", } - -def split_big_text(text, max_len=510): - # 定义全角和半角标点符号 - punctuation = "".join(splits) - - # 切割文本 - segments = re.split('([' + punctuation + '])', text) - - # 初始化结果列表和当前片段 - result = [] - current_segment = '' - - for segment in segments: - # 如果当前片段加上新的片段长度超过max_len，就将当前片段加入结果列表，并重置当前片段 - if len(current_segment + segment) > max_len: - result.append(current_segment) - current_segment = segment - else: - current_segment += segment - - # 将最后一个片段加入结果列表 - if current_segment: - result.append(current_segment) - - return result - - - -def split(todo_text): - todo_text = todo_text.replace("……", "。").replace("——", "，") - if todo_text[-1] not in splits: - todo_text += "。" - i_split_head = i_split_tail = 0 - len_text = len(todo_text) - todo_texts = [] - while 1: - if i_split_head >= len_text: - break # 结尾一定有标点，所以直接跳出即可，最后一段在上次已加入 - if todo_text[i_split_head] in splits: - i_split_head += 1 - todo_texts.append(todo_text[i_split_tail:i_split_head]) - i_split_tail = i_split_head - else: - i_split_head += 1 - return todo_texts - - -# 不切 -@register_method("cut0") -def cut0(inp): - if not set(inp).issubset(punctuation): - return inp - else: - return "/n" - - -# 凑四句一切 -@register_method("cut1") -def cut1(inp): - inp = inp.strip("\n") - inps = split(inp) - split_idx = list(range(0, len(inps), 4)) - # split_idx[-1] = None - split_idx.append(None) - if len(split_idx) > 1: - opts = [] - for idx in range(len(split_idx) - 1): - opts.append("".join(inps[split_idx[idx]: split_idx[idx + 1]])) - else: - opts = [inp] - opts = [item for item in opts if not set(item).issubset(punctuation)] - return "\n".join(opts) - -# 凑50字一切 -@register_method("cut2") -def cut2(inp): - inp = inp.strip("\n") - inps = split(inp) - if len(inps) < 2: - return inp - opts = [] - summ = 0 - tmp_str = "" - for i in range(len(inps)): - summ += len(inps[i]) - tmp_str += inps[i] - if summ > 50: - summ = 0 - opts.append(tmp_str) - tmp_str = "" - if tmp_str != "": - opts.append(tmp_str) - # print(opts) - if len(opts) > 1 and len(opts[-1]) < 50: ##如果最后一个太短了，和前一个合一起 - opts[-2] = opts[-2] + opts[-1] - opts = opts[:-1] - opts = [item for item in opts if not set(item).issubset(punctuation)] - return "\n".join(opts) - -# 按中文句号。切 -@register_method("cut3") -def cut3(inp): - inp = inp.strip("\n") - opts = ["%s" % item for item in inp.strip("。").split("。")] - opts = [item for item in opts if not set(item).issubset(punctuation)] - return "\n".join(opts) - -#按英文句号.切 -@register_method("cut4") -def cut4(inp): - inp = inp.strip("\n") - opts = ["%s" % item for item in inp.strip(".").split(".")] - opts = [item for item in opts if not set(item).issubset(punctuation)] - return "\n".join(opts) - -# 按标点符号切 -# contributed by https://github.com/AI-Hobbyist/GPT-SoVITS/blob/main/GPT_SoVITS/inference_webui.py -@register_method("cut5") -def cut5(inp): - # if not re.search(r'[^\w\s]', inp[-1]): - # inp += '。' - inp = inp.strip("\n") - # punds = r'[,.;?!、，。？！;：…]' - punds = r'[,.;?!、，。？！；：:…]' - items = re.split(f'({punds})', inp) - mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])] - # 在句子不存在符号或句尾无符号的时候保证文本完整 - if len(items)%2 == 1: - mergeitems.append(items[-1]) - opts = [item for item in mergeitems if not set(item).issubset(punctuation)] - opts = "\n".join(opts) - return opts - - - -if __name__ == '__main__': - method = get_method("cut5") - print(method("你好，我是小明。你好，我是小红。你好，我是小刚。你好，我是小张。")) - diff --git a/tts/GPT-SoVITS/GPT_SoVITS/configs/tts_infer.yaml b/tts/GPT-SoVITS/GPT_SoVITS/configs/tts_infer.yaml deleted file mode 100644 index c89c977..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/configs/tts_infer.yaml +++ /dev/null @@ -1,14 +0,0 @@ -custom: - bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large - cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base - device: cuda - is_half: true - t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt - vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth -default: - bert_base_path: GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large - cnhuhbert_base_path: GPT_SoVITS/pretrained_models/chinese-hubert-base - device: cpu - is_half: false - t2s_weights_path: GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt - vits_weights_path: GPT_SoVITS/pretrained_models/s2G488k.pth \ No newline at end of file diff --git a/tts/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py b/tts/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py index 7dffbdb..dc155bd 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/feature_extractor/cnhubert.py @@ -20,16 +20,13 @@ cnhubert_base_path = None class CNHubert(nn.Module): - def __init__(self, base_path:str=None): + def __init__(self): super().__init__() - if base_path is None: - base_path = cnhubert_base_path - self.model = HubertModel.from_pretrained(base_path) + self.model = HubertModel.from_pretrained(cnhubert_base_path) self.feature_extractor = Wav2Vec2FeatureExtractor.from_pretrained( - base_path + cnhubert_base_path ) - def forward(self, x): input_values = self.feature_extractor( x, return_tensors="pt", sampling_rate=16000 diff --git a/tts/GPT-SoVITS/GPT_SoVITS/inference_gui.py b/tts/GPT-SoVITS/GPT_SoVITS/inference_gui.py index 830c66d..f6cfdc5 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/inference_gui.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/inference_gui.py @@ -7,7 +7,7 @@ import soundfile as sf from tools.i18n.i18n import I18nAuto i18n = I18nAuto() -from GPT_SoVITS.inference_webui_old import change_gpt_weights, change_sovits_weights, get_tts_wav +from GPT_SoVITS.inference_webui import change_gpt_weights, change_sovits_weights, get_tts_wav class GPTSoVITSGUI(QMainWindow): diff --git a/tts/GPT-SoVITS/GPT_SoVITS/inference_webui.py b/tts/GPT-SoVITS/GPT_SoVITS/inference_webui.py index 16ac146..4fe8045 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/inference_webui.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/inference_webui.py @@ -6,12 +6,8 @@ 全部按英文识别 全部按日文识别 ''' -import random -import os, sys -now_dir = os.getcwd() -sys.path.append(now_dir) - import os, re, logging +import LangSegment logging.getLogger("markdown_it").setLevel(logging.ERROR) logging.getLogger("urllib3").setLevel(logging.ERROR) logging.getLogger("httpcore").setLevel(logging.ERROR) @@ -22,7 +18,31 @@ logging.getLogger("torchaudio._extension").setLevel(logging.ERROR) import pdb import torch +if os.path.exists("./gweight.txt"): + with open("./gweight.txt", 'r', encoding="utf-8") as file: + gweight_data = file.read() + gpt_path = os.environ.get( + "gpt_path", gweight_data) +else: + gpt_path = os.environ.get( + "gpt_path", "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt") +if os.path.exists("./sweight.txt"): + with open("./sweight.txt", 'r', encoding="utf-8") as file: + sweight_data = file.read() + sovits_path = os.environ.get("sovits_path", sweight_data) +else: + sovits_path = os.environ.get("sovits_path", "GPT_SoVITS/pretrained_models/s2G488k.pth") +# gpt_path = os.environ.get( +# "gpt_path", "pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt" +# ) +# sovits_path = os.environ.get("sovits_path", "pretrained_models/s2G488k.pth") +cnhubert_base_path = os.environ.get( + "cnhubert_base_path", "GPT_SoVITS/pretrained_models/chinese-hubert-base" +) +bert_path = os.environ.get( + "bert_path", "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large" +) infer_ttswebui = os.environ.get("infer_ttswebui", 9872) infer_ttswebui = int(infer_ttswebui) is_share = os.environ.get("is_share", "False") @@ -30,14 +50,21 @@ is_share = eval(is_share) if "_CUDA_VISIBLE_DEVICES" in os.environ: os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available() -gpt_path = os.environ.get("gpt_path", None) -sovits_path = os.environ.get("sovits_path", None) -cnhubert_base_path = os.environ.get("cnhubert_base_path", None) -bert_path = os.environ.get("bert_path", None) - import gradio as gr -from TTS_infer_pack.TTS import TTS, TTS_Config -from TTS_infer_pack.text_segmentation_method import get_method +from transformers import AutoModelForMaskedLM, AutoTokenizer +import numpy as np +import librosa +from feature_extractor import cnhubert + +cnhubert.cnhubert_base_path = cnhubert_base_path + +from module.models import SynthesizerTrn +from AR.models.t2s_lightning_module import Text2SemanticLightningModule +from text import cleaned_text_to_sequence +from text.cleaner import clean_text +from time import time as ttime +from module.mel_processing import spectrogram_torch +from my_utils import load_audio from tools.i18n.i18n import I18nAuto i18n = I18nAuto() @@ -46,11 +73,131 @@ i18n = I18nAuto() if torch.cuda.is_available(): device = "cuda" -# elif torch.backends.mps.is_available(): -# device = "mps" else: device = "cpu" - + +tokenizer = AutoTokenizer.from_pretrained(bert_path) +bert_model = AutoModelForMaskedLM.from_pretrained(bert_path) +if is_half == True: + bert_model = bert_model.half().to(device) +else: + bert_model = bert_model.to(device) + + +def get_bert_feature(text, word2ph): + with torch.no_grad(): + inputs = tokenizer(text, return_tensors="pt") + for i in inputs: + inputs[i] = inputs[i].to(device) + res = bert_model(**inputs, output_hidden_states=True) + res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] + assert len(word2ph) == len(text) + phone_level_feature = [] + for i in range(len(word2ph)): + repeat_feature = res[i].repeat(word2ph[i], 1) + phone_level_feature.append(repeat_feature) + phone_level_feature = torch.cat(phone_level_feature, dim=0) + return phone_level_feature.T + + +class DictToAttrRecursive(dict): + def __init__(self, input_dict): + super().__init__(input_dict) + for key, value in input_dict.items(): + if isinstance(value, dict): + value = DictToAttrRecursive(value) + self[key] = value + setattr(self, key, value) + + def __getattr__(self, item): + try: + return self[item] + except KeyError: + raise AttributeError(f"Attribute {item} not found") + + def __setattr__(self, key, value): + if isinstance(value, dict): + value = DictToAttrRecursive(value) + super(DictToAttrRecursive, self).__setitem__(key, value) + super().__setattr__(key, value) + + def __delattr__(self, item): + try: + del self[item] + except KeyError: + raise AttributeError(f"Attribute {item} not found") + + +ssl_model = cnhubert.get_model() +if is_half == True: + ssl_model = ssl_model.half().to(device) +else: + ssl_model = ssl_model.to(device) + + +def change_sovits_weights(sovits_path): + global vq_model, hps + dict_s2 = torch.load(sovits_path, map_location="cpu") + hps = dict_s2["config"] + hps = DictToAttrRecursive(hps) + hps.model.semantic_frame_rate = "25hz" + vq_model = SynthesizerTrn( + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + n_speakers=hps.data.n_speakers, + **hps.model + ) + if ("pretrained" not in sovits_path): + del vq_model.enc_q + if is_half == True: + vq_model = vq_model.half().to(device) + else: + vq_model = vq_model.to(device) + vq_model.eval() + print(vq_model.load_state_dict(dict_s2["weight"], strict=False)) + with open("./sweight.txt", "w", encoding="utf-8") as f: + f.write(sovits_path) + + +change_sovits_weights(sovits_path) + + +def change_gpt_weights(gpt_path): + global hz, max_sec, t2s_model, config + hz = 50 + dict_s1 = torch.load(gpt_path, map_location="cpu") + config = dict_s1["config"] + max_sec = config["data"]["max_sec"] + t2s_model = Text2SemanticLightningModule(config, "****", is_train=False) + t2s_model.load_state_dict(dict_s1["weight"]) + if is_half == True: + t2s_model = t2s_model.half() + t2s_model = t2s_model.to(device) + t2s_model.eval() + total = sum([param.nelement() for param in t2s_model.parameters()]) + print("Number of parameter: %.2fM" % (total / 1e6)) + with open("./gweight.txt", "w", encoding="utf-8") as f: f.write(gpt_path) + + +change_gpt_weights(gpt_path) + + +def get_spepc(hps, filename): + audio = load_audio(filename, int(hps.data.sampling_rate)) + audio = torch.FloatTensor(audio) + audio_norm = audio + audio_norm = audio_norm.unsqueeze(0) + spec = spectrogram_torch( + audio_norm, + hps.data.filter_length, + hps.data.sampling_rate, + hps.data.hop_length, + hps.data.win_length, + center=False, + ) + return spec + + dict_language = { i18n("中文"): "all_zh",#全部按中文识别 i18n("英文"): "en",#全部按英文识别#######不变 @@ -60,67 +207,314 @@ dict_language = { i18n("多语种混合"): "auto",#多语种启动切分识别语种 } -cut_method = { - i18n("不切"):"cut0", - i18n("凑四句一切"): "cut1", - i18n("凑50字一切"): "cut2", - i18n("按中文句号。切"): "cut3", - i18n("按英文句号.切"): "cut4", - i18n("按标点符号切"): "cut5", -} -tts_config = TTS_Config("GPT_SoVITS/configs/tts_infer.yaml") -tts_config.device = device -tts_config.is_half = is_half -if gpt_path is not None: - tts_config.t2s_weights_path = gpt_path -if sovits_path is not None: - tts_config.vits_weights_path = sovits_path -if cnhubert_base_path is not None: - tts_config.cnhuhbert_base_path = cnhubert_base_path -if bert_path is not None: - tts_config.bert_base_path = bert_path +def clean_text_inf(text, language): + phones, word2ph, norm_text = clean_text(text, language) + phones = cleaned_text_to_sequence(phones) + return phones, word2ph, norm_text + +dtype=torch.float16 if is_half == True else torch.float32 +def get_bert_inf(phones, word2ph, norm_text, language): + language=language.replace("all_","") + if language == "zh": + bert = get_bert_feature(norm_text, word2ph).to(device)#.to(dtype) + else: + bert = torch.zeros( + (1024, len(phones)), + dtype=torch.float16 if is_half == True else torch.float32, + ).to(device) + + return bert + + +splits = {"，", "。", "？", "！", ",", ".", "?", "!", "~", ":", "：", "—", "…", } + + +def get_first(text): + pattern = "[" + "".join(re.escape(sep) for sep in splits) + "]" + text = re.split(pattern, text)[0].strip() + return text + + +def get_phones_and_bert(text,language): + if language in {"en","all_zh","all_ja"}: + language = language.replace("all_","") + if language == "en": + LangSegment.setfilters(["en"]) + formattext = " ".join(tmp["text"] for tmp in LangSegment.getTexts(text)) + else: + # 因无法区别中日文汉字,以用户输入为准 + formattext = text + while " " in formattext: + formattext = formattext.replace(" ", " ") + phones, word2ph, norm_text = clean_text_inf(formattext, language) + if language == "zh": + bert = get_bert_feature(norm_text, word2ph).to(device) + else: + bert = torch.zeros( + (1024, len(phones)), + dtype=torch.float16 if is_half == True else torch.float32, + ).to(device) + elif language in {"zh", "ja","auto"}: + textlist=[] + langlist=[] + LangSegment.setfilters(["zh","ja","en","ko"]) + if language == "auto": + for tmp in LangSegment.getTexts(text): + if tmp["lang"] == "ko": + langlist.append("zh") + textlist.append(tmp["text"]) + else: + langlist.append(tmp["lang"]) + textlist.append(tmp["text"]) + else: + for tmp in LangSegment.getTexts(text): + if tmp["lang"] == "en": + langlist.append(tmp["lang"]) + else: + # 因无法区别中日文汉字,以用户输入为准 + langlist.append(language) + textlist.append(tmp["text"]) + print(textlist) + print(langlist) + phones_list = [] + bert_list = [] + norm_text_list = [] + for i in range(len(textlist)): + lang = langlist[i] + phones, word2ph, norm_text = clean_text_inf(textlist[i], lang) + bert = get_bert_inf(phones, word2ph, norm_text, lang) + phones_list.append(phones) + norm_text_list.append(norm_text) + bert_list.append(bert) + bert = torch.cat(bert_list, dim=1) + phones = sum(phones_list, []) + norm_text = ''.join(norm_text_list) + + return phones,bert.to(dtype),norm_text + + +def merge_short_text_in_array(texts, threshold): + if (len(texts)) < 2: + return texts + result = [] + text = "" + for ele in texts: + text += ele + if len(text) >= threshold: + result.append(text) + text = "" + if (len(text) > 0): + if len(result) == 0: + result.append(text) + else: + result[len(result) - 1] += text + return result + +def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language, how_to_cut=i18n("不切"), top_k=20, top_p=0.6, temperature=0.6, ref_free = False): + if prompt_text is None or len(prompt_text) == 0: + ref_free = True + t0 = ttime() + prompt_language = dict_language[prompt_language] + text_language = dict_language[text_language] + if not ref_free: + prompt_text = prompt_text.strip("\n") + if (prompt_text[-1] not in splits): prompt_text += "。" if prompt_language != "en" else "." + print(i18n("实际输入的参考文本:"), prompt_text) + text = text.strip("\n") + if (text[0] not in splits and len(get_first(text)) < 4): text = "。" + text if text_language != "en" else "." + text -print(tts_config) -tts_pipeline = TTS(tts_config) -gpt_path = tts_config.t2s_weights_path -sovits_path = tts_config.vits_weights_path + print(i18n("实际输入的目标文本:"), text) + zero_wav = np.zeros( + int(hps.data.sampling_rate * 0.3), + dtype=np.float16 if is_half == True else np.float32, + ) + with torch.no_grad(): + wav16k, sr = librosa.load(ref_wav_path, sr=16000) + if (wav16k.shape[0] > 160000 or wav16k.shape[0] < 48000): + raise OSError(i18n("参考音频在3~10秒范围外，请更换！")) + wav16k = torch.from_numpy(wav16k) + zero_wav_torch = torch.from_numpy(zero_wav) + if is_half == True: + wav16k = wav16k.half().to(device) + zero_wav_torch = zero_wav_torch.half().to(device) + else: + wav16k = wav16k.to(device) + zero_wav_torch = zero_wav_torch.to(device) + wav16k = torch.cat([wav16k, zero_wav_torch]) + ssl_content = ssl_model.model(wav16k.unsqueeze(0))[ + "last_hidden_state" + ].transpose( + 1, 2 + ) # .float() + codes = vq_model.extract_latent(ssl_content) + + prompt_semantic = codes[0, 0] + t1 = ttime() + + if (how_to_cut == i18n("凑四句一切")): + text = cut1(text) + elif (how_to_cut == i18n("凑50字一切")): + text = cut2(text) + elif (how_to_cut == i18n("按中文句号。切")): + text = cut3(text) + elif (how_to_cut == i18n("按英文句号.切")): + text = cut4(text) + elif (how_to_cut == i18n("按标点符号切")): + text = cut5(text) + while "\n\n" in text: + text = text.replace("\n\n", "\n") + print(i18n("实际输入的目标文本(切句后):"), text) + texts = text.split("\n") + texts = merge_short_text_in_array(texts, 5) + audio_opt = [] + if not ref_free: + phones1,bert1,norm_text1=get_phones_and_bert(prompt_text, prompt_language) + + for text in texts: + # 解决输入目标文本的空行导致报错的问题 + if (len(text.strip()) == 0): + continue + if (text[-1] not in splits): text += "。" if text_language != "en" else "." + print(i18n("实际输入的目标文本(每句):"), text) + phones2,bert2,norm_text2=get_phones_and_bert(text, text_language) + print(i18n("前端处理后的文本(每句):"), norm_text2) + if not ref_free: + bert = torch.cat([bert1, bert2], 1) + all_phoneme_ids = torch.LongTensor(phones1+phones2).to(device).unsqueeze(0) + else: + bert = bert2 + all_phoneme_ids = torch.LongTensor(phones2).to(device).unsqueeze(0) + + bert = bert.to(device).unsqueeze(0) + all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device) + prompt = prompt_semantic.unsqueeze(0).to(device) + t2 = ttime() + with torch.no_grad(): + # pred_semantic = t2s_model.model.infer( + pred_semantic, idx = t2s_model.model.infer_panel( + all_phoneme_ids, + all_phoneme_len, + None if ref_free else prompt, + bert, + # prompt_phone_len=ph_offset, + top_k=top_k, + top_p=top_p, + temperature=temperature, + early_stop_num=hz * max_sec, + ) + t3 = ttime() + # print(pred_semantic.shape,idx) + pred_semantic = pred_semantic[:, -idx:].unsqueeze( + 0 + ) # .unsqueeze(0)#mq要多unsqueeze一次 + refer = get_spepc(hps, ref_wav_path) # .to(device) + if is_half == True: + refer = refer.half().to(device) + else: + refer = refer.to(device) + # audio = vq_model.decode(pred_semantic, all_phoneme_ids, refer).detach().cpu().numpy()[0, 0] + audio = ( + vq_model.decode( + pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refer + ) + .detach() + .cpu() + .numpy()[0, 0] + ) ###试试重建不带上prompt部分 + max_audio=np.abs(audio).max()#简单防止16bit爆音 + if max_audio>1:audio/=max_audio + audio_opt.append(audio) + audio_opt.append(zero_wav) + t4 = ttime() + print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t3 - t2, t4 - t3)) + yield hps.data.sampling_rate, (np.concatenate(audio_opt, 0) * 32768).astype( + np.int16 + ) + + +def split(todo_text): + todo_text = todo_text.replace("……", "。").replace("——", "，") + if todo_text[-1] not in splits: + todo_text += "。" + i_split_head = i_split_tail = 0 + len_text = len(todo_text) + todo_texts = [] + while 1: + if i_split_head >= len_text: + break # 结尾一定有标点，所以直接跳出即可，最后一段在上次已加入 + if todo_text[i_split_head] in splits: + i_split_head += 1 + todo_texts.append(todo_text[i_split_tail:i_split_head]) + i_split_tail = i_split_head + else: + i_split_head += 1 + return todo_texts + + +def cut1(inp): + inp = inp.strip("\n") + inps = split(inp) + split_idx = list(range(0, len(inps), 4)) + split_idx[-1] = None + if len(split_idx) > 1: + opts = [] + for idx in range(len(split_idx) - 1): + opts.append("".join(inps[split_idx[idx]: split_idx[idx + 1]])) + else: + opts = [inp] + return "\n".join(opts) + + +def cut2(inp): + inp = inp.strip("\n") + inps = split(inp) + if len(inps) < 2: + return inp + opts = [] + summ = 0 + tmp_str = "" + for i in range(len(inps)): + summ += len(inps[i]) + tmp_str += inps[i] + if summ > 50: + summ = 0 + opts.append(tmp_str) + tmp_str = "" + if tmp_str != "": + opts.append(tmp_str) + # print(opts) + if len(opts) > 1 and len(opts[-1]) < 50: ##如果最后一个太短了，和前一个合一起 + opts[-2] = opts[-2] + opts[-1] + opts = opts[:-1] + return "\n".join(opts) + + +def cut3(inp): + inp = inp.strip("\n") + return "\n".join(["%s" % item for item in inp.strip("。").split("。")]) + + +def cut4(inp): + inp = inp.strip("\n") + return "\n".join(["%s" % item for item in inp.strip(".").split(".")]) + + +# contributed by https://github.com/AI-Hobbyist/GPT-SoVITS/blob/main/GPT_SoVITS/inference_webui.py +def cut5(inp): + # if not re.search(r'[^\w\s]', inp[-1]): + # inp += '。' + inp = inp.strip("\n") + punds = r'[,.;?!、，。？！;：…]' + items = re.split(f'({punds})', inp) + mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])] + # 在句子不存在符号或句尾无符号的时候保证文本完整 + if len(items)%2 == 1: + mergeitems.append(items[-1]) + opt = "\n".join(mergeitems) + return opt -def inference(text, text_lang, - ref_audio_path, prompt_text, - prompt_lang, top_k, - top_p, temperature, - text_split_method, batch_size, - speed_factor, ref_text_free, - split_bucket,fragment_interval, - seed, keep_random, parallel_infer, - repetition_penalty - ): - seed = -1 if keep_random else seed - actual_seed = seed if seed not in [-1, "", None] else random.randrange(1 << 32) - inputs={ - "text": text, - "text_lang": dict_language[text_lang], - "ref_audio_path": ref_audio_path, - "prompt_text": prompt_text if not ref_text_free else "", - "prompt_lang": dict_language[prompt_lang], - "top_k": top_k, - "top_p": top_p, - "temperature": temperature, - "text_split_method": cut_method[text_split_method], - "batch_size":int(batch_size), - "speed_factor":float(speed_factor), - "split_bucket":split_bucket, - "return_fragment":False, - "fragment_interval":fragment_interval, - "seed":actual_seed, - "parallel_infer": parallel_infer, - "repetition_penalty": repetition_penalty, - } - for item in tts_pipeline.run(inputs): - yield item, actual_seed - def custom_sort_key(s): # 使用正则表达式提取字符串中的数字部分和非数字部分 parts = re.split('(\d+)', s) @@ -158,108 +552,65 @@ with gr.Blocks(title="GPT-SoVITS WebUI") as app: gr.Markdown( value=i18n("本软件以MIT协议开源, 作者不对软件具备任何控制力, 使用软件者、传播软件导出的声音者自负全责.
如不认可该条款, 则不能使用或引用软件包内任何代码和文件. 详见根目录LICENSE.") ) - - with gr.Column(): - # with gr.Group(): + with gr.Group(): gr.Markdown(value=i18n("模型切换")) with gr.Row(): GPT_dropdown = gr.Dropdown(label=i18n("GPT模型列表"), choices=sorted(GPT_names, key=custom_sort_key), value=gpt_path, interactive=True) SoVITS_dropdown = gr.Dropdown(label=i18n("SoVITS模型列表"), choices=sorted(SoVITS_names, key=custom_sort_key), value=sovits_path, interactive=True) refresh_button = gr.Button(i18n("刷新模型路径"), variant="primary") refresh_button.click(fn=change_choices, inputs=[], outputs=[SoVITS_dropdown, GPT_dropdown]) - SoVITS_dropdown.change(tts_pipeline.init_vits_weights, [SoVITS_dropdown], []) - GPT_dropdown.change(tts_pipeline.init_t2s_weights, [GPT_dropdown], []) - - with gr.Row(): - with gr.Column(): - gr.Markdown(value=i18n("*请上传并填写参考信息")) + SoVITS_dropdown.change(change_sovits_weights, [SoVITS_dropdown], []) + GPT_dropdown.change(change_gpt_weights, [GPT_dropdown], []) + gr.Markdown(value=i18n("*请上传并填写参考信息")) + with gr.Row(): inp_ref = gr.Audio(label=i18n("请上传3~10秒内参考音频，超过会报错！"), type="filepath") - prompt_text = gr.Textbox(label=i18n("参考音频的文本"), value="", lines=2) - with gr.Row(): - prompt_language = gr.Dropdown( - label=i18n("参考音频的语种"), choices=[i18n("中文"), i18n("英文"), i18n("日文"), i18n("中英混合"), i18n("日英混合"), i18n("多语种混合")], value=i18n("中文") - ) - with gr.Column(): - ref_text_free = gr.Checkbox(label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。"), value=False, interactive=True, show_label=True) - gr.Markdown(i18n("使用无参考文本模式时建议使用微调的GPT，听不清参考音频说的啥(不晓得写啥)可以开，开启后无视填写的参考文本。")) - - with gr.Column(): - gr.Markdown(value=i18n("*请填写需要合成的目标文本和语种模式")) - text = gr.Textbox(label=i18n("需要合成的文本"), value="", lines=16, max_lines=16) + with gr.Column(): + ref_text_free = gr.Checkbox(label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。"), value=False, interactive=True, show_label=True) + gr.Markdown(i18n("使用无参考文本模式时建议使用微调的GPT，听不清参考音频说的啥(不晓得写啥)可以开，开启后无视填写的参考文本。")) + prompt_text = gr.Textbox(label=i18n("参考音频的文本"), value="") + prompt_language = gr.Dropdown( + label=i18n("参考音频的语种"), choices=[i18n("中文"), i18n("英文"), i18n("日文"), i18n("中英混合"), i18n("日英混合"), i18n("多语种混合")], value=i18n("中文") + ) + gr.Markdown(value=i18n("*请填写需要合成的目标文本和语种模式")) + with gr.Row(): + text = gr.Textbox(label=i18n("需要合成的文本"), value="") text_language = gr.Dropdown( label=i18n("需要合成的语种"), choices=[i18n("中文"), i18n("英文"), i18n("日文"), i18n("中英混合"), i18n("日英混合"), i18n("多语种混合")], value=i18n("中文") ) - - - with gr.Group(): - gr.Markdown(value=i18n("推理设置")) - with gr.Row(): - - with gr.Column(): - batch_size = gr.Slider(minimum=1,maximum=200,step=1,label=i18n("batch_size"),value=20,interactive=True) - fragment_interval = gr.Slider(minimum=0.01,maximum=1,step=0.01,label=i18n("分段间隔(秒)"),value=0.3,interactive=True) - speed_factor = gr.Slider(minimum=0.25,maximum=4,step=0.05,label="speed_factor",value=1.0,interactive=True) + how_to_cut = gr.Radio( + label=i18n("怎么切"), + choices=[i18n("不切"), i18n("凑四句一切"), i18n("凑50字一切"), i18n("按中文句号。切"), i18n("按英文句号.切"), i18n("按标点符号切"), ], + value=i18n("凑四句一切"), + interactive=True, + ) + with gr.Row(): + gr.Markdown(value=i18n("gpt采样参数(无参考文本时不要太低)：")) top_k = gr.Slider(minimum=1,maximum=100,step=1,label=i18n("top_k"),value=5,interactive=True) top_p = gr.Slider(minimum=0,maximum=1,step=0.05,label=i18n("top_p"),value=1,interactive=True) temperature = gr.Slider(minimum=0,maximum=1,step=0.05,label=i18n("temperature"),value=1,interactive=True) - repetition_penalty = gr.Slider(minimum=0,maximum=2,step=0.05,label=i18n("重复惩罚"),value=1.35,interactive=True) - with gr.Column(): - how_to_cut = gr.Radio( - label=i18n("怎么切"), - choices=[i18n("不切"), i18n("凑四句一切"), i18n("凑50字一切"), i18n("按中文句号。切"), i18n("按英文句号.切"), i18n("按标点符号切"), ], - value=i18n("凑四句一切"), - interactive=True, - ) - with gr.Row(): - parallel_infer = gr.Checkbox(label=i18n("并行推理(速度更快，但可能增大复读概率)"), value=True, interactive=True, show_label=True) - split_bucket = gr.Checkbox(label=i18n("数据分桶(并行推理时会降低一点计算量)"), value=True, interactive=True, show_label=True) - seed = gr.Number(label=i18n("随机种子"),value=-1) - keep_random = gr.Checkbox(label=i18n("保持随机"), value=True, interactive=True, show_label=True) + inference_button = gr.Button(i18n("合成语音"), variant="primary") + output = gr.Audio(label=i18n("输出的语音")) - # with gr.Column(): - output = gr.Audio(label=i18n("输出的语音")) - with gr.Row(): - inference_button = gr.Button(i18n("合成语音"), variant="primary") - stop_infer = gr.Button(i18n("终止合成"), variant="primary") - - inference_button.click( - inference, - [ - text,text_language, inp_ref, - prompt_text, prompt_language, - top_k, top_p, temperature, - how_to_cut, batch_size, - speed_factor, ref_text_free, - split_bucket,fragment_interval, - seed, keep_random, parallel_infer, - repetition_penalty - ], - [output, seed], + get_tts_wav, + [inp_ref, prompt_text, prompt_language, text, text_language, how_to_cut, top_k, top_p, temperature, ref_text_free], + [output], ) - stop_infer.click(tts_pipeline.stop, [], []) - with gr.Group(): gr.Markdown(value=i18n("文本切分工具。太长的文本合成出来效果不一定好，所以太长建议先切。合成会根据文本的换行分开合成再拼起来。")) with gr.Row(): - text_inp = gr.Textbox(label=i18n("需要合成的切分前文本"), value="", lines=4) - with gr.Column(): - _how_to_cut = gr.Radio( - label=i18n("怎么切"), - choices=[i18n("不切"), i18n("凑四句一切"), i18n("凑50字一切"), i18n("按中文句号。切"), i18n("按英文句号.切"), i18n("按标点符号切"), ], - value=i18n("凑四句一切"), - interactive=True, - ) - cut_text= gr.Button(i18n("切分"), variant="primary") - - def to_cut(text_inp, how_to_cut): - if len(text_inp.strip()) == 0 or text_inp==[]: - return "" - method = get_method(cut_method[how_to_cut]) - return method(text_inp) - - text_opt = gr.Textbox(label=i18n("切分后文本"), value="", lines=4) - cut_text.click(to_cut, [text_inp, _how_to_cut], [text_opt]) + text_inp = gr.Textbox(label=i18n("需要合成的切分前文本"), value="") + button1 = gr.Button(i18n("凑四句一切"), variant="primary") + button2 = gr.Button(i18n("凑50字一切"), variant="primary") + button3 = gr.Button(i18n("按中文句号。切"), variant="primary") + button4 = gr.Button(i18n("按英文句号.切"), variant="primary") + button5 = gr.Button(i18n("按标点符号切"), variant="primary") + text_opt = gr.Textbox(label=i18n("切分后文本"), value="") + button1.click(cut1, [text_inp], [text_opt]) + button2.click(cut2, [text_inp], [text_opt]) + button3.click(cut3, [text_inp], [text_opt]) + button4.click(cut4, [text_inp], [text_opt]) + button5.click(cut5, [text_inp], [text_opt]) gr.Markdown(value=i18n("后续将支持转音素、手工修改音素、语音合成分步执行。")) app.queue(concurrency_count=511, max_size=1022).launch( diff --git a/tts/GPT-SoVITS/GPT_SoVITS/inference_webui_old.py b/tts/GPT-SoVITS/GPT_SoVITS/inference_webui_old.py deleted file mode 100644 index ee09962..0000000 --- a/tts/GPT-SoVITS/GPT_SoVITS/inference_webui_old.py +++ /dev/null @@ -1,622 +0,0 @@ -''' -按中英混合识别 -按日英混合识别 -多语种启动切分识别语种 -全部按中文识别 -全部按英文识别 -全部按日文识别 -''' -import os, re, logging -import LangSegment -logging.getLogger("markdown_it").setLevel(logging.ERROR) -logging.getLogger("urllib3").setLevel(logging.ERROR) -logging.getLogger("httpcore").setLevel(logging.ERROR) -logging.getLogger("httpx").setLevel(logging.ERROR) -logging.getLogger("asyncio").setLevel(logging.ERROR) -logging.getLogger("charset_normalizer").setLevel(logging.ERROR) -logging.getLogger("torchaudio._extension").setLevel(logging.ERROR) -import pdb -import torch - -if os.path.exists("./gweight.txt"): - with open("./gweight.txt", 'r', encoding="utf-8") as file: - gweight_data = file.read() - gpt_path = os.environ.get( - "gpt_path", gweight_data) -else: - gpt_path = os.environ.get( - "gpt_path", "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt") - -if os.path.exists("./sweight.txt"): - with open("./sweight.txt", 'r', encoding="utf-8") as file: - sweight_data = file.read() - sovits_path = os.environ.get("sovits_path", sweight_data) -else: - sovits_path = os.environ.get("sovits_path", "GPT_SoVITS/pretrained_models/s2G488k.pth") -# gpt_path = os.environ.get( -# "gpt_path", "pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt" -# ) -# sovits_path = os.environ.get("sovits_path", "pretrained_models/s2G488k.pth") -cnhubert_base_path = os.environ.get( - "cnhubert_base_path", "GPT_SoVITS/pretrained_models/chinese-hubert-base" -) -bert_path = os.environ.get( - "bert_path", "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large" -) -infer_ttswebui = os.environ.get("infer_ttswebui", 9872) -infer_ttswebui = int(infer_ttswebui) -is_share = os.environ.get("is_share", "False") -is_share = eval(is_share) -if "_CUDA_VISIBLE_DEVICES" in os.environ: - os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] -is_half = eval(os.environ.get("is_half", "True")) and not torch.backends.mps.is_available() -import gradio as gr -from transformers import AutoModelForMaskedLM, AutoTokenizer -import numpy as np -import librosa -from feature_extractor import cnhubert - -cnhubert.cnhubert_base_path = cnhubert_base_path - -from module.models import SynthesizerTrn -from AR.models.t2s_lightning_module import Text2SemanticLightningModule -from text import cleaned_text_to_sequence -from text.cleaner import clean_text -from time import time as ttime -from module.mel_processing import spectrogram_torch -from my_utils import load_audio -from tools.i18n.i18n import I18nAuto - -i18n = I18nAuto() - -os.environ['PYTORCH_ENABLE_MPS_FALLBACK'] = '1' # 确保直接启动推理UI时也能够设置。 - -if torch.cuda.is_available(): - device = "cuda" -else: - device = "cpu" - -tokenizer = AutoTokenizer.from_pretrained(bert_path) -bert_model = AutoModelForMaskedLM.from_pretrained(bert_path) -if is_half == True: - bert_model = bert_model.half().to(device) -else: - bert_model = bert_model.to(device) - - -def get_bert_feature(text, word2ph): - with torch.no_grad(): - inputs = tokenizer(text, return_tensors="pt") - for i in inputs: - inputs[i] = inputs[i].to(device) - res = bert_model(**inputs, output_hidden_states=True) - res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] - assert len(word2ph) == len(text) - phone_level_feature = [] - for i in range(len(word2ph)): - repeat_feature = res[i].repeat(word2ph[i], 1) - phone_level_feature.append(repeat_feature) - phone_level_feature = torch.cat(phone_level_feature, dim=0) - return phone_level_feature.T - - -class DictToAttrRecursive(dict): - def __init__(self, input_dict): - super().__init__(input_dict) - for key, value in input_dict.items(): - if isinstance(value, dict): - value = DictToAttrRecursive(value) - self[key] = value - setattr(self, key, value) - - def __getattr__(self, item): - try: - return self[item] - except KeyError: - raise AttributeError(f"Attribute {item} not found") - - def __setattr__(self, key, value): - if isinstance(value, dict): - value = DictToAttrRecursive(value) - super(DictToAttrRecursive, self).__setitem__(key, value) - super().__setattr__(key, value) - - def __delattr__(self, item): - try: - del self[item] - except KeyError: - raise AttributeError(f"Attribute {item} not found") - - -ssl_model = cnhubert.get_model() -if is_half == True: - ssl_model = ssl_model.half().to(device) -else: - ssl_model = ssl_model.to(device) - - -def change_sovits_weights(sovits_path): - global vq_model, hps - dict_s2 = torch.load(sovits_path, map_location="cpu") - hps = dict_s2["config"] - hps = DictToAttrRecursive(hps) - hps.model.semantic_frame_rate = "25hz" - vq_model = SynthesizerTrn( - hps.data.filter_length // 2 + 1, - hps.train.segment_size // hps.data.hop_length, - n_speakers=hps.data.n_speakers, - **hps.model - ) - if ("pretrained" not in sovits_path): - del vq_model.enc_q - if is_half == True: - vq_model = vq_model.half().to(device) - else: - vq_model = vq_model.to(device) - vq_model.eval() - print(vq_model.load_state_dict(dict_s2["weight"], strict=False)) - with open("./sweight.txt", "w", encoding="utf-8") as f: - f.write(sovits_path) - - -change_sovits_weights(sovits_path) - - -def change_gpt_weights(gpt_path): - global hz, max_sec, t2s_model, config - hz = 50 - dict_s1 = torch.load(gpt_path, map_location="cpu") - config = dict_s1["config"] - max_sec = config["data"]["max_sec"] - t2s_model = Text2SemanticLightningModule(config, "****", is_train=False) - t2s_model.load_state_dict(dict_s1["weight"]) - if is_half == True: - t2s_model = t2s_model.half() - t2s_model = t2s_model.to(device) - t2s_model.eval() - total = sum([param.nelement() for param in t2s_model.parameters()]) - print("Number of parameter: %.2fM" % (total / 1e6)) - with open("./gweight.txt", "w", encoding="utf-8") as f: f.write(gpt_path) - - -change_gpt_weights(gpt_path) - - -def get_spepc(hps, filename): - audio = load_audio(filename, int(hps.data.sampling_rate)) - audio = torch.FloatTensor(audio) - audio_norm = audio - audio_norm = audio_norm.unsqueeze(0) - spec = spectrogram_torch( - audio_norm, - hps.data.filter_length, - hps.data.sampling_rate, - hps.data.hop_length, - hps.data.win_length, - center=False, - ) - return spec - - -dict_language = { - i18n("中文"): "all_zh",#全部按中文识别 - i18n("英文"): "en",#全部按英文识别#######不变 - i18n("日文"): "all_ja",#全部按日文识别 - i18n("中英混合"): "zh",#按中英混合识别####不变 - i18n("日英混合"): "ja",#按日英混合识别####不变 - i18n("多语种混合"): "auto",#多语种启动切分识别语种 -} - - -def clean_text_inf(text, language): - phones, word2ph, norm_text = clean_text(text, language) - phones = cleaned_text_to_sequence(phones) - return phones, word2ph, norm_text - -dtype=torch.float16 if is_half == True else torch.float32 -def get_bert_inf(phones, word2ph, norm_text, language): - language=language.replace("all_","") - if language == "zh": - bert = get_bert_feature(norm_text, word2ph).to(device)#.to(dtype) - else: - bert = torch.zeros( - (1024, len(phones)), - dtype=torch.float16 if is_half == True else torch.float32, - ).to(device) - - return bert - - -splits = {"，", "。", "？", "！", ",", ".", "?", "!", "~", ":", "：", "—", "…", } - - -def get_first(text): - pattern = "[" + "".join(re.escape(sep) for sep in splits) + "]" - text = re.split(pattern, text)[0].strip() - return text - - -def get_phones_and_bert(text,language): - if language in {"en","all_zh","all_ja"}: - language = language.replace("all_","") - if language == "en": - LangSegment.setfilters(["en"]) - formattext = " ".join(tmp["text"] for tmp in LangSegment.getTexts(text)) - else: - # 因无法区别中日文汉字,以用户输入为准 - formattext = text - while " " in formattext: - formattext = formattext.replace(" ", " ") - phones, word2ph, norm_text = clean_text_inf(formattext, language) - if language == "zh": - bert = get_bert_feature(norm_text, word2ph).to(device) - else: - bert = torch.zeros( - (1024, len(phones)), - dtype=torch.float16 if is_half == True else torch.float32, - ).to(device) - elif language in {"zh", "ja","auto"}: - textlist=[] - langlist=[] - LangSegment.setfilters(["zh","ja","en","ko"]) - if language == "auto": - for tmp in LangSegment.getTexts(text): - if tmp["lang"] == "ko": - langlist.append("zh") - textlist.append(tmp["text"]) - else: - langlist.append(tmp["lang"]) - textlist.append(tmp["text"]) - else: - for tmp in LangSegment.getTexts(text): - if tmp["lang"] == "en": - langlist.append(tmp["lang"]) - else: - # 因无法区别中日文汉字,以用户输入为准 - langlist.append(language) - textlist.append(tmp["text"]) - print(textlist) - print(langlist) - phones_list = [] - bert_list = [] - norm_text_list = [] - for i in range(len(textlist)): - lang = langlist[i] - phones, word2ph, norm_text = clean_text_inf(textlist[i], lang) - bert = get_bert_inf(phones, word2ph, norm_text, lang) - phones_list.append(phones) - norm_text_list.append(norm_text) - bert_list.append(bert) - bert = torch.cat(bert_list, dim=1) - phones = sum(phones_list, []) - norm_text = ''.join(norm_text_list) - - return phones,bert.to(dtype),norm_text - - -def merge_short_text_in_array(texts, threshold): - if (len(texts)) < 2: - return texts - result = [] - text = "" - for ele in texts: - text += ele - if len(text) >= threshold: - result.append(text) - text = "" - if (len(text) > 0): - if len(result) == 0: - result.append(text) - else: - result[len(result) - 1] += text - return result - -def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language, how_to_cut=i18n("不切"), top_k=20, top_p=0.6, temperature=0.6, ref_free = False): - if prompt_text is None or len(prompt_text) == 0: - ref_free = True - t0 = ttime() - prompt_language = dict_language[prompt_language] - text_language = dict_language[text_language] - if not ref_free: - prompt_text = prompt_text.strip("\n") - if (prompt_text[-1] not in splits): prompt_text += "。" if prompt_language != "en" else "." - print(i18n("实际输入的参考文本:"), prompt_text) - text = text.strip("\n") - if (text[0] not in splits and len(get_first(text)) < 4): text = "。" + text if text_language != "en" else "." + text - - print(i18n("实际输入的目标文本:"), text) - zero_wav = np.zeros( - int(hps.data.sampling_rate * 0.3), - dtype=np.float16 if is_half == True else np.float32, - ) - with torch.no_grad(): - wav16k, sr = librosa.load(ref_wav_path, sr=16000) - if (wav16k.shape[0] > 160000 or wav16k.shape[0] < 48000): - raise OSError(i18n("参考音频在3~10秒范围外，请更换！")) - wav16k = torch.from_numpy(wav16k) - zero_wav_torch = torch.from_numpy(zero_wav) - if is_half == True: - wav16k = wav16k.half().to(device) - zero_wav_torch = zero_wav_torch.half().to(device) - else: - wav16k = wav16k.to(device) - zero_wav_torch = zero_wav_torch.to(device) - wav16k = torch.cat([wav16k, zero_wav_torch]) - ssl_content = ssl_model.model(wav16k.unsqueeze(0))[ - "last_hidden_state" - ].transpose( - 1, 2 - ) # .float() - codes = vq_model.extract_latent(ssl_content) - - prompt_semantic = codes[0, 0] - t1 = ttime() - - if (how_to_cut == i18n("凑四句一切")): - text = cut1(text) - elif (how_to_cut == i18n("凑50字一切")): - text = cut2(text) - elif (how_to_cut == i18n("按中文句号。切")): - text = cut3(text) - elif (how_to_cut == i18n("按英文句号.切")): - text = cut4(text) - elif (how_to_cut == i18n("按标点符号切")): - text = cut5(text) - while "\n\n" in text: - text = text.replace("\n\n", "\n") - print(i18n("实际输入的目标文本(切句后):"), text) - texts = text.split("\n") - texts = merge_short_text_in_array(texts, 5) - audio_opt = [] - if not ref_free: - phones1,bert1,norm_text1=get_phones_and_bert(prompt_text, prompt_language) - - for text in texts: - # 解决输入目标文本的空行导致报错的问题 - if (len(text.strip()) == 0): - continue - if (text[-1] not in splits): text += "。" if text_language != "en" else "." - print(i18n("实际输入的目标文本(每句):"), text) - phones2,bert2,norm_text2=get_phones_and_bert(text, text_language) - print(i18n("前端处理后的文本(每句):"), norm_text2) - if not ref_free: - bert = torch.cat([bert1, bert2], 1) - all_phoneme_ids = torch.LongTensor(phones1+phones2).to(device).unsqueeze(0) - else: - bert = bert2 - all_phoneme_ids = torch.LongTensor(phones2).to(device).unsqueeze(0) - - bert = bert.to(device).unsqueeze(0) - all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device) - prompt = prompt_semantic.unsqueeze(0).to(device) - t2 = ttime() - with torch.no_grad(): - # pred_semantic = t2s_model.model.infer( - pred_semantic, idx = t2s_model.model.infer_panel( - all_phoneme_ids, - all_phoneme_len, - None if ref_free else prompt, - bert, - # prompt_phone_len=ph_offset, - top_k=top_k, - top_p=top_p, - temperature=temperature, - early_stop_num=hz * max_sec, - ) - t3 = ttime() - # print(pred_semantic.shape,idx) - pred_semantic = pred_semantic[:, -idx:].unsqueeze( - 0 - ) # .unsqueeze(0)#mq要多unsqueeze一次 - refer = get_spepc(hps, ref_wav_path) # .to(device) - if is_half == True: - refer = refer.half().to(device) - else: - refer = refer.to(device) - # audio = vq_model.decode(pred_semantic, all_phoneme_ids, refer).detach().cpu().numpy()[0, 0] - audio = ( - vq_model.decode( - pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refer - ) - .detach() - .cpu() - .numpy()[0, 0] - ) ###试试重建不带上prompt部分 - max_audio=np.abs(audio).max()#简单防止16bit爆音 - if max_audio>1:audio/=max_audio - audio_opt.append(audio) - audio_opt.append(zero_wav) - t4 = ttime() - print("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t3 - t2, t4 - t3)) - yield hps.data.sampling_rate, (np.concatenate(audio_opt, 0) * 32768).astype( - np.int16 - ) - - -def split(todo_text): - todo_text = todo_text.replace("……", "。").replace("——", "，") - if todo_text[-1] not in splits: - todo_text += "。" - i_split_head = i_split_tail = 0 - len_text = len(todo_text) - todo_texts = [] - while 1: - if i_split_head >= len_text: - break # 结尾一定有标点，所以直接跳出即可，最后一段在上次已加入 - if todo_text[i_split_head] in splits: - i_split_head += 1 - todo_texts.append(todo_text[i_split_tail:i_split_head]) - i_split_tail = i_split_head - else: - i_split_head += 1 - return todo_texts - - -def cut1(inp): - inp = inp.strip("\n") - inps = split(inp) - split_idx = list(range(0, len(inps), 4)) - split_idx[-1] = None - if len(split_idx) > 1: - opts = [] - for idx in range(len(split_idx) - 1): - opts.append("".join(inps[split_idx[idx]: split_idx[idx + 1]])) - else: - opts = [inp] - return "\n".join(opts) - - -def cut2(inp): - inp = inp.strip("\n") - inps = split(inp) - if len(inps) < 2: - return inp - opts = [] - summ = 0 - tmp_str = "" - for i in range(len(inps)): - summ += len(inps[i]) - tmp_str += inps[i] - if summ > 50: - summ = 0 - opts.append(tmp_str) - tmp_str = "" - if tmp_str != "": - opts.append(tmp_str) - # print(opts) - if len(opts) > 1 and len(opts[-1]) < 50: ##如果最后一个太短了，和前一个合一起 - opts[-2] = opts[-2] + opts[-1] - opts = opts[:-1] - return "\n".join(opts) - - -def cut3(inp): - inp = inp.strip("\n") - return "\n".join(["%s" % item for item in inp.strip("。").split("。")]) - - -def cut4(inp): - inp = inp.strip("\n") - return "\n".join(["%s" % item for item in inp.strip(".").split(".")]) - - -# contributed by https://github.com/AI-Hobbyist/GPT-SoVITS/blob/main/GPT_SoVITS/inference_webui.py -def cut5(inp): - # if not re.search(r'[^\w\s]', inp[-1]): - # inp += '。' - inp = inp.strip("\n") - punds = r'[,.;?!、，。？！;：…]' - items = re.split(f'({punds})', inp) - mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])] - # 在句子不存在符号或句尾无符号的时候保证文本完整 - if len(items)%2 == 1: - mergeitems.append(items[-1]) - opt = "\n".join(mergeitems) - return opt - - -def custom_sort_key(s): - # 使用正则表达式提取字符串中的数字部分和非数字部分 - parts = re.split('(\d+)', s) - # 将数字部分转换为整数，非数字部分保持不变 - parts = [int(part) if part.isdigit() else part for part in parts] - return parts - - -def change_choices(): - SoVITS_names, GPT_names = get_weights_names() - return {"choices": sorted(SoVITS_names, key=custom_sort_key), "__type__": "update"}, {"choices": sorted(GPT_names, key=custom_sort_key), "__type__": "update"} - - -pretrained_sovits_name = "GPT_SoVITS/pretrained_models/s2G488k.pth" -pretrained_gpt_name = "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt" -SoVITS_weight_root = "SoVITS_weights" -GPT_weight_root = "GPT_weights" -os.makedirs(SoVITS_weight_root, exist_ok=True) -os.makedirs(GPT_weight_root, exist_ok=True) - - -def get_weights_names(): - SoVITS_names = [pretrained_sovits_name] - for name in os.listdir(SoVITS_weight_root): - if name.endswith(".pth"): SoVITS_names.append("%s/%s" % (SoVITS_weight_root, name)) - GPT_names = [pretrained_gpt_name] - for name in os.listdir(GPT_weight_root): - if name.endswith(".ckpt"): GPT_names.append("%s/%s" % (GPT_weight_root, name)) - return SoVITS_names, GPT_names - - -SoVITS_names, GPT_names = get_weights_names() - -with gr.Blocks(title="GPT-SoVITS WebUI") as app: - gr.Markdown( - value=i18n("本软件以MIT协议开源, 作者不对软件具备任何控制力, 使用软件者、传播软件导出的声音者自负全责.
如不认可该条款, 则不能使用或引用软件包内任何代码和文件. 详见根目录LICENSE.") - ) - with gr.Group(): - gr.Markdown(value=i18n("模型切换")) - with gr.Row(): - GPT_dropdown = gr.Dropdown(label=i18n("GPT模型列表"), choices=sorted(GPT_names, key=custom_sort_key), value=gpt_path, interactive=True) - SoVITS_dropdown = gr.Dropdown(label=i18n("SoVITS模型列表"), choices=sorted(SoVITS_names, key=custom_sort_key), value=sovits_path, interactive=True) - refresh_button = gr.Button(i18n("刷新模型路径"), variant="primary") - refresh_button.click(fn=change_choices, inputs=[], outputs=[SoVITS_dropdown, GPT_dropdown]) - SoVITS_dropdown.change(change_sovits_weights, [SoVITS_dropdown], []) - GPT_dropdown.change(change_gpt_weights, [GPT_dropdown], []) - gr.Markdown(value=i18n("*请上传并填写参考信息")) - with gr.Row(): - inp_ref = gr.Audio(label=i18n("请上传3~10秒内参考音频，超过会报错！"), type="filepath") - with gr.Column(): - ref_text_free = gr.Checkbox(label=i18n("开启无参考文本模式。不填参考文本亦相当于开启。"), value=False, interactive=True, show_label=True) - gr.Markdown(i18n("使用无参考文本模式时建议使用微调的GPT，听不清参考音频说的啥(不晓得写啥)可以开，开启后无视填写的参考文本。")) - prompt_text = gr.Textbox(label=i18n("参考音频的文本"), value="") - prompt_language = gr.Dropdown( - label=i18n("参考音频的语种"), choices=[i18n("中文"), i18n("英文"), i18n("日文"), i18n("中英混合"), i18n("日英混合"), i18n("多语种混合")], value=i18n("中文") - ) - gr.Markdown(value=i18n("*请填写需要合成的目标文本和语种模式")) - with gr.Row(): - text = gr.Textbox(label=i18n("需要合成的文本"), value="") - text_language = gr.Dropdown( - label=i18n("需要合成的语种"), choices=[i18n("中文"), i18n("英文"), i18n("日文"), i18n("中英混合"), i18n("日英混合"), i18n("多语种混合")], value=i18n("中文") - ) - how_to_cut = gr.Radio( - label=i18n("怎么切"), - choices=[i18n("不切"), i18n("凑四句一切"), i18n("凑50字一切"), i18n("按中文句号。切"), i18n("按英文句号.切"), i18n("按标点符号切"), ], - value=i18n("凑四句一切"), - interactive=True, - ) - with gr.Row(): - gr.Markdown(value=i18n("gpt采样参数(无参考文本时不要太低)：")) - top_k = gr.Slider(minimum=1,maximum=100,step=1,label=i18n("top_k"),value=5,interactive=True) - top_p = gr.Slider(minimum=0,maximum=1,step=0.05,label=i18n("top_p"),value=1,interactive=True) - temperature = gr.Slider(minimum=0,maximum=1,step=0.05,label=i18n("temperature"),value=1,interactive=True) - inference_button = gr.Button(i18n("合成语音"), variant="primary") - output = gr.Audio(label=i18n("输出的语音")) - - inference_button.click( - get_tts_wav, - [inp_ref, prompt_text, prompt_language, text, text_language, how_to_cut, top_k, top_p, temperature, ref_text_free], - [output], - ) - - gr.Markdown(value=i18n("文本切分工具。太长的文本合成出来效果不一定好，所以太长建议先切。合成会根据文本的换行分开合成再拼起来。")) - with gr.Row(): - text_inp = gr.Textbox(label=i18n("需要合成的切分前文本"), value="") - button1 = gr.Button(i18n("凑四句一切"), variant="primary") - button2 = gr.Button(i18n("凑50字一切"), variant="primary") - button3 = gr.Button(i18n("按中文句号。切"), variant="primary") - button4 = gr.Button(i18n("按英文句号.切"), variant="primary") - button5 = gr.Button(i18n("按标点符号切"), variant="primary") - text_opt = gr.Textbox(label=i18n("切分后文本"), value="") - button1.click(cut1, [text_inp], [text_opt]) - button2.click(cut2, [text_inp], [text_opt]) - button3.click(cut3, [text_inp], [text_opt]) - button4.click(cut4, [text_inp], [text_opt]) - button5.click(cut5, [text_inp], [text_opt]) - gr.Markdown(value=i18n("后续将支持转音素、手工修改音素、语音合成分步执行。")) - -app.queue(concurrency_count=511, max_size=1022).launch( - server_name="0.0.0.0", - inbrowser=True, - share=is_share, - server_port=infer_ttswebui, - quiet=True, -) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/module/models.py b/tts/GPT-SoVITS/GPT_SoVITS/module/models.py index 26840cc..29676f4 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/module/models.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/module/models.py @@ -1,6 +1,5 @@ import copy import math -from typing import List import torch from torch import nn from torch.nn import functional as F @@ -16,7 +15,6 @@ from module.mrte_model import MRTE from module.quantize import ResidualVectorQuantizer from text import symbols from torch.cuda.amp import autocast -import contextlib class StochasticDurationPredictor(nn.Module): @@ -892,10 +890,9 @@ class SynthesizerTrn(nn.Module): self.ssl_proj = nn.Conv1d(ssl_dim, ssl_dim, 1, stride=1) self.quantizer = ResidualVectorQuantizer(dimension=ssl_dim, n_q=1, bins=1024) - self.freeze_quantizer = freeze_quantizer - # if freeze_quantizer: - # self.ssl_proj.requires_grad_(False) - # self.quantizer.requires_grad_(False) + if freeze_quantizer: + self.ssl_proj.requires_grad_(False) + self.quantizer.requires_grad_(False) #self.quantizer.eval() # self.enc_p.text_embedding.requires_grad_(False) # self.enc_p.encoder_text.requires_grad_(False) @@ -908,11 +905,6 @@ class SynthesizerTrn(nn.Module): ge = self.ref_enc(y * y_mask, y_mask) with autocast(enabled=False): - maybe_no_grad = torch.no_grad() if self.freeze_quantizer else contextlib.nullcontext() - with maybe_no_grad: - if self.freeze_quantizer: - self.ssl_proj.eval() - self.quantizer.eval() ssl = self.ssl_proj(ssl) quantized, codes, commit_loss, quantized_list = self.quantizer( ssl, layers=[0] @@ -994,55 +986,6 @@ class SynthesizerTrn(nn.Module): o = self.dec((z * y_mask)[:, :, :], g=ge) return o - - - @torch.no_grad() - def batched_decode(self, codes, y_lengths, text, text_lengths, refer, noise_scale=0.5): - ge = None - if refer is not None: - refer_lengths = torch.LongTensor([refer.size(2)]).to(refer.device) - refer_mask = torch.unsqueeze( - commons.sequence_mask(refer_lengths, refer.size(2)), 1 - ).to(refer.dtype) - ge = self.ref_enc(refer * refer_mask, refer_mask) - - # y_mask = torch.unsqueeze(commons.sequence_mask(y_lengths, codes.size(2)), 1).to( - # codes.dtype - # ) - y_lengths = (y_lengths * 2).long().to(codes.device) - text_lengths = text_lengths.long().to(text.device) - # y_lengths = torch.LongTensor([codes.size(2) * 2]).to(codes.device) - # text_lengths = torch.LongTensor([text.size(-1)]).to(text.device) - - # 假设padding之后再decode没有问题, 影响未知，但听起来好像没问题？ - quantized = self.quantizer.decode(codes) - if self.semantic_frame_rate == "25hz": - quantized = F.interpolate( - quantized, size=int(quantized.shape[-1] * 2), mode="nearest" - ) - - x, m_p, logs_p, y_mask = self.enc_p( - quantized, y_lengths, text, text_lengths, ge - ) - z_p = m_p + torch.randn_like(m_p) * torch.exp(logs_p) * noise_scale - - z = self.flow(z_p, y_mask, g=ge, reverse=True) - z_masked = (z * y_mask)[:, :, :] - - # 串行。把padding部分去掉再decode - o_list:List[torch.Tensor] = [] - for i in range(z_masked.shape[0]): - z_slice = z_masked[i, :, :y_lengths[i]].unsqueeze(0) - o = self.dec(z_slice, g=ge)[0, 0, :].detach() - o_list.append(o) - - # 并行（会有问题）。先decode，再把padding的部分去掉 - # o = self.dec(z_masked, g=ge) - # upsample_rate = int(math.prod(self.upsample_rates)) - # o_lengths = y_lengths*upsample_rate - # o_list = [o[i, 0, :idx].detach() for i, idx in enumerate(o_lengths)] - - return o_list def extract_latent(self, x): ssl = self.ssl_proj(x) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/1-get-text.py b/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/1-get-text.py index ba5259b..d612893 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/1-get-text.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/1-get-text.py @@ -117,12 +117,9 @@ if os.path.exists(txt_path) == False: try: wav_name, spk_name, language, text = line.split("|") # todo.append([name,text,"zh"]) - if language in language_v1_to_language_v2.keys(): - todo.append( - [wav_name, text, language_v1_to_language_v2.get(language, language)] - ) - else: - print(f"\033[33m[Waring] The {language = } of {wav_name} is not supported for training.\033[0m") + todo.append( + [wav_name, text, language_v1_to_language_v2.get(language, language)] + ) except: print(line, traceback.format_exc()) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py b/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py index c6fe92e..8152b7d 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/prepare_datasets/2-get-hubert-wav32k.py @@ -82,7 +82,7 @@ def name2go(wav_name,wav_path): tensor_wav16 = tensor_wav16.to(device) ssl=model.model(tensor_wav16.unsqueeze(0))["last_hidden_state"].transpose(1,2).cpu()#torch.Size([1, 768, 215]) if np.isnan(ssl.detach().numpy()).sum()!= 0: - nan_fails.append((wav_name,wav_path)) + nan_fails.append(wav_name) print("nan filtered:%s"%wav_name) return wavfile.write( @@ -90,7 +90,7 @@ def name2go(wav_name,wav_path): 32000, tmp_audio32.astype("int16"), ) - my_save(ssl,hubert_path) + my_save(ssl,hubert_path ) with open(inp_text,"r",encoding="utf8")as f: lines=f.read().strip("\n").split("\n") @@ -113,8 +113,8 @@ for line in lines[int(i_part)::int(all_parts)]: if(len(nan_fails)>0 and is_half==True): is_half=False model=model.float() - for wav in nan_fails: + for wav_name in nan_fails: try: - name2go(wav[0],wav[1]) + name2go(wav_name) except: print(wav_name,traceback.format_exc()) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/s1_train.py b/tts/GPT-SoVITS/GPT_SoVITS/s1_train.py index ece295d..43cfa19 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/s1_train.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/s1_train.py @@ -79,17 +79,15 @@ class my_model_ckpt(ModelCheckpoint): to_save_od["config"] = self.config to_save_od["info"] = "GPT-e%s" % (trainer.current_epoch + 1) # torch.save( - # print(os.environ) - if(os.environ.get("LOCAL_RANK","0")=="0"): - my_save( - to_save_od, - "%s/%s-e%s.ckpt" - % ( - self.half_weights_save_dir, - self.exp_name, - trainer.current_epoch + 1, - ), - ) + my_save( + to_save_od, + "%s/%s-e%s.ckpt" + % ( + self.half_weights_save_dir, + self.exp_name, + trainer.current_epoch + 1, + ), + ) self._save_last_checkpoint(trainer, monitor_candidates) diff --git a/tts/GPT-SoVITS/GPT_SoVITS/text/english.py b/tts/GPT-SoVITS/GPT_SoVITS/text/english.py index 30fafb5..68ce789 100644 --- a/tts/GPT-SoVITS/GPT_SoVITS/text/english.py +++ b/tts/GPT-SoVITS/GPT_SoVITS/text/english.py @@ -320,7 +320,7 @@ class en_G2p(G2p): # 尝试分离所有格 if re.match(r"^([a-z]+)('s)$", word): - phones = self.qryword(word[:-2])[:] + phones = self.qryword(word[:-2]) # P T K F TH HH 无声辅音结尾 's 发 ['S'] if phones[-1] in ['P', 'T', 'K', 'F', 'TH', 'HH']: phones.extend(['S']) @@ -359,4 +359,4 @@ def g2p(text): if __name__ == "__main__": print(g2p("hello")) print(g2p(text_normalize("e.g. I used openai's AI tool to draw a picture."))) - print(g2p(text_normalize("In this; paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))) + print(g2p(text_normalize("In this; paper, we propose 1 DSPGAN, a GAN-based universal vocoder."))) \ No newline at end of file diff --git a/tts/GPT-SoVITS/README.md b/tts/GPT-SoVITS/README.md index 1122516..82a2fea 100644 --- a/tts/GPT-SoVITS/README.md +++ b/tts/GPT-SoVITS/README.md @@ -1,263 +1,71 @@ -
-

GPT-SoVITS-WebUI

-A Powerful Few-shot Voice Conversion and Text-to-Speech WebUI.

+# GPT-SoVITS CLI -[![madewithlove](https://img.shields.io/badge/made_with-%E2%9D%A4-red?style=for-the-badge&labelColor=orange)](https://github.com/RVC-Boss/GPT-SoVITS) +This repository contains a command-line interface (CLI) script for the GPT-SoVITS project, which allows you to perform text-to-speech (TTS) synthesis using custom models. -
+## Prerequisites -[![Open In Colab](https://img.shields.io/badge/Colab-F9AB00?style=for-the-badge&logo=googlecolab&color=525252)](https://colab.research.google.com/github/RVC-Boss/GPT-SoVITS/blob/main/colab_webui.ipynb) -[![License](https://img.shields.io/badge/LICENSE-MIT-green.svg?style=for-the-badge)](https://github.com/RVC-Boss/GPT-SoVITS/blob/main/LICENSE) -[![Huggingface](https://img.shields.io/badge/🤗%20-Models%20Repo-yellow.svg?style=for-the-badge)](https://huggingface.co/lj1995/GPT-SoVITS/tree/main) -[![Discord](https://img.shields.io/discord/1198701940511617164?color=%23738ADB&label=Discord&style=for-the-badge)](https://discord.gg/dnrgs5GHfG) +- Docker +## Setup -[**English**](./README.md) | [**中文简体**](./docs/cn/README.md) | [**日本語**](./docs/ja/README.md) | [**한국어**](./docs/ko/README.md) +### Build the Docker Image -
+To build the Docker image, run the following command: ---- - -## Features: - -1. **Zero-shot TTS:** Input a 5-second vocal sample and experience instant text-to-speech conversion. - -2. **Few-shot TTS:** Fine-tune the model with just 1 minute of training data for improved voice similarity and realism. - -3. **Cross-lingual Support:** Inference in languages different from the training dataset, currently supporting English, Japanese, and Chinese. - -4. **WebUI Tools:** Integrated tools include voice accompaniment separation, automatic training set segmentation, Chinese ASR, and text labeling, assisting beginners in creating training datasets and GPT/SoVITS models. - -**Check out our [demo video](https://www.bilibili.com/video/BV12g4y1m7Uw) here!** - -Unseen speakers few-shot fine-tuning demo: - -https://github.com/RVC-Boss/GPT-SoVITS/assets/129054828/05bee1fa-bdd8-4d85-9350-80c060ab47fb - -**User guide: [简体中文](https://www.yuque.com/baicaigongchang1145haoyuangong/ib3g1e) | [English](https://rentry.co/GPT-SoVITS-guide#/)** - -## Installation - -For users in China region, you can [click here](https://www.codewithgpu.com/i/RVC-Boss/GPT-SoVITS/GPT-SoVITS-Official) to use AutoDL Cloud Docker to experience the full functionality online. - -### Tested Environments - -- Python 3.9, PyTorch 2.0.1, CUDA 11 -- Python 3.10.13, PyTorch 2.1.2, CUDA 12.3 -- Python 3.9, PyTorch 2.2.2, macOS 14.4.1 (Apple silicon) -- Python 3.9, PyTorch 2.2.2, CPU devices - -_Note: numba==0.56.4 requires py<3.11_ - -### Windows - -If you are a Windows user (tested with win>=10), you can directly download the [pre-packaged distribution](https://huggingface.co/lj1995/GPT-SoVITS-windows-package/resolve/main/GPT-SoVITS-beta.7z?download=true) and double-click on _go-webui.bat_ to start GPT-SoVITS-WebUI. - -Users in China region can download [the 0217 package](https://www.icloud.com.cn/iclouddrive/061bfkcVJcBfsMfLF5R2XKdTQ#GPT-SoVITS-beta0217) or [the 0306fix2 package](https://www.icloud.com.cn/iclouddrive/09aaTLf96aa92dbLe0fPNM5CQ#GPT-SoVITS-beta0306fix2) by clicking the links and then selecting "Download a copy." - -_Note: The 0306fix2 version doubles the inference speed and fixes all issues with the no reference text mode._ - -### Linux - -```bash -conda create -n GPTSoVits python=3.9 -conda activate GPTSoVits -bash install.sh +```sh +docker build --no-cache -t gpt-sovits:latest . ``` -### macOS +### Run the Docker Container -**Note: The models trained with GPUs on Macs result in significantly lower quality compared to those trained on other devices, so we are temporarily using CPUs instead.** +To run the Docker container, use the following command: -1. Install Xcode command-line tools by running `xcode-select --install` -2. Install FFmpeg by running `brew install ffmpeg` or `conda install ffmpeg`. -3. Install the program by running the following commands: - -```bash -conda create -n GPTSoVits python=3.9 -conda activate GPTSoVits - -pip install -r requirements.txt +```sh +docker run -it --privileged --gpus all gpt-sovits:latest ``` -### Install Manually +## Usage -#### Install Dependences +To run the CLI script, use the following command: -```bash -pip install -r requirements.txt +```sh +python fakeyou_infer.py [OPTIONS] ``` -#### Install FFmpeg +### Options -##### Conda Users +- `--use_pretrained_gpt`: Use pretrained GPT model. If enabled, the script will use the default pretrained models for GPT and SoVITS and enable the ZeroShot capabilities. +- `--gpt_model`: Path to GPT model checkpoint. Required if `--use_pretrained_gpt` is not enabled. +- `--sovits_model`: Path to SoVITS model checkpoint. Required if `--use_pretrained_gpt` is not enabled. +- `--ref_audio`: Path to reference wav file. Required unless `--ref_free` is enabled. +- `--ref_text`: Path to reference text file. Required unless `--ref_free` is enabled. +- `--target_language`: Language of the target text. Choices: `chinese`, `english`, `japanese`, `chinese+english`, `japanese+english`, `automatic`. +- `--ref_language`: Language of the reference text. Choices: `chinese`, `english`, `japanese`, `chinese+english`, `japanese+english`, `automatic`. Required unless `--ref_free` is enabled. +- `--target_text`: Path to the target text file. Required. +- `--how_to_cut`: How to cut the text for synthesis. Default: `No slice`. Options: `No slice`, `Slice once every 4 sentences`, `Cut per 50 characters`, `Slice by Chinese punct`, `Slice by English punct`, `Slice by every punct`. +- `--top_k`: Top K sampling. Default: 20. +- `--top_p`: Top P sampling. Default: 0.6. +- `--temperature`: Sampling temperature. Default: 0.6. +- `--ref_free`: Perform reference-free TTS. +- `--output_path`: Path to save the output wav file. Default: `output`. -```bash -conda install ffmpeg +### Example Commands + +#### Using Pretrained Models + +```sh +python fakeyou_infer.py --use_pretrained_gpt --ref_audio data/voice/sponge.wav --ref_text data/voice/ref.txt --target_text data/voice/input.txt --target_language english --how_to_cut Slice once every 4 sentences --output_path output/ ``` -##### Ubuntu/Debian Users +#### Using Custom Models with Reference Audio -```bash -sudo apt install ffmpeg -sudo apt install libsox-dev -conda install -c conda-forge 'ffmpeg<7' +```sh +python fakeyou_infer.py --gpt_model GPT_weights/your_gpt_model.ckpt --sovits_model SoVITS_weights/your_sovits_model.pth --ref_audio data/voice/sponge.wav --ref_text data/voice/ref.txt --target_text data/voice/input.txt --target_language english --ref_language english --how_to_cut Slice once every 4 sentences --output_path output/ ``` -##### Windows Users - -Download and place [ffmpeg.exe](https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/ffmpeg.exe) and [ffprobe.exe](https://huggingface.co/lj1995/VoiceConversionWebUI/blob/main/ffprobe.exe) in the GPT-SoVITS root. - -### Using Docker - -#### docker-compose.yaml configuration - -0. Regarding image tags: Due to rapid updates in the codebase and the slow process of packaging and testing images, please check [Docker Hub](https://hub.docker.com/r/breakstring/gpt-sovits) for the currently packaged latest images and select as per your situation, or alternatively, build locally using a Dockerfile according to your own needs. -1. Environment Variables： - -- is_half: Controls half-precision/double-precision. This is typically the cause if the content under the directories 4-cnhubert/5-wav32k is not generated correctly during the "SSL extracting" step. Adjust to True or False based on your actual situation. - -2. Volumes Configuration，The application's root directory inside the container is set to /workspace. The default docker-compose.yaml lists some practical examples for uploading/downloading content. -3. shm_size： The default available memory for Docker Desktop on Windows is too small, which can cause abnormal operations. Adjust according to your own situation. -4. Under the deploy section, GPU-related settings should be adjusted cautiously according to your system and actual circumstances. - -#### Running with docker compose +#### Using Custom Models Without Reference Audio +```sh +python fakeyou_infer.py --gpt_model GPT_weights/your_gpt_model.ckpt --sovits_model SoVITS_weights/your_sovits_model.pth --target_text data/voice/input.txt --target_language english --ref_free --how_to_cut "Slice once every 4 sentences" --output_path output/ ``` -docker compose -f "docker-compose.yaml" up -d -``` - -#### Running with docker command - -As above, modify the corresponding parameters based on your actual situation, then run the following command: - -``` -docker run --rm -it --gpus=all --env=is_half=False --volume=G:\GPT-SoVITS-DockerTest\output:/workspace/output --volume=G:\GPT-SoVITS-DockerTest\logs:/workspace/logs --volume=G:\GPT-SoVITS-DockerTest\SoVITS_weights:/workspace/SoVITS_weights --workdir=/workspace -p 9880:9880 -p 9871:9871 -p 9872:9872 -p 9873:9873 -p 9874:9874 --shm-size="16G" -d breakstring/gpt-sovits:xxxxx -``` - -## Pretrained Models - -Download pretrained models from [GPT-SoVITS Models](https://huggingface.co/lj1995/GPT-SoVITS) and place them in `GPT_SoVITS/pretrained_models`. - -For UVR5 (Vocals/Accompaniment Separation & Reverberation Removal, additionally), download models from [UVR5 Weights](https://huggingface.co/lj1995/VoiceConversionWebUI/tree/main/uvr5_weights) and place them in `tools/uvr5/uvr5_weights`. - -Users in China region can download these two models by entering the links below and clicking "Download a copy" - -- [GPT-SoVITS Models](https://www.icloud.com.cn/iclouddrive/056y_Xog_HXpALuVUjscIwTtg#GPT-SoVITS_Models) - -- [UVR5 Weights](https://www.icloud.com.cn/iclouddrive/0bekRKDiJXboFhbfm3lM2fVbA#UVR5_Weights) - -For Chinese ASR (additionally), download models from [Damo ASR Model](https://modelscope.cn/models/damo/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch/files), [Damo VAD Model](https://modelscope.cn/models/damo/speech_fsmn_vad_zh-cn-16k-common-pytorch/files), and [Damo Punc Model](https://modelscope.cn/models/damo/punc_ct-transformer_zh-cn-common-vocab272727-pytorch/files) and place them in `tools/asr/models`. - -For English or Japanese ASR (additionally), download models from [Faster Whisper Large V3](https://huggingface.co/Systran/faster-whisper-large-v3) and place them in `tools/asr/models`. Also, [other models](https://huggingface.co/Systran) may have the similar effect with smaller disk footprint. - -Users in China region can download this model by entering the links below - -- [Faster Whisper Large V3](https://www.icloud.com/iclouddrive/0c4pQxFs7oWyVU1iMTq2DbmLA#faster-whisper-large-v3) (clicking "Download a copy") - -- [Faster Whisper Large V3](https://hf-mirror.com/Systran/faster-whisper-large-v3) (HuggingFace mirror site) - -## Dataset Format - -The TTS annotation .list file format: - -``` -vocal_path|speaker_name|language|text -``` - -Language dictionary: - -- 'zh': Chinese -- 'ja': Japanese -- 'en': English - -Example: - -``` -D:\GPT-SoVITS\xxx/xxx.wav|xxx|en|I like playing Genshin. -``` - -## Todo List - -- [ ] **High Priority:** - - - [x] Localization in Japanese and English. - - [x] User guide. - - [x] Japanese and English dataset fine tune training. - -- [ ] **Features:** - - [ ] Zero-shot voice conversion (5s) / few-shot voice conversion (1min). - - [ ] TTS speaking speed control. - - [ ] Enhanced TTS emotion control. - - [ ] Experiment with changing SoVITS token inputs to probability distribution of vocabs. - - [ ] Improve English and Japanese text frontend. - - [ ] Develop tiny and larger-sized TTS models. - - [x] Colab scripts. - - [ ] Try expand training dataset (2k hours -> 10k hours). - - [ ] better sovits base model (enhanced audio quality) - - [ ] model mix - -## (Optional) If you need, here will provide the command line operation mode -Use the command line to open the WebUI for UVR5 -``` -python tools/uvr5/webui.py "" -``` -If you can't open a browser, follow the format below for UVR processing,This is using mdxnet for audio processing -``` -python mdxnet.py --model --input_root --output_vocal --output_ins --agg_level --format --device --is_half_precision -``` -This is how the audio segmentation of the dataset is done using the command line -``` -python audio_slicer.py \ - --input_path "" \ - --output_root "" \ - --threshold \ - --min_length \ - --min_interval - --hop_size -``` -This is how dataset ASR processing is done using the command line(Only Chinese) -``` -python tools/asr/funasr_asr.py -i -o -``` -ASR processing is performed through Faster_Whisper(ASR marking except Chinese) - -(No progress bars, GPU performance may cause time delays) -``` -python ./tools/asr/fasterwhisper_asr.py -i -o -l -``` -A custom list save path is enabled - -## Credits - -Special thanks to the following projects and contributors: - -### Theoretical -- [ar-vits](https://github.com/innnky/ar-vits) -- [SoundStorm](https://github.com/yangdongchao/SoundStorm/tree/master/soundstorm/s1/AR) -- [vits](https://github.com/jaywalnut310/vits) -- [TransferTTS](https://github.com/hcy71o/TransferTTS/blob/master/models.py#L556) -- [contentvec](https://github.com/auspicious3000/contentvec/) -- [hifi-gan](https://github.com/jik876/hifi-gan) -- [fish-speech](https://github.com/fishaudio/fish-speech/blob/main/tools/llama/generate.py#L41) -### Pretrained Models -- [Chinese Speech Pretrain](https://github.com/TencentGameMate/chinese_speech_pretrain) -- [Chinese-Roberta-WWM-Ext-Large](https://huggingface.co/hfl/chinese-roberta-wwm-ext-large) -### Text Frontend for Inference -- [paddlespeech zh_normalization](https://github.com/PaddlePaddle/PaddleSpeech/tree/develop/paddlespeech/t2s/frontend/zh_normalization) -- [LangSegment](https://github.com/juntaosun/LangSegment) -### WebUI Tools -- [ultimatevocalremovergui](https://github.com/Anjok07/ultimatevocalremovergui) -- [audio-slicer](https://github.com/openvpi/audio-slicer) -- [SubFix](https://github.com/cronrpc/SubFix) -- [FFmpeg](https://github.com/FFmpeg/FFmpeg) -- [gradio](https://github.com/gradio-app/gradio) -- [faster-whisper](https://github.com/SYSTRAN/faster-whisper) -- [FunASR](https://github.com/alibaba-damo-academy/FunASR) - -## Thanks to all contributors for their efforts - - - - diff --git a/tts/GPT-SoVITS/api.py b/tts/GPT-SoVITS/api.py new file mode 100644 index 0000000..7e06032 --- /dev/null +++ b/tts/GPT-SoVITS/api.py @@ -0,0 +1,736 @@ +""" +# api.py usage + +` python api.py -dr "123.wav" -dt "一二三。" -dl "zh" ` + +## 执行参数: + +`-s` - `SoVITS模型路径, 可在 config.py 中指定` +`-g` - `GPT模型路径, 可在 config.py 中指定` + +调用请求缺少参考音频时使用 +`-dr` - `默认参考音频路径` +`-dt` - `默认参考音频文本` +`-dl` - `默认参考音频语种, "中文","英文","日文","zh","en","ja"` + +`-d` - `推理设备, "cuda","cpu"` +`-a` - `绑定地址, 默认"127.0.0.1"` +`-p` - `绑定端口, 默认9880, 可在 config.py 中指定` +`-fp` - `覆盖 config.py 使用全精度` +`-hp` - `覆盖 config.py 使用半精度` +`-sm` - `流式返回模式, 默认不启用, "close","c", "normal","n", "keepalive","k"` +·-mt` - `返回的音频编码格式, 流式默认ogg, 非流式默认wav, "wav", "ogg", "aac"` +·-cp` - `文本切分符号设定, 默认为空, 以",.，。"字符串的方式传入` + +`-hb` - `cnhubert路径` +`-b` - `bert路径` + +## 调用: + +### 推理 + +endpoint: `/` + +使用执行参数指定的参考音频: +GET: + `http://127.0.0.1:9880?text=先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。&text_language=zh` +POST: +```json +{ + "text": "先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。", + "text_language": "zh" +} +``` + +使用执行参数指定的参考音频并设定分割符号: +GET: + `http://127.0.0.1:9880?text=先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。&text_language=zh&cut_punc=，。` +POST: +```json +{ + "text": "先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。", + "text_language": "zh", + "cut_punc": "，。", +} +``` + +手动指定当次推理所使用的参考音频: +GET: + `http://127.0.0.1:9880?refer_wav_path=123.wav&prompt_text=一二三。&prompt_language=zh&text=先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。&text_language=zh` +POST: +```json +{ + "refer_wav_path": "123.wav", + "prompt_text": "一二三。", + "prompt_language": "zh", + "text": "先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。", + "text_language": "zh" +} +``` + +RESP: +成功: 直接返回 wav 音频流， http code 200 +失败: 返回包含错误信息的 json, http code 400 + + +### 更换默认参考音频 + +endpoint: `/change_refer` + +key与推理端一样 + +GET: + `http://127.0.0.1:9880/change_refer?refer_wav_path=123.wav&prompt_text=一二三。&prompt_language=zh` +POST: +```json +{ + "refer_wav_path": "123.wav", + "prompt_text": "一二三。", + "prompt_language": "zh" +} +``` + +RESP: +成功: json, http code 200 +失败: json, 400 + + +### 命令控制 + +endpoint: `/control` + +command: +"restart": 重新运行 +"exit": 结束运行 + +GET: + `http://127.0.0.1:9880/control?command=restart` +POST: +```json +{ + "command": "restart" +} +``` + +RESP: 无 + +""" + + +import argparse +import os,re +import sys + +now_dir = os.getcwd() +sys.path.append(now_dir) +sys.path.append("%s/GPT_SoVITS" % (now_dir)) + +import signal +import LangSegment +from time import time as ttime +import torch +import librosa +import soundfile as sf +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import StreamingResponse, JSONResponse +import uvicorn +from transformers import AutoModelForMaskedLM, AutoTokenizer +import numpy as np +from feature_extractor import cnhubert +from io import BytesIO +from module.models import SynthesizerTrn +from AR.models.t2s_lightning_module import Text2SemanticLightningModule +from text import cleaned_text_to_sequence +from text.cleaner import clean_text +from module.mel_processing import spectrogram_torch +from my_utils import load_audio +import config as global_config +import logging +import subprocess + + +class DefaultRefer: + def __init__(self, path, text, language): + self.path = args.default_refer_path + self.text = args.default_refer_text + self.language = args.default_refer_language + + def is_ready(self) -> bool: + return is_full(self.path, self.text, self.language) + + +def is_empty(*items): # 任意一项不为空返回False + for item in items: + if item is not None and item != "": + return False + return True + + +def is_full(*items): # 任意一项为空返回False + for item in items: + if item is None or item == "": + return False + return True + + +def change_sovits_weights(sovits_path): + global vq_model, hps + dict_s2 = torch.load(sovits_path, map_location="cpu") + hps = dict_s2["config"] + hps = DictToAttrRecursive(hps) + hps.model.semantic_frame_rate = "25hz" + model_params_dict = vars(hps.model) + vq_model = SynthesizerTrn( + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + n_speakers=hps.data.n_speakers, + **model_params_dict + ) + if ("pretrained" not in sovits_path): + del vq_model.enc_q + if is_half == True: + vq_model = vq_model.half().to(device) + else: + vq_model = vq_model.to(device) + vq_model.eval() + vq_model.load_state_dict(dict_s2["weight"], strict=False) + + +def change_gpt_weights(gpt_path): + global hz, max_sec, t2s_model, config + hz = 50 + dict_s1 = torch.load(gpt_path, map_location="cpu") + config = dict_s1["config"] + max_sec = config["data"]["max_sec"] + t2s_model = Text2SemanticLightningModule(config, "****", is_train=False) + t2s_model.load_state_dict(dict_s1["weight"]) + if is_half == True: + t2s_model = t2s_model.half() + t2s_model = t2s_model.to(device) + t2s_model.eval() + total = sum([param.nelement() for param in t2s_model.parameters()]) + logger.info("Number of parameter: %.2fM" % (total / 1e6)) + + +def get_bert_feature(text, word2ph): + with torch.no_grad(): + inputs = tokenizer(text, return_tensors="pt") + for i in inputs: + inputs[i] = inputs[i].to(device) #####输入是long不用管精度问题，精度随bert_model + res = bert_model(**inputs, output_hidden_states=True) + res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] + assert len(word2ph) == len(text) + phone_level_feature = [] + for i in range(len(word2ph)): + repeat_feature = res[i].repeat(word2ph[i], 1) + phone_level_feature.append(repeat_feature) + phone_level_feature = torch.cat(phone_level_feature, dim=0) + # if(is_half==True):phone_level_feature=phone_level_feature.half() + return phone_level_feature.T + + +def clean_text_inf(text, language): + phones, word2ph, norm_text = clean_text(text, language) + phones = cleaned_text_to_sequence(phones) + return phones, word2ph, norm_text + + +def get_bert_inf(phones, word2ph, norm_text, language): + language=language.replace("all_","") + if language == "zh": + bert = get_bert_feature(norm_text, word2ph).to(device)#.to(dtype) + else: + bert = torch.zeros( + (1024, len(phones)), + dtype=torch.float16 if is_half == True else torch.float32, + ).to(device) + + return bert + + +def get_phones_and_bert(text,language): + if language in {"en","all_zh","all_ja"}: + language = language.replace("all_","") + if language == "en": + LangSegment.setfilters(["en"]) + formattext = " ".join(tmp["text"] for tmp in LangSegment.getTexts(text)) + else: + # 因无法区别中日文汉字,以用户输入为准 + formattext = text + while " " in formattext: + formattext = formattext.replace(" ", " ") + phones, word2ph, norm_text = clean_text_inf(formattext, language) + if language == "zh": + bert = get_bert_feature(norm_text, word2ph).to(device) + else: + bert = torch.zeros( + (1024, len(phones)), + dtype=torch.float16 if is_half == True else torch.float32, + ).to(device) + elif language in {"zh", "ja","auto"}: + textlist=[] + langlist=[] + LangSegment.setfilters(["zh","ja","en","ko"]) + if language == "auto": + for tmp in LangSegment.getTexts(text): + if tmp["lang"] == "ko": + langlist.append("zh") + textlist.append(tmp["text"]) + else: + langlist.append(tmp["lang"]) + textlist.append(tmp["text"]) + else: + for tmp in LangSegment.getTexts(text): + if tmp["lang"] == "en": + langlist.append(tmp["lang"]) + else: + # 因无法区别中日文汉字,以用户输入为准 + langlist.append(language) + textlist.append(tmp["text"]) + # logger.info(textlist) + # logger.info(langlist) + phones_list = [] + bert_list = [] + norm_text_list = [] + for i in range(len(textlist)): + lang = langlist[i] + phones, word2ph, norm_text = clean_text_inf(textlist[i], lang) + bert = get_bert_inf(phones, word2ph, norm_text, lang) + phones_list.append(phones) + norm_text_list.append(norm_text) + bert_list.append(bert) + bert = torch.cat(bert_list, dim=1) + phones = sum(phones_list, []) + norm_text = ''.join(norm_text_list) + + return phones,bert.to(torch.float16 if is_half == True else torch.float32),norm_text + + +class DictToAttrRecursive: + def __init__(self, input_dict): + for key, value in input_dict.items(): + if isinstance(value, dict): + # 如果值是字典，递归调用构造函数 + setattr(self, key, DictToAttrRecursive(value)) + else: + setattr(self, key, value) + + +def get_spepc(hps, filename): + audio = load_audio(filename, int(hps.data.sampling_rate)) + audio = torch.FloatTensor(audio) + audio_norm = audio + audio_norm = audio_norm.unsqueeze(0) + spec = spectrogram_torch(audio_norm, hps.data.filter_length, hps.data.sampling_rate, hps.data.hop_length, + hps.data.win_length, center=False) + return spec + + +def pack_audio(audio_bytes, data, rate): + if media_type == "ogg": + audio_bytes = pack_ogg(audio_bytes, data, rate) + elif media_type == "aac": + audio_bytes = pack_aac(audio_bytes, data, rate) + else: + # wav无法流式, 先暂存raw + audio_bytes = pack_raw(audio_bytes, data, rate) + + return audio_bytes + + +def pack_ogg(audio_bytes, data, rate): + with sf.SoundFile(audio_bytes, mode='w', samplerate=rate, channels=1, format='ogg') as audio_file: + audio_file.write(data) + + return audio_bytes + + +def pack_raw(audio_bytes, data, rate): + audio_bytes.write(data.tobytes()) + + return audio_bytes + + +def pack_wav(audio_bytes, rate): + data = np.frombuffer(audio_bytes.getvalue(),dtype=np.int16) + wav_bytes = BytesIO() + sf.write(wav_bytes, data, rate, format='wav') + + return wav_bytes + + +def pack_aac(audio_bytes, data, rate): + process = subprocess.Popen([ + 'ffmpeg', + '-f', 's16le', # 输入16位有符号小端整数PCM + '-ar', str(rate), # 设置采样率 + '-ac', '1', # 单声道 + '-i', 'pipe:0', # 从管道读取输入 + '-c:a', 'aac', # 音频编码器为AAC + '-b:a', '192k', # 比特率 + '-vn', # 不包含视频 + '-f', 'adts', # 输出AAC数据流格式 + 'pipe:1' # 将输出写入管道 + ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + out, _ = process.communicate(input=data.tobytes()) + audio_bytes.write(out) + + return audio_bytes + + +def read_clean_buffer(audio_bytes): + audio_chunk = audio_bytes.getvalue() + audio_bytes.truncate(0) + audio_bytes.seek(0) + + return audio_bytes, audio_chunk + + +def cut_text(text, punc): + punc_list = [p for p in punc if p in {",", ".", ";", "?", "!", "、", "，", "。", "？", "！", "；", "：", "…"}] + if len(punc_list) > 0: + punds = r"[" + "".join(punc_list) + r"]" + text = text.strip("\n") + items = re.split(f"({punds})", text) + mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])] + # 在句子不存在符号或句尾无符号的时候保证文本完整 + if len(items)%2 == 1: + mergeitems.append(items[-1]) + text = "\n".join(mergeitems) + + while "\n\n" in text: + text = text.replace("\n\n", "\n") + + return text + + +def only_punc(text): + return not any(t.isalnum() or t.isalpha() for t in text) + + +def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language): + t0 = ttime() + prompt_text = prompt_text.strip("\n") + prompt_language, text = prompt_language, text.strip("\n") + zero_wav = np.zeros(int(hps.data.sampling_rate * 0.3), dtype=np.float16 if is_half == True else np.float32) + with torch.no_grad(): + wav16k, sr = librosa.load(ref_wav_path, sr=16000) + wav16k = torch.from_numpy(wav16k) + zero_wav_torch = torch.from_numpy(zero_wav) + if (is_half == True): + wav16k = wav16k.half().to(device) + zero_wav_torch = zero_wav_torch.half().to(device) + else: + wav16k = wav16k.to(device) + zero_wav_torch = zero_wav_torch.to(device) + wav16k = torch.cat([wav16k, zero_wav_torch]) + ssl_content = ssl_model.model(wav16k.unsqueeze(0))["last_hidden_state"].transpose(1, 2) # .float() + codes = vq_model.extract_latent(ssl_content) + prompt_semantic = codes[0, 0] + t1 = ttime() + prompt_language = dict_language[prompt_language.lower()] + text_language = dict_language[text_language.lower()] + phones1, bert1, norm_text1 = get_phones_and_bert(prompt_text, prompt_language) + texts = text.split("\n") + audio_bytes = BytesIO() + + for text in texts: + # 简单防止纯符号引发参考音频泄露 + if only_punc(text): + continue + + audio_opt = [] + phones2, bert2, norm_text2 = get_phones_and_bert(text, text_language) + bert = torch.cat([bert1, bert2], 1) + + all_phoneme_ids = torch.LongTensor(phones1 + phones2).to(device).unsqueeze(0) + bert = bert.to(device).unsqueeze(0) + all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device) + prompt = prompt_semantic.unsqueeze(0).to(device) + t2 = ttime() + with torch.no_grad(): + # pred_semantic = t2s_model.model.infer( + pred_semantic, idx = t2s_model.model.infer_panel( + all_phoneme_ids, + all_phoneme_len, + prompt, + bert, + # prompt_phone_len=ph_offset, + top_k=config['inference']['top_k'], + early_stop_num=hz * max_sec) + t3 = ttime() + # print(pred_semantic.shape,idx) + pred_semantic = pred_semantic[:, -idx:].unsqueeze(0) # .unsqueeze(0)#mq要多unsqueeze一次 + refer = get_spepc(hps, ref_wav_path) # .to(device) + if (is_half == True): + refer = refer.half().to(device) + else: + refer = refer.to(device) + # audio = vq_model.decode(pred_semantic, all_phoneme_ids, refer).detach().cpu().numpy()[0, 0] + audio = \ + vq_model.decode(pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), + refer).detach().cpu().numpy()[ + 0, 0] ###试试重建不带上prompt部分 + audio_opt.append(audio) + audio_opt.append(zero_wav) + t4 = ttime() + audio_bytes = pack_audio(audio_bytes,(np.concatenate(audio_opt, 0) * 32768).astype(np.int16),hps.data.sampling_rate) + # logger.info("%.3f\t%.3f\t%.3f\t%.3f" % (t1 - t0, t2 - t1, t3 - t2, t4 - t3)) + if stream_mode == "normal": + audio_bytes, audio_chunk = read_clean_buffer(audio_bytes) + yield audio_chunk + + if not stream_mode == "normal": + if media_type == "wav": + audio_bytes = pack_wav(audio_bytes,hps.data.sampling_rate) + yield audio_bytes.getvalue() + + + +def handle_control(command): + if command == "restart": + os.execl(g_config.python_exec, g_config.python_exec, *sys.argv) + elif command == "exit": + os.kill(os.getpid(), signal.SIGTERM) + exit(0) + + +def handle_change(path, text, language): + if is_empty(path, text, language): + return JSONResponse({"code": 400, "message": '缺少任意一项以下参数: "path", "text", "language"'}, status_code=400) + + if path != "" or path is not None: + default_refer.path = path + if text != "" or text is not None: + default_refer.text = text + if language != "" or language is not None: + default_refer.language = language + + logger.info(f"当前默认参考音频路径: {default_refer.path}") + logger.info(f"当前默认参考音频文本: {default_refer.text}") + logger.info(f"当前默认参考音频语种: {default_refer.language}") + logger.info(f"is_ready: {default_refer.is_ready()}") + + + return JSONResponse({"code": 0, "message": "Success"}, status_code=200) + + +def handle(refer_wav_path, prompt_text, prompt_language, text, text_language, cut_punc): + if ( + refer_wav_path == "" or refer_wav_path is None + or prompt_text == "" or prompt_text is None + or prompt_language == "" or prompt_language is None + ): + refer_wav_path, prompt_text, prompt_language = ( + default_refer.path, + default_refer.text, + default_refer.language, + ) + if not default_refer.is_ready(): + return JSONResponse({"code": 400, "message": "未指定参考音频且接口无预设"}, status_code=400) + + if cut_punc == None: + text = cut_text(text,default_cut_punc) + else: + text = cut_text(text,cut_punc) + + return StreamingResponse(get_tts_wav(refer_wav_path, prompt_text, prompt_language, text, text_language), media_type="audio/"+media_type) + + + + +# -------------------------------- +# 初始化部分 +# -------------------------------- +dict_language = { + "中文": "all_zh", + "英文": "en", + "日文": "all_ja", + "中英混合": "zh", + "日英混合": "ja", + "多语种混合": "auto", #多语种启动切分识别语种 + "all_zh": "all_zh", + "en": "en", + "all_ja": "all_ja", + "zh": "zh", + "ja": "ja", + "auto": "auto", +} + +# logger +logging.config.dictConfig(uvicorn.config.LOGGING_CONFIG) +logger = logging.getLogger('uvicorn') + +# 获取配置 +g_config = global_config.Config() + +# 获取参数 +parser = argparse.ArgumentParser(description="GPT-SoVITS api") + +parser.add_argument("-s", "--sovits_path", type=str, default=g_config.sovits_path, help="SoVITS模型路径") +parser.add_argument("-g", "--gpt_path", type=str, default=g_config.gpt_path, help="GPT模型路径") +parser.add_argument("-dr", "--default_refer_path", type=str, default="", help="默认参考音频路径") +parser.add_argument("-dt", "--default_refer_text", type=str, default="", help="默认参考音频文本") +parser.add_argument("-dl", "--default_refer_language", type=str, default="", help="默认参考音频语种") +parser.add_argument("-d", "--device", type=str, default=g_config.infer_device, help="cuda / cpu") +parser.add_argument("-a", "--bind_addr", type=str, default="0.0.0.0", help="default: 0.0.0.0") +parser.add_argument("-p", "--port", type=int, default=g_config.api_port, help="default: 9880") +parser.add_argument("-fp", "--full_precision", action="store_true", default=False, help="覆盖config.is_half为False, 使用全精度") +parser.add_argument("-hp", "--half_precision", action="store_true", default=False, help="覆盖config.is_half为True, 使用半精度") +# bool值的用法为 `python ./api.py -fp ...` +# 此时 full_precision==True, half_precision==False +parser.add_argument("-sm", "--stream_mode", type=str, default="close", help="流式返回模式, close / normal / keepalive") +parser.add_argument("-mt", "--media_type", type=str, default="wav", help="音频编码格式, wav / ogg / aac") +parser.add_argument("-cp", "--cut_punc", type=str, default="", help="文本切分符号设定, 符号范围,.;?!、，。？！；：…") +# 切割常用分句符为 `python ./api.py -cp ".?!。？！"` +parser.add_argument("-hb", "--hubert_path", type=str, default=g_config.cnhubert_path, help="覆盖config.cnhubert_path") +parser.add_argument("-b", "--bert_path", type=str, default=g_config.bert_path, help="覆盖config.bert_path") + +args = parser.parse_args() +sovits_path = args.sovits_path +gpt_path = args.gpt_path +device = args.device +port = args.port +host = args.bind_addr +cnhubert_base_path = args.hubert_path +bert_path = args.bert_path +default_cut_punc = args.cut_punc + +# 应用参数配置 +default_refer = DefaultRefer(args.default_refer_path, args.default_refer_text, args.default_refer_language) + +# 模型路径检查 +if sovits_path == "": + sovits_path = g_config.pretrained_sovits_path + logger.warn(f"未指定SoVITS模型路径, fallback后当前值: {sovits_path}") +if gpt_path == "": + gpt_path = g_config.pretrained_gpt_path + logger.warn(f"未指定GPT模型路径, fallback后当前值: {gpt_path}") + +# 指定默认参考音频, 调用方 未提供/未给全 参考音频参数时使用 +if default_refer.path == "" or default_refer.text == "" or default_refer.language == "": + default_refer.path, default_refer.text, default_refer.language = "", "", "" + logger.info("未指定默认参考音频") +else: + logger.info(f"默认参考音频路径: {default_refer.path}") + logger.info(f"默认参考音频文本: {default_refer.text}") + logger.info(f"默认参考音频语种: {default_refer.language}") + +# 获取半精度 +is_half = g_config.is_half +if args.full_precision: + is_half = False +if args.half_precision: + is_half = True +if args.full_precision and args.half_precision: + is_half = g_config.is_half # 炒饭fallback +logger.info(f"半精: {is_half}") + +# 流式返回模式 +if args.stream_mode.lower() in ["normal","n"]: + stream_mode = "normal" + logger.info("流式返回已开启") +else: + stream_mode = "close" + +# 音频编码格式 +if args.media_type.lower() in ["aac","ogg"]: + media_type = args.media_type.lower() +elif stream_mode == "close": + media_type = "wav" +else: + media_type = "ogg" +logger.info(f"编码格式: {media_type}") + +# 初始化模型 +cnhubert.cnhubert_base_path = cnhubert_base_path +tokenizer = AutoTokenizer.from_pretrained(bert_path) +bert_model = AutoModelForMaskedLM.from_pretrained(bert_path) +ssl_model = cnhubert.get_model() +if is_half: + bert_model = bert_model.half().to(device) + ssl_model = ssl_model.half().to(device) +else: + bert_model = bert_model.to(device) + ssl_model = ssl_model.to(device) +change_sovits_weights(sovits_path) +change_gpt_weights(gpt_path) + + + + +# -------------------------------- +# 接口部分 +# -------------------------------- +app = FastAPI() + +@app.post("/set_model") +async def set_model(request: Request): + json_post_raw = await request.json() + global gpt_path + gpt_path=json_post_raw.get("gpt_model_path") + global sovits_path + sovits_path=json_post_raw.get("sovits_model_path") + logger.info("gptpath"+gpt_path+";vitspath"+sovits_path) + change_sovits_weights(sovits_path) + change_gpt_weights(gpt_path) + return "ok" + + +@app.post("/control") +async def control(request: Request): + json_post_raw = await request.json() + return handle_control(json_post_raw.get("command")) + + +@app.get("/control") +async def control(command: str = None): + return handle_control(command) + + +@app.post("/change_refer") +async def change_refer(request: Request): + json_post_raw = await request.json() + return handle_change( + json_post_raw.get("refer_wav_path"), + json_post_raw.get("prompt_text"), + json_post_raw.get("prompt_language") + ) + + +@app.get("/change_refer") +async def change_refer( + refer_wav_path: str = None, + prompt_text: str = None, + prompt_language: str = None +): + return handle_change(refer_wav_path, prompt_text, prompt_language) + + +@app.post("/") +async def tts_endpoint(request: Request): + json_post_raw = await request.json() + return handle( + json_post_raw.get("refer_wav_path"), + json_post_raw.get("prompt_text"), + json_post_raw.get("prompt_language"), + json_post_raw.get("text"), + json_post_raw.get("text_language"), + json_post_raw.get("cut_punc"), + ) + + +@app.get("/") +async def tts_endpoint( + refer_wav_path: str = None, + prompt_text: str = None, + prompt_language: str = None, + text: str = None, + text_language: str = None, + cut_punc: str = None, +): + return handle(refer_wav_path, prompt_text, prompt_language, text, text_language, cut_punc) + + +if __name__ == "__main__": + uvicorn.run(app, host=host, port=port, workers=1) diff --git a/tts/GPT-SoVITS/api_v2.py b/tts/GPT-SoVITS/api_v2.py deleted file mode 100644 index aaa56e0..0000000 --- a/tts/GPT-SoVITS/api_v2.py +++ /dev/null @@ -1,453 +0,0 @@ -""" -# WebAPI文档 - -` python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/tts_infer.yaml ` - -## 执行参数: - `-a` - `绑定地址, 默认"127.0.0.1"` - `-p` - `绑定端口, 默认9880` - `-c` - `TTS配置文件路径, 默认"GPT_SoVITS/configs/tts_infer.yaml"` - -## 调用: - -### 推理 - -endpoint: `/tts` -GET: -``` -http://127.0.0.1:9880/tts?text=先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。&text_lang=zh&ref_audio_path=archive_jingyuan_1.wav&prompt_lang=zh&prompt_text=我是「罗浮」云骑将军景元。不必拘谨，「将军」只是一时的身份，你称呼我景元便可&text_split_method=cut5&batch_size=1&media_type=wav&streaming_mode=true -``` - -POST: -```json -{ - "text": "", # str.(required) text to be synthesized - "text_lang": "", # str.(required) language of the text to be synthesized - "ref_audio_path": "", # str.(required) reference audio path. - "prompt_text": "", # str.(optional) prompt text for the reference audio - "prompt_lang": "", # str.(required) language of the prompt text for the reference audio - "top_k": 5, # int.(optional) top k sampling - "top_p": 1, # float.(optional) top p sampling - "temperature": 1, # float.(optional) temperature for sampling - "text_split_method": "cut5", # str.(optional) text split method, see text_segmentation_method.py for details. - "batch_size": 1, # int.(optional) batch size for inference - "batch_threshold": 0.75, # float.(optional) threshold for batch splitting. - "split_bucket": true, # bool.(optional) whether to split the batch into multiple buckets. - "speed_factor":1.0, # float.(optional) control the speed of the synthesized audio. - "fragment_interval":0.3, # float.(optional) to control the interval of the audio fragment. - "seed": -1, # int.(optional) random seed for reproducibility. - "media_type": "wav", # str.(optional) media type of the output audio, support "wav", "raw", "ogg", "aac". - "streaming_mode": false, # bool.(optional) whether to return a streaming response. - "parallel_infer": True, # bool.(optional) whether to use parallel inference. - "repetition_penalty": 1.35 # float.(optional) repetition penalty for T2S model. -} -``` - -RESP: -成功: 直接返回 wav 音频流， http code 200 -失败: 返回包含错误信息的 json, http code 400 - -### 命令控制 - -endpoint: `/control` - -command: -"restart": 重新运行 -"exit": 结束运行 - -GET: -``` -http://127.0.0.1:9880/control?command=restart -``` -POST: -```json -{ - "command": "restart" -} -``` - -RESP: 无 - - -### 切换GPT模型 - -endpoint: `/set_gpt_weights` - -GET: -``` -http://127.0.0.1:9880/set_gpt_weights?weights_path=GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt -``` -RESP: -成功: 返回"success", http code 200 -失败: 返回包含错误信息的 json, http code 400 - - -### 切换Sovits模型 - -endpoint: `/set_sovits_weights` - -GET: -``` -http://127.0.0.1:9880/set_sovits_weights?weights_path=GPT_SoVITS/pretrained_models/s2G488k.pth -``` - -RESP: -成功: 返回"success", http code 200 -失败: 返回包含错误信息的 json, http code 400 - -""" -import os -import sys -import traceback -from typing import Generator - -now_dir = os.getcwd() -sys.path.append(now_dir) -sys.path.append("%s/GPT_SoVITS" % (now_dir)) - -import argparse -import subprocess -import wave -import signal -import numpy as np -import soundfile as sf -from fastapi import FastAPI, Request, HTTPException, Response -from fastapi.responses import StreamingResponse, JSONResponse -from fastapi import FastAPI, UploadFile, File -import uvicorn -from io import BytesIO -from tools.i18n.i18n import I18nAuto -from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config -from GPT_SoVITS.TTS_infer_pack.text_segmentation_method import get_method_names as get_cut_method_names -from fastapi.responses import StreamingResponse -from pydantic import BaseModel -# print(sys.path) -i18n = I18nAuto() -cut_method_names = get_cut_method_names() - -parser = argparse.ArgumentParser(description="GPT-SoVITS api") -parser.add_argument("-c", "--tts_config", type=str, default="GPT_SoVITS/configs/tts_infer.yaml", help="tts_infer路径") -parser.add_argument("-a", "--bind_addr", type=str, default="127.0.0.1", help="default: 127.0.0.1") -parser.add_argument("-p", "--port", type=int, default="9880", help="default: 9880") -args = parser.parse_args() -config_path = args.tts_config -# device = args.device -port = args.port -host = args.bind_addr -argv = sys.argv - -if config_path in [None, ""]: - config_path = "GPT-SoVITS/configs/tts_infer.yaml" - -tts_config = TTS_Config(config_path) -tts_pipeline = TTS(tts_config) - -APP = FastAPI() -class TTS_Request(BaseModel): - text: str = None - text_lang: str = None - ref_audio_path: str = None - prompt_lang: str = None - prompt_text: str = "" - top_k:int = 5 - top_p:float = 1 - temperature:float = 1 - text_split_method:str = "cut5" - batch_size:int = 1 - batch_threshold:float = 0.75 - split_bucket:bool = True - speed_factor:float = 1.0 - fragment_interval:float = 0.3 - seed:int = -1 - media_type:str = "wav" - streaming_mode:bool = False - parallel_infer:bool = True - repetition_penalty:float = 1.35 - -### modify from https://github.com/RVC-Boss/GPT-SoVITS/pull/894/files -def pack_ogg(io_buffer:BytesIO, data:np.ndarray, rate:int): - with sf.SoundFile(io_buffer, mode='w', samplerate=rate, channels=1, format='ogg') as audio_file: - audio_file.write(data) - return io_buffer - - -def pack_raw(io_buffer:BytesIO, data:np.ndarray, rate:int): - io_buffer.write(data.tobytes()) - return io_buffer - - -def pack_wav(io_buffer:BytesIO, data:np.ndarray, rate:int): - io_buffer = BytesIO() - sf.write(io_buffer, data, rate, format='wav') - return io_buffer - -def pack_aac(io_buffer:BytesIO, data:np.ndarray, rate:int): - process = subprocess.Popen([ - 'ffmpeg', - '-f', 's16le', # 输入16位有符号小端整数PCM - '-ar', str(rate), # 设置采样率 - '-ac', '1', # 单声道 - '-i', 'pipe:0', # 从管道读取输入 - '-c:a', 'aac', # 音频编码器为AAC - '-b:a', '192k', # 比特率 - '-vn', # 不包含视频 - '-f', 'adts', # 输出AAC数据流格式 - 'pipe:1' # 将输出写入管道 - ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, _ = process.communicate(input=data.tobytes()) - io_buffer.write(out) - return io_buffer - -def pack_audio(io_buffer:BytesIO, data:np.ndarray, rate:int, media_type:str): - if media_type == "ogg": - io_buffer = pack_ogg(io_buffer, data, rate) - elif media_type == "aac": - io_buffer = pack_aac(io_buffer, data, rate) - elif media_type == "wav": - io_buffer = pack_wav(io_buffer, data, rate) - else: - io_buffer = pack_raw(io_buffer, data, rate) - io_buffer.seek(0) - return io_buffer - - - -# from https://huggingface.co/spaces/coqui/voice-chat-with-mistral/blob/main/app.py -def wave_header_chunk(frame_input=b"", channels=1, sample_width=2, sample_rate=32000): - # This will create a wave header then append the frame input - # It should be first on a streaming wav file - # Other frames better should not have it (else you will hear some artifacts each chunk start) - wav_buf = BytesIO() - with wave.open(wav_buf, "wb") as vfout: - vfout.setnchannels(channels) - vfout.setsampwidth(sample_width) - vfout.setframerate(sample_rate) - vfout.writeframes(frame_input) - - wav_buf.seek(0) - return wav_buf.read() - - -def handle_control(command:str): - if command == "restart": - os.execl(sys.executable, sys.executable, *argv) - elif command == "exit": - os.kill(os.getpid(), signal.SIGTERM) - exit(0) - - -def check_params(req:dict): - text:str = req.get("text", "") - text_lang:str = req.get("text_lang", "") - ref_audio_path:str = req.get("ref_audio_path", "") - streaming_mode:bool = req.get("streaming_mode", False) - media_type:str = req.get("media_type", "wav") - prompt_lang:str = req.get("prompt_lang", "") - text_split_method:str = req.get("text_split_method", "cut5") - - if ref_audio_path in [None, ""]: - return JSONResponse(status_code=400, content={"message": "ref_audio_path is required"}) - if text in [None, ""]: - return JSONResponse(status_code=400, content={"message": "text is required"}) - if (text_lang in [None, ""]) : - return JSONResponse(status_code=400, content={"message": "text_lang is required"}) - elif text_lang.lower() not in tts_config.languages: - return JSONResponse(status_code=400, content={"message": "text_lang is not supported"}) - if (prompt_lang in [None, ""]) : - return JSONResponse(status_code=400, content={"message": "prompt_lang is required"}) - elif prompt_lang.lower() not in tts_config.languages: - return JSONResponse(status_code=400, content={"message": "prompt_lang is not supported"}) - if media_type not in ["wav", "raw", "ogg", "aac"]: - return JSONResponse(status_code=400, content={"message": "media_type is not supported"}) - elif media_type == "ogg" and not streaming_mode: - return JSONResponse(status_code=400, content={"message": "ogg format is not supported in non-streaming mode"}) - - if text_split_method not in cut_method_names: - return JSONResponse(status_code=400, content={"message": f"text_split_method:{text_split_method} is not supported"}) - - return None - -async def tts_handle(req:dict): - """ - Text to speech handler. - - Args: - req (dict): - { - "text": "", # str.(required) text to be synthesized - "text_lang: "", # str.(required) language of the text to be synthesized - "ref_audio_path": "", # str.(required) reference audio path - "prompt_text": "", # str.(optional) prompt text for the reference audio - "prompt_lang": "", # str.(required) language of the prompt text for the reference audio - "top_k": 5, # int. top k sampling - "top_p": 1, # float. top p sampling - "temperature": 1, # float. temperature for sampling - "text_split_method": "cut5", # str. text split method, see text_segmentation_method.py for details. - "batch_size": 1, # int. batch size for inference - "batch_threshold": 0.75, # float. threshold for batch splitting. - "split_bucket: True, # bool. whether to split the batch into multiple buckets. - "speed_factor":1.0, # float. control the speed of the synthesized audio. - "fragment_interval":0.3, # float. to control the interval of the audio fragment. - "seed": -1, # int. random seed for reproducibility. - "media_type": "wav", # str. media type of the output audio, support "wav", "raw", "ogg", "aac". - "streaming_mode": False, # bool. whether to return a streaming response. - "parallel_infer": True, # bool.(optional) whether to use parallel inference. - "repetition_penalty": 1.35 # float.(optional) repetition penalty for T2S model. - } - returns: - StreamingResponse: audio stream response. - """ - - streaming_mode = req.get("streaming_mode", False) - media_type = req.get("media_type", "wav") - - check_res = check_params(req) - if check_res is not None: - return check_res - - if streaming_mode: - req["return_fragment"] = True - - try: - tts_generator=tts_pipeline.run(req) - - if streaming_mode: - def streaming_generator(tts_generator:Generator, media_type:str): - if media_type == "wav": - yield wave_header_chunk() - media_type = "raw" - for sr, chunk in tts_generator: - yield pack_audio(BytesIO(), chunk, sr, media_type).getvalue() - # _media_type = f"audio/{media_type}" if not (streaming_mode and media_type in ["wav", "raw"]) else f"audio/x-{media_type}" - return StreamingResponse(streaming_generator(tts_generator, media_type, ), media_type=f"audio/{media_type}") - - else: - sr, audio_data = next(tts_generator) - audio_data = pack_audio(BytesIO(), audio_data, sr, media_type).getvalue() - return Response(audio_data, media_type=f"audio/{media_type}") - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"tts failed", "Exception": str(e)}) - - - - - - -@APP.get("/control") -async def control(command: str = None): - if command is None: - return JSONResponse(status_code=400, content={"message": "command is required"}) - handle_control(command) - - - -@APP.get("/tts") -async def tts_get_endpoint( - text: str = None, - text_lang: str = None, - ref_audio_path: str = None, - prompt_lang: str = None, - prompt_text: str = "", - top_k:int = 5, - top_p:float = 1, - temperature:float = 1, - text_split_method:str = "cut0", - batch_size:int = 1, - batch_threshold:float = 0.75, - split_bucket:bool = True, - speed_factor:float = 1.0, - fragment_interval:float = 0.3, - seed:int = -1, - media_type:str = "wav", - streaming_mode:bool = False, - parallel_infer:bool = True, - repetition_penalty:float = 1.35 - ): - req = { - "text": text, - "text_lang": text_lang.lower(), - "ref_audio_path": ref_audio_path, - "prompt_text": prompt_text, - "prompt_lang": prompt_lang.lower(), - "top_k": top_k, - "top_p": top_p, - "temperature": temperature, - "text_split_method": text_split_method, - "batch_size":int(batch_size), - "batch_threshold":float(batch_threshold), - "speed_factor":float(speed_factor), - "split_bucket":split_bucket, - "fragment_interval":fragment_interval, - "seed":seed, - "media_type":media_type, - "streaming_mode":streaming_mode, - "parallel_infer":parallel_infer, - "repetition_penalty":float(repetition_penalty) - } - return await tts_handle(req) - - -@APP.post("/tts") -async def tts_post_endpoint(request: TTS_Request): - req = request.dict() - return await tts_handle(req) - - -@APP.get("/set_refer_audio") -async def set_refer_aduio(refer_audio_path: str = None): - try: - tts_pipeline.set_ref_audio(refer_audio_path) - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"set refer audio failed", "Exception": str(e)}) - return JSONResponse(status_code=200, content={"message": "success"}) - - -# @APP.post("/set_refer_audio") -# async def set_refer_aduio_post(audio_file: UploadFile = File(...)): -# try: -# # 检查文件类型，确保是音频文件 -# if not audio_file.content_type.startswith("audio/"): -# return JSONResponse(status_code=400, content={"message": "file type is not supported"}) - -# os.makedirs("uploaded_audio", exist_ok=True) -# save_path = os.path.join("uploaded_audio", audio_file.filename) -# # 保存音频文件到服务器上的一个目录 -# with open(save_path , "wb") as buffer: -# buffer.write(await audio_file.read()) - -# tts_pipeline.set_ref_audio(save_path) -# except Exception as e: -# return JSONResponse(status_code=400, content={"message": f"set refer audio failed", "Exception": str(e)}) -# return JSONResponse(status_code=200, content={"message": "success"}) - -@APP.get("/set_gpt_weights") -async def set_gpt_weights(weights_path: str = None): - try: - if weights_path in ["", None]: - return JSONResponse(status_code=400, content={"message": "gpt weight path is required"}) - tts_pipeline.init_t2s_weights(weights_path) - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"change gpt weight failed", "Exception": str(e)}) - - return JSONResponse(status_code=200, content={"message": "success"}) - - -@APP.get("/set_sovits_weights") -async def set_sovits_weights(weights_path: str = None): - try: - if weights_path in ["", None]: - return JSONResponse(status_code=400, content={"message": "sovits weight path is required"}) - tts_pipeline.init_vits_weights(weights_path) - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"change sovits weight failed", "Exception": str(e)}) - return JSONResponse(status_code=200, content={"message": "success"}) - - - -if __name__ == "__main__": - try: - uvicorn.run(app="api_v2:APP", host=host, port=port, workers=1) - except Exception as e: - traceback.print_exc() - os.kill(os.getpid(), signal.SIGTERM) - exit(0) diff --git a/tts/GPT-SoVITS/api_v3.py b/tts/GPT-SoVITS/api_v3.py deleted file mode 100644 index e830113..0000000 --- a/tts/GPT-SoVITS/api_v3.py +++ /dev/null @@ -1,469 +0,0 @@ -""" -# WebAPI文档 (3.0) - 使用了缓存技术，初始化时使用LRU Cache TTS 实例，缓存加载模型的世界，达到减少切换不同语音时的推理时间 - -` python api_v2.py -a 127.0.0.1 -p 9880 -c GPT_SoVITS/configs/tts_infer.yaml ` - -## 执行参数: - `-a` - `绑定地址, 默认"127.0.0.1"` - `-p` - `绑定端口, 默认9880` - `-c` - `TTS配置文件路径, 默认"GPT_SoVITS/configs/tts_infer.yaml"` - -## 调用: - -### 推理 - -endpoint: `/tts` -GET: -``` -http://127.0.0.1:9880/tts?text=先帝创业未半而中道崩殂，今天下三分，益州疲弊，此诚危急存亡之秋也。&text_lang=zh&ref_audio_path=archive_jingyuan_1.wav&prompt_lang=zh&prompt_text=我是「罗浮」云骑将军景元。不必拘谨，「将军」只是一时的身份，你称呼我景元便可&text_split_method=cut5&batch_size=1&media_type=wav&streaming_mode=true -``` - -POST: -```json -{ - "text": "", # str.(required) text to be synthesized - "text_lang": "", # str.(required) language of the text to be synthesized - "ref_audio_path": "", # str.(required) reference audio path. - "prompt_text": "", # str.(optional) prompt text for the reference audio - "prompt_lang": "", # str.(required) language of the prompt text for the reference audio - "top_k": 5, # int.(optional) top k sampling - "top_p": 1, # float.(optional) top p sampling - "temperature": 1, # float.(optional) temperature for sampling - "text_split_method": "cut5", # str.(optional) text split method, see text_segmentation_method.py for details. - "batch_size": 1, # int.(optional) batch size for inference - "batch_threshold": 0.75, # float.(optional) threshold for batch splitting. - "split_bucket": true, # bool.(optional) whether to split the batch into multiple buckets. - "speed_factor":1.0, # float.(optional) control the speed of the synthesized audio. - "fragment_interval":0.3, # float.(optional) to control the interval of the audio fragment. - "seed": -1, # int.(optional) random seed for reproducibility. - "media_type": "wav", # str.(optional) media type of the output audio, support "wav", "raw", "ogg", "aac". - "streaming_mode": false, # bool.(optional) whether to return a streaming response. - "parallel_infer": True, # bool.(optional) whether to use parallel inference. - "repetition_penalty": 1.35, # float.(optional) repetition penalty for T2S model. - "tts_infer_yaml_path": “GPT_SoVITS/configs/tts_infer.yaml” # str.(optional) tts infer yaml path -} -``` - -RESP: -成功: 直接返回 wav 音频流， http code 200 -失败: 返回包含错误信息的 json, http code 400 - -### 命令控制 - -endpoint: `/control` - -command: -"restart": 重新运行 -"exit": 结束运行 - -GET: -``` -http://127.0.0.1:9880/control?command=restart -``` -POST: -```json -{ - "command": "restart" -} -``` - -RESP: 无 - - -### 切换GPT模型 - -endpoint: `/set_gpt_weights` - -GET: -``` -http://127.0.0.1:9880/set_gpt_weights?weights_path=GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt -``` -RESP: -成功: 返回"success", http code 200 -失败: 返回包含错误信息的 json, http code 400 - - -### 切换Sovits模型 - -endpoint: `/set_sovits_weights` - -GET: -``` -http://127.0.0.1:9880/set_sovits_weights?weights_path=GPT_SoVITS/pretrained_models/s2G488k.pth -``` - -RESP: -成功: 返回"success", http code 200 -失败: 返回包含错误信息的 json, http code 400 - -""" -import os -import sys -import traceback -from typing import Generator - -import torch - -now_dir = os.getcwd() -sys.path.append(now_dir) -sys.path.append("%s/GPT_SoVITS" % (now_dir)) - -import argparse -import subprocess -import wave -import signal -import numpy as np -import soundfile as sf -from fastapi import Response -from fastapi.responses import JSONResponse -from fastapi import FastAPI -import uvicorn -from io import BytesIO -from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config -from GPT_SoVITS.TTS_infer_pack.text_segmentation_method import get_method_names as get_cut_method_names -from fastapi.responses import StreamingResponse -from pydantic import BaseModel -from functools import lru_cache - -cut_method_names = get_cut_method_names() - -parser = argparse.ArgumentParser(description="GPT-SoVITS api") -parser.add_argument("-a", "--bind_addr", type=str, default="0.0.0.0", help="default: 0.0.0.0") -parser.add_argument("-p", "--port", type=int, default="9880", help="default: 9880") -args = parser.parse_args() -port = args.port -host = args.bind_addr -argv = sys.argv - -APP = FastAPI() - - -class TTS_Request(BaseModel): - text: str = None - text_lang: str = None - ref_audio_path: str = None - prompt_lang: str = None - prompt_text: str = "" - top_k: int = 5 - top_p: float = 1 - temperature: float = 1 - text_split_method: str = "cut5" - batch_size: int = 1 - batch_threshold: float = 0.75 - split_bucket: bool = True - speed_factor: float = 1.0 - fragment_interval: float = 0.3 - seed: int = -1 - media_type: str = "wav" - streaming_mode: bool = False - parallel_infer: bool = True - repetition_penalty: float = 1.35 - tts_infer_yaml_path: str = None - """推理时需要加载的声音模型的yaml配置文件路径，如：GPT_SoVITS/configs/tts_infer.yaml""" - - -@lru_cache(maxsize=10) -def get_tts_instance(tts_config: TTS_Config) -> TTS: - print(f"load tts config from {tts_config.configs_path}") - return TTS(tts_config) - - -def pack_ogg(io_buffer: BytesIO, data: np.ndarray, rate: int): - """modify from https://github.com/RVC-Boss/GPT-SoVITS/pull/894/files""" - with sf.SoundFile(io_buffer, mode='w', samplerate=rate, channels=1, format='ogg') as audio_file: - audio_file.write(data) - return io_buffer - - -def pack_raw(io_buffer: BytesIO, data: np.ndarray, rate: int): - io_buffer.write(data.tobytes()) - return io_buffer - - -def pack_wav(io_buffer: BytesIO, data: np.ndarray, rate: int): - io_buffer = BytesIO() - sf.write(io_buffer, data, rate, format='wav') - return io_buffer - - -def pack_aac(io_buffer: BytesIO, data: np.ndarray, rate: int): - process = subprocess.Popen([ - 'ffmpeg', - '-f', 's16le', # 输入16位有符号小端整数PCM - '-ar', str(rate), # 设置采样率 - '-ac', '1', # 单声道 - '-i', 'pipe:0', # 从管道读取输入 - '-c:a', 'aac', # 音频编码器为AAC - '-b:a', '192k', # 比特率 - '-vn', # 不包含视频 - '-f', 'adts', # 输出AAC数据流格式 - 'pipe:1' # 将输出写入管道 - ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - out, _ = process.communicate(input=data.tobytes()) - io_buffer.write(out) - return io_buffer - - -def pack_audio(io_buffer: BytesIO, data: np.ndarray, rate: int, media_type: str): - if media_type == "ogg": - io_buffer = pack_ogg(io_buffer, data, rate) - elif media_type == "aac": - io_buffer = pack_aac(io_buffer, data, rate) - elif media_type == "wav": - io_buffer = pack_wav(io_buffer, data, rate) - else: - io_buffer = pack_raw(io_buffer, data, rate) - io_buffer.seek(0) - return io_buffer - - -# from https://huggingface.co/spaces/coqui/voice-chat-with-mistral/blob/main/app.py -def wave_header_chunk(frame_input=b"", channels=1, sample_width=2, sample_rate=32000): - # This will create a wave header then append the frame input - # It should be first on a streaming wav file - # Other frames better should not have it (else you will hear some artifacts each chunk start) - wav_buf = BytesIO() - with wave.open(wav_buf, "wb") as vfout: - vfout.setnchannels(channels) - vfout.setsampwidth(sample_width) - vfout.setframerate(sample_rate) - vfout.writeframes(frame_input) - - wav_buf.seek(0) - return wav_buf.read() - - -def handle_control(command: str): - if command == "restart": - os.execl(sys.executable, sys.executable, *argv) - elif command == "exit": - os.kill(os.getpid(), signal.SIGTERM) - exit(0) - - -def check_params(req: dict, tts_config: TTS_Config): - text: str = req.get("text", "") - text_lang: str = req.get("text_lang", "") - ref_audio_path: str = req.get("ref_audio_path", "") - streaming_mode: bool = req.get("streaming_mode", False) - media_type: str = req.get("media_type", "wav") - prompt_lang: str = req.get("prompt_lang", "") - text_split_method: str = req.get("text_split_method", "cut5") - - if ref_audio_path in [None, ""]: - return JSONResponse(status_code=400, content={"message": "ref_audio_path is required"}) - if text in [None, ""]: - return JSONResponse(status_code=400, content={"message": "text is required"}) - if (text_lang in [None, ""]): - return JSONResponse(status_code=400, content={"message": "text_lang is required"}) - elif text_lang.lower() not in tts_config.languages: - return JSONResponse(status_code=400, content={"message": "text_lang is not supported"}) - if (prompt_lang in [None, ""]): - return JSONResponse(status_code=400, content={"message": "prompt_lang is required"}) - elif prompt_lang.lower() not in tts_config.languages: - return JSONResponse(status_code=400, content={"message": "prompt_lang is not supported"}) - if media_type not in ["wav", "raw", "ogg", "aac"]: - return JSONResponse(status_code=400, content={"message": "media_type is not supported"}) - elif media_type == "ogg" and not streaming_mode: - return JSONResponse(status_code=400, content={"message": "ogg format is not supported in non-streaming mode"}) - - if text_split_method not in cut_method_names: - return JSONResponse(status_code=400, - content={"message": f"text_split_method:{text_split_method} is not supported"}) - - return None - - -async def tts_handle(req: dict): - """ - Text to speech handler. - - Args: - req (dict): - { - "text": "", # str.(required) text to be synthesized - "text_lang: "", # str.(required) language of the text to be synthesized - "ref_audio_path": "", # str.(required) reference audio path - "prompt_text": "", # str.(optional) prompt text for the reference audio - "prompt_lang": "", # str.(required) language of the prompt text for the reference audio - "top_k": 5, # int. top k sampling - "top_p": 1, # float. top p sampling - "temperature": 1, # float. temperature for sampling - "text_split_method": "cut5", # str. text split method, see text_segmentation_method.py for details. - "batch_size": 1, # int. batch size for inference - "batch_threshold": 0.75, # float. threshold for batch splitting. - "split_bucket: True, # bool. whether to split the batch into multiple buckets. - "speed_factor":1.0, # float. control the speed of the synthesized audio. - "fragment_interval":0.3, # float. to control the interval of the audio fragment. - "seed": -1, # int. random seed for reproducibility. - "media_type": "wav", # str. media type of the output audio, support "wav", "raw", "ogg", "aac". - "streaming_mode": False, # bool. whether to return a streaming response. - "parallel_infer": True, # bool.(optional) whether to use parallel inference. - "repetition_penalty": 1.35 # float.(optional) repetition penalty for T2S model. - } - returns: - StreamingResponse: audio stream response. - """ - - streaming_mode = req.get("streaming_mode", False) - media_type = req.get("media_type", "wav") - tts_infer_yaml_path = req.get("tts_infer_yaml_path", "GPT_SoVITS/configs/tts_infer.yaml") - - tts_config = TTS_Config(tts_infer_yaml_path) - check_res = check_params(req, tts_config) - if check_res is not None: - return check_res - - if streaming_mode: - req["return_fragment"] = True - - try: - tts_instance = get_tts_instance(tts_config) - - move_to_original(tts_instance, tts_config) - - tts_generator = tts_instance.run(req) - - if streaming_mode: - def streaming_generator(tts_generator: Generator, media_type: str): - if media_type == "wav": - yield wave_header_chunk() - media_type = "raw" - for sr, chunk in tts_generator: - yield pack_audio(BytesIO(), chunk, sr, media_type).getvalue() - move_to_cpu(tts_instance) - - # _media_type = f"audio/{media_type}" if not (streaming_mode and media_type in ["wav", "raw"]) else f"audio/x-{media_type}" - return StreamingResponse(streaming_generator(tts_generator, media_type, ), media_type=f"audio/{media_type}") - - else: - sr, audio_data = next(tts_generator) - audio_data = pack_audio(BytesIO(), audio_data, sr, media_type).getvalue() - move_to_cpu(tts_instance) - return Response(audio_data, media_type=f"audio/{media_type}") - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"tts failed", "Exception": str(e)}) - - -def move_to_cpu(tts): - cpu_device = torch.device('cpu') - tts.set_device(cpu_device, False) - tts.enable_half_precision(False, False) - print("Moved TTS models to CPU to save GPU memory.") - - -def move_to_original(tts: TTS, tts_config: TTS_Config): - tts.set_device(tts_config.device, False) - tts.enable_half_precision(tts_config.is_half, False) - print("Moved TTS models back to original device for performance.") - - -@APP.get("/control") -async def control(command: str = None): - if command is None: - return JSONResponse(status_code=400, content={"message": "command is required"}) - handle_control(command) - - -@APP.get("/tts") -async def tts_get_endpoint( - text: str = None, - text_lang: str = None, - ref_audio_path: str = None, - prompt_lang: str = None, - prompt_text: str = "", - top_k: int = 5, - top_p: float = 1, - temperature: float = 1, - text_split_method: str = "cut0", - batch_size: int = 1, - batch_threshold: float = 0.75, - split_bucket: bool = True, - speed_factor: float = 1.0, - fragment_interval: float = 0.3, - seed: int = -1, - media_type: str = "wav", - streaming_mode: bool = False, - parallel_infer: bool = True, - repetition_penalty: float = 1.35, - tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml" -): - req = { - "text": text, - "text_lang": text_lang.lower(), - "ref_audio_path": ref_audio_path, - "prompt_text": prompt_text, - "prompt_lang": prompt_lang.lower(), - "top_k": top_k, - "top_p": top_p, - "temperature": temperature, - "text_split_method": text_split_method, - "batch_size": int(batch_size), - "batch_threshold": float(batch_threshold), - "speed_factor": float(speed_factor), - "split_bucket": split_bucket, - "fragment_interval": fragment_interval, - "seed": seed, - "media_type": media_type, - "streaming_mode": streaming_mode, - "parallel_infer": parallel_infer, - "repetition_penalty": float(repetition_penalty), - "tts_infer_yaml_path": tts_infer_yaml_path - } - - return await tts_handle(req) - - -@APP.post("/tts") -async def tts_post_endpoint(request: TTS_Request): - req = request.dict() - return await tts_handle(req) - - -@APP.get("/set_refer_audio") -async def set_refer_audio(refer_audio_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): - try: - tts_config = TTS_Config(tts_infer_yaml_path) - tts_instance = get_tts_instance(tts_config) - tts_instance.set_ref_audio(refer_audio_path) - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"set refer audio failed", "Exception": str(e)}) - return JSONResponse(status_code=200, content={"message": "success"}) - - -@APP.get("/set_gpt_weights") -async def set_gpt_weights(weights_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): - try: - if weights_path in ["", None]: - return JSONResponse(status_code=400, content={"message": "gpt weight path is required"}) - - tts_config = TTS_Config(tts_infer_yaml_path) - tts_instance = get_tts_instance(tts_config) - tts_instance.init_t2s_weights(weights_path) - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"change gpt weight failed", "Exception": str(e)}) - - return JSONResponse(status_code=200, content={"message": "success"}) - - -@APP.get("/set_sovits_weights") -async def set_sovits_weights(weights_path: str = None, tts_infer_yaml_path: str = "GPT_SoVITS/configs/tts_infer.yaml"): - try: - if weights_path in ["", None]: - return JSONResponse(status_code=400, content={"message": "sovits weight path is required"}) - - tts_config = TTS_Config(tts_infer_yaml_path) - tts_instance = get_tts_instance(tts_config) - tts_instance.init_vits_weights(weights_path) - except Exception as e: - return JSONResponse(status_code=400, content={"message": f"change sovits weight failed", "Exception": str(e)}) - return JSONResponse(status_code=200, content={"message": "success"}) - - -if __name__ == "__main__": - try: - uvicorn.run(APP, host=host, port=port, workers=1) - except Exception as e: - traceback.print_exc() - os.kill(os.getpid(), signal.SIGTERM) - exit(0) diff --git a/tts/GPT-SoVITS/fakeyou_infer.py b/tts/GPT-SoVITS/fakeyou_infer.py index c89cdab..92b2cf2 100644 --- a/tts/GPT-SoVITS/fakeyou_infer.py +++ b/tts/GPT-SoVITS/fakeyou_infer.py @@ -1,172 +1,600 @@ -import sys import os - -# Add current working directory and specific subdirectories to sys.path -now_dir = os.getcwd() -sys.path.insert(0, now_dir) -sys.path.insert(0, os.path.join(now_dir, "tools")) -sys.path.insert(0, os.path.join(now_dir, "tools/damo_asr")) -sys.path.insert(0, os.path.join(now_dir, "GPT_SoVITS")) -sys.path.insert(0, os.path.join(now_dir, "tools/uvr5")) - -print("Current working directory:", os.getcwd()) -print("Python sys.path:", sys.path) - +import sys +import re +import logging import argparse -import traceback -import random import torch -import numpy as np import soundfile as sf -import yaml -from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config -from GPT_SoVITS.TTS_infer_pack.text_segmentation_method import get_method_names as get_cut_method_names +import numpy as np +import librosa +from time import time as ttime +from transformers import AutoModelForMaskedLM, AutoTokenizer + +# Add GPT_SoVITS to the path +sys.path.append(os.path.join(os.getcwd(), "GPT_SoVITS")) + +# Importing necessary modules +import LangSegment +from module.models import SynthesizerTrn +from AR.models.t2s_lightning_module import Text2SemanticLightningModule +from text import cleaned_text_to_sequence +from text.cleaner import clean_text +from module.mel_processing import spectrogram_torch +from my_utils import load_audio +from tools.i18n.i18n import I18nAuto + +# Additional Imports and GPU Info +import pdb -# Print environment variables print("Env vars:") print(os.environ) + def print_gpu_info(): print('========================================') - print('Python interpreter:', sys.executable) - print('PyTorch version:', torch.__version__) + print('Python interpreter', sys.executable) + print('PyTorch version', torch.__version__) print('CUDA Available?', torch.cuda.is_available()) - print('CUDA Device count:', torch.cuda.device_count()) - print('CUDA architectures library was compiled for:', torch.cuda.get_arch_list()) + print('CUDA Device count', torch.cuda.device_count()) + print('CUDA architectures library was compiled for', torch.cuda.get_arch_list()) print('========================================', flush=True) print_gpu_info() -cut_method_names = get_cut_method_names() +# Setup logging +logging.getLogger("markdown_it").setLevel(logging.ERROR) +logging.getLogger("urllib3").setLevel(logging.ERROR) +logging.getLogger("httpcore").setLevel(logging.ERROR) +logging.getLogger("httpx").setLevel(logging.ERROR) +logging.getLogger("asyncio").setLevel(logging.ERROR) +logging.getLogger("charset_normalizer").setLevel(logging.ERROR) +logging.getLogger("torchaudio._extension").setLevel(logging.ERROR) -def load_yaml_config(yaml_path): - with open(yaml_path, 'r', encoding='utf-8') as f: - config_data = yaml.safe_load(f) - return config_data +# Define language dictionary +dict_language = { + "chinese": "all_zh", + "english": "en", + "japanese": "all_ja", + "chinese+english": "zh", + "japanese+english": "ja", + "automatic": "auto", +} -def create_tts_config(config_path): - config_data = load_yaml_config(config_path) - config = config_data.get('custom', config_data.get('default', {})) +# Define default paths for pretrained models +pretrained_gpt_path = "GPT_SoVITS/pretrained_models/s1bert25hz-2kh-longer-epoch=68e-step=50232.ckpt" +pretrained_sovits_path = "GPT_SoVITS/pretrained_models/s2G488k.pth" - tts_config = TTS_Config(config) - tts_config.device = config.get('device', 'cpu') - tts_config.is_half = config.get('is_half', False) +# Paths for cnhubert and bert models +cnhubert_base_path = os.environ.get("cnhubert_base_path", "GPT_SoVITS/pretrained_models/chinese-hubert-base") +bert_path = os.environ.get("bert_path", "GPT_SoVITS/pretrained_models/chinese-roberta-wwm-ext-large") - return tts_config +if "_CUDA_VISIBLE_DEVICES" in os.environ: + os.environ["CUDA_VISIBLE_DEVICES"] = os.environ["_CUDA_VISIBLE_DEVICES"] +is_half = eval(os.environ.get("is_half", "True")) and torch.cuda.is_available() -def print_tts_config(config): - print("---------------------------------------------TTS Config---------------------------------------------") - print(f"device : {config.device}") - print(f"is_half : {config.is_half}") - print(f"t2s_weights_path : {config.t2s_weights_path}") - print(f"vits_weights_path : {config.vits_weights_path}") - print(f"bert_base_path : {config.bert_base_path}") - print(f"cnhuhbert_base_path : {config.cnhuhbert_base_path}") - print("----------------------------------------------------------------------------------------------------") +# Initialize transformers and utility classes +i18n = I18nAuto() -def inference(config, text, text_lang, ref_audio_path, prompt_text, prompt_lang, top_k, top_p, temperature, text_split_method, batch_size, speed_factor, ref_text_free, split_bucket, fragment_interval, seed, keep_random, parallel_infer, repetition_penalty, output_path): - seed = -1 if keep_random else seed - actual_seed = seed if seed not in [-1, "", None] else random.randrange(1 << 32) - inputs = { - "text": text, - "text_lang": text_lang, - "ref_audio_path": ref_audio_path, - "prompt_text": prompt_text if not ref_text_free else "", - "prompt_lang": prompt_lang, - "top_k": top_k, - "top_p": top_p, - "temperature": temperature, - "text_split_method": text_split_method, - "batch_size": int(batch_size), - "speed_factor": float(speed_factor), - "split_bucket": split_bucket, - "return_fragment": False, - "fragment_interval": fragment_interval, - "seed": actual_seed, - "parallel_infer": parallel_infer, - "repetition_penalty": repetition_penalty, - } +# Setup device +device = "cuda" if torch.cuda.is_available() else "cpu" + +# Ensure feature_extractor module is imported +try: + from feature_extractor import cnhubert +except ImportError: + print("Make sure the feature_extractor module is in the GPT_SoVITS directory.") + sys.exit(1) + +cnhubert.cnhubert_base_path = cnhubert_base_path + +# Define global variables for loaded model paths +loaded_gpt_path = None +loaded_sovits_path = None + +# Example: Get BERT feature +def get_bert_feature(text, word2ph): + global tokenizer, bert_model + if not "tokenizer" in globals() or not "bert_model" in globals(): + tokenizer = AutoTokenizer.from_pretrained(bert_path) + bert_model = AutoModelForMaskedLM.from_pretrained(bert_path) + if is_half: + bert_model = bert_model.half().to(device) + else: + bert_model = bert_model.to(device) + with torch.no_grad(): + inputs = tokenizer(text, return_tensors="pt") + for i in inputs: + inputs[i] = inputs[i].to(device) + res = bert_model(**inputs, output_hidden_states=True) + res = torch.cat(res["hidden_states"][-3:-2], -1)[0].cpu()[1:-1] + assert len(word2ph) == len(text) + phone_level_feature = [] + for i in range(len(word2ph)): + repeat_feature = res[i].repeat(word2ph[i], 1) + phone_level_feature.append(repeat_feature) + phone_level_feature = torch.cat(phone_level_feature, dim=0) + return phone_level_feature.T + +class DictToAttrRecursive(dict): + def __init__(self, input_dict): + super().__init__(input_dict) + for key, value in input_dict.items(): + if isinstance(value, dict): + value = DictToAttrRecursive(value) + self[key] = value + setattr(self, key, value) + + def __getattr__(self, item): + try: + return self[item] + except KeyError: + raise AttributeError(f"Attribute {item} not found") + + def __setattr__(self, key, value): + if isinstance(value, dict): + value = DictToAttrRecursive(value) + super(DictToAttrRecursive, self).__setitem__(key, value) + super().__setattr__(key, value) + + def __delattr__(self, item): + try: + del self[item] + except KeyError: + raise AttributeError(f"Attribute {item} not found") + +def change_sovits_weights(sovits_path): + global vq_model, hps + dict_s2 = torch.load(sovits_path, map_location="cpu") + hps = dict_s2["config"] + hps = DictToAttrRecursive(hps) + hps.model.semantic_frame_rate = "25hz" + vq_model = SynthesizerTrn( + hps.data.filter_length // 2 + 1, + hps.train.segment_size // hps.data.hop_length, + n_speakers=hps.data.n_speakers, + **hps.model + ) + if "pretrained" not in sovits_path: + del vq_model.enc_q + if is_half: + vq_model = vq_model.half().to(device) + else: + vq_model = vq_model.to(device) + vq_model.eval() + print(vq_model.load_state_dict(dict_s2["weight"], strict=False)) + with open("./sweight.txt", "w", encoding="utf-8") as f: + f.write(sovits_path) + +def change_gpt_weights(gpt_path): + global hz, max_sec, t2s_model, config + hz = 50 + dict_s1 = torch.load(gpt_path, map_location="cpu") + config = dict_s1["config"] + max_sec = config["data"]["max_sec"] + t2s_model = Text2SemanticLightningModule(config, "****", is_train=False) + t2s_model.load_state_dict(dict_s1["weight"]) + if is_half: + t2s_model = t2s_model.half() + t2s_model = t2s_model.to(device) + t2s_model.eval() + total = sum([param.nelement() for param in t2s_model.parameters()]) + print("Number of parameter: %.2fM" % (total / 1e6)) + with open("./gweight.txt", "w", encoding="utf-8") as f: + f.write(gpt_path) + +def get_spepc(hps, filename): + audio = load_audio(filename, int(hps.data.sampling_rate)) + audio = torch.FloatTensor(audio) + audio_norm = audio + audio_norm = audio_norm.unsqueeze(0) + spec = spectrogram_torch( + audio_norm, + hps.data.filter_length, + hps.data.sampling_rate, + hps.data.hop_length, + hps.data.win_length, + center=False, + ) + return spec + +def clean_text_inf(text, language): + phones, word2ph, norm_text = clean_text(text, language) + print(f'Phonemes for "{text}":\n{phones}') + phones = cleaned_text_to_sequence(phones) + return phones, word2ph, norm_text + +dtype = torch.float16 if is_half else torch.float32 + +def get_bert_inf(phones, word2ph, norm_text, language): + language = language.replace("all_", "") + if language == "zh": + bert = get_bert_feature(norm_text, word2ph).to(device) + else: + bert = torch.zeros( + (1024, len(phones)), + dtype=torch.float16 if is_half else torch.float32, + ).to(device) + return bert + +splits = {"，", "。", "？", "！", ",", ".", "?", "!", "~", ":", "：", "—", "…", } + +def get_first(text): + pattern = "[" + "".join(re.escape(sep) for sep in splits) + "]" + text = re.split(pattern, text)[0].strip() + return text + +def get_phones_and_bert(text, language): + phones = None + bert = None + norm_text = None - tts_pipeline = TTS(config) + if language in {"en", "all_zh", "all_ja"}: + language = language.replace("all_", "") + if language == "en": + LangSegment.setfilters(["en"]) + formattext = " ".join(tmp["text"] for tmp in LangSegment.getTexts(text)) + else: + formattext = text + while " " in formattext: + formattext = formattext.replace(" ", " ") + phones, word2ph, norm_text = clean_text_inf(formattext, language) + if language == "zh": + bert = get_bert_feature(norm_text, word2ph).to(device) + else: + bert = torch.zeros( + (1024, len(phones)), + dtype=torch.float16 if is_half else torch.float32, + ).to(device) + elif language in {"zh", "ja", "auto"}: + textlist = [] + langlist = [] + LangSegment.setfilters(["zh", "ja", "en", "ko"]) + if language == "auto": + for tmp in LangSegment.getTexts(text): + if tmp["lang"] == "ko": + langlist.append("zh") + textlist.append(tmp["text"]) + else: + langlist.append(tmp["lang"]) + textlist.append(tmp["text"]) + else: + for tmp in LangSegment.getTexts(text): + if tmp["lang"] == "en": + langlist.append(tmp["lang"]) + else: + langlist.append(language) + textlist.append(tmp["text"]) + print(textlist) + print(langlist) + phones_list = [] + bert_list = [] + norm_text_list = [] + for i in range(len(textlist)): + lang = langlist[i] + phones, word2ph, norm_text = clean_text_inf(textlist[i], lang) + bert = get_bert_inf(phones, word2ph, norm_text, lang) + phones_list.append(phones) + norm_text_list.append(norm_text) + bert_list.append(bert) + bert = torch.cat(bert_list, dim=1) + phones = sum(phones_list, []) + norm_text = ''.join(norm_text_list) + else: + raise ValueError(f"Unsupported language: {language}") - print(f"Using Text2Semantic model: {config.t2s_weights_path}") - print(f"Using VITS model: {config.vits_weights_path}") + return phones, bert.to(dtype), norm_text - for item in tts_pipeline.run(inputs): - audio_data = item[1] # assuming item[1] is the audio data - if not os.path.exists(output_path): - os.makedirs(output_path) - output_file = os.path.join(output_path, f"output_{actual_seed}.wav") - sf.write(output_file, audio_data, 32000, subtype='PCM_16') - print(f"Output saved to {output_file}, Seed: {actual_seed}") +def merge_short_text_in_array(texts, threshold): + if len(texts) < 2: + return texts + result = [] + text = "" + for ele in texts: + text += ele + if len(text) >= threshold: + result.append(text) + text = "" + if len(text) > 0: + if len(result) == 0: + result.append(text) + else: + result[len(result) - 1] += text + return result +ssl_model = cnhubert.get_model() +if is_half: + ssl_model = ssl_model.half().to(device) +else: + ssl_model = ssl_model.to(device) + +def get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language, how_to_cut=i18n("不切"), top_k=20, top_p=0.6, temperature=0.6, ref_free=False): + t0 = ttime() + if prompt_text is None or len(prompt_text) == 0: + ref_free = True + prompt_language = dict_language.get(prompt_language, None) + if not ref_free and prompt_language is None: + raise ValueError(f"Unsupported language: {prompt_language}") + text_language = dict_language[text_language] + + if not ref_free: + prompt_text = prompt_text.strip("\n") + if prompt_text[-1] not in splits: + prompt_text += "。" if prompt_language != "en" else "." + print(f"Reference text: {prompt_text}") + phones1, bert1, norm_text1 = get_phones_and_bert(prompt_text, prompt_language) + text = text.strip("\n") + if text[0] not in splits and len(get_first(text)) < 4: + text = "。" + text if text_language != "en" else "." + text + + print(f"Full text: {text}") + zero_wav = np.zeros( + int(hps.data.sampling_rate * 0.3), + dtype=np.float16 if is_half else np.float32, + ) + + if not ref_free: + with torch.no_grad(): + wav16k, sr = librosa.load(ref_wav_path, sr=16000) + if wav16k.shape[0] > 160000 or wav16k.shape[0] < 48000: + raise OSError("参考音频在3~10秒范围外，请更换！") + wav16k = torch.from_numpy(wav16k) + zero_wav_torch = torch.from_numpy(zero_wav) + if is_half: + wav16k = wav16k.half().to(device) + zero_wav_torch = zero_wav_torch.half().to(device) + else: + wav16k = wav16k.to(device) + zero_wav_torch = zero_wav_torch.to(device) + wav16k = torch.cat([wav16k, zero_wav_torch]) + ssl_content = ssl_model.model(wav16k.unsqueeze(0))["last_hidden_state"].transpose(1, 2) + codes = vq_model.extract_latent(ssl_content) + prompt_semantic = codes[0, 0] + t1 = ttime() + + if how_to_cut == "Slice once every 4 sentences": + text = cut1(text) + elif how_to_cut == "Cut per 50 characters": + text = cut2(text) + elif how_to_cut == "Slice by Chinese punct": + text = cut3(text) + elif how_to_cut == "Slice by English punct": + text = cut4(text) + elif how_to_cut == "Slice by every punct": + text = cut5(text) + while "\n\n" in text: + text = text.replace("\n\n", "\n") + print(f"Text after slicing:\n{text}") + texts = text.split("\n") + texts = merge_short_text_in_array(texts, 5) + audio_opt = [] + + t2 = ttime() + t3, t4, t5 = 0, 0, 0 + + for text in texts: + t3s = ttime() + if len(text.strip()) == 0: + continue + if text[-1] not in splits: + text += "。" if text_language != "en" else "." + print(f"Text being generated: {text}") + phones2, bert2, norm_text2 = get_phones_and_bert(text, text_language) + print(f"Text after frontend processing: {norm_text2}") + if not ref_free: + bert = torch.cat([bert1, bert2], 1) + all_phoneme_ids = torch.LongTensor(phones1 + phones2).to(device).unsqueeze(0) + else: + bert = bert2 + all_phoneme_ids = torch.LongTensor(phones2).to(device).unsqueeze(0) + + bert = bert.to(device).unsqueeze(0) + all_phoneme_len = torch.tensor([all_phoneme_ids.shape[-1]]).to(device) + prompt = None if ref_free else prompt_semantic.unsqueeze(0).to(device) + t3e = ttime() + t3 += t3e - t3s + with torch.no_grad(): + pred_semantic, idx = t2s_model.model.infer_panel( + all_phoneme_ids, + all_phoneme_len, + prompt, + bert, + top_k=top_k, + top_p=top_p, + temperature=temperature, + early_stop_num=hz * max_sec, + ) + t4e = ttime() + t4 += t4e - t3e + pred_semantic = pred_semantic[:, -idx:].unsqueeze(0) + refer = get_spepc(hps, ref_wav_path) if not ref_free else None + if refer is not None: + if is_half: + refer = refer.half().to(device) + else: + refer = refer.to(device) + audio = ( + vq_model.decode( + pred_semantic, torch.LongTensor(phones2).to(device).unsqueeze(0), refer + ) + .detach() + .cpu() + .numpy()[0, 0] + ) + max_audio = np.abs(audio).max() + if max_audio > 1: + audio /= max_audio + audio_opt.append(audio) + audio_opt.append(zero_wav) + t5e = ttime() + t5 += t5e - t4e + return ( + (t1 - t0, t2 - t1, t3, t4, t5), + hps.data.sampling_rate, + (np.concatenate(audio_opt, 0) * 32768).astype(np.int16) + ) + +def split(todo_text): + todo_text = todo_text.replace("……", "。").replace("——", "，") + if todo_text[-1] not in splits: + todo_text += "。" + i_split_head = i_split_tail = 0 + len_text = len(todo_text) + todo_texts = [] + while 1: + if i_split_head >= len_text: + break + if todo_text[i_split_head] in splits: + i_split_head += 1 + todo_texts.append(todo_text[i_split_tail:i_split_head]) + i_split_tail = i_split_head + else: + i_split_head += 1 + return todo_texts + +def cut1(inp): + inp = inp.strip("\n") + inps = split(inp) + split_idx = list(range(0, len(inps), 4)) + split_idx[-1] = None + if len(split_idx) > 1: + opts = [] + for idx in range(len(split_idx) - 1): + opts.append("".join(inps[split_idx[idx]: split_idx[idx + 1]])) + else: + opts = [inp] + return "\n".join(opts) + +def cut2(inp): + inp = inp.strip("\n") + inps = split(inp) + if len(inps) < 2: + return inp + opts = [] + summ = 0 + tmp_str = "" + for i in range(len(inps)): + summ += len(inps[i]) + tmp_str += inps[i] + if summ > 50: + summ = 0 + opts.append(tmp_str) + tmp_str = "" + if tmp_str != "": + opts.append(tmp_str) + if len(opts) > 1 and len(opts[-1]) < 50: + opts[-2] = opts[-2] + opts[-1] + opts = opts[:-1] + return "\n".join(opts) + +def cut3(inp): + inp = inp.strip("\n") + return "\n".join(["%s" % item for item in inp.strip("。").split("。")]) + +def cut4(inp): + inp = inp.strip("\n") + return "\n".join(["%s" % item for item in inp.strip(".").split(".")]) + +def cut5(inp): + inp = inp.strip("\n") + punds = r'[,.;?!、，。？！;：…]' + items = re.split(f'({punds})', inp) + mergeitems = ["".join(group) for group in zip(items[::2], items[1::2])] + if len(items) % 2 == 1: + mergeitems.append(items[-1]) + opt = "\n".join(mergeitems) + return opt + +def gptsovits_inference(gpt_model, sovits_model, ref_wav_path, prompt_text, prompt_language, text, text_language, how_to_cut="No slice", top_k=20, top_p=0.6, temperature=0.6, ref_free=False, output_path="output"): + global loaded_gpt_path, loaded_sovits_path + + if not ref_free and not os.path.exists(ref_wav_path): + print("You must input a reference audio path!") + return None + + gpt_path = gpt_model + sovits_path = sovits_model + + if loaded_gpt_path != gpt_path: + change_gpt_weights(gpt_path) + loaded_gpt_path = gpt_path + + if loaded_sovits_path != sovits_path: + change_sovits_weights(sovits_path) + loaded_sovits_path = sovits_path + + if not os.path.exists(output_path): + os.makedirs(output_path, exist_ok=True) + + output_count = 1 + while True: + opt_filename = f"GPTSoVITS_{output_count}.wav" + current_output_path = os.path.join(output_path, opt_filename) + if not os.path.exists(current_output_path): + break + output_count += 1 + + times, sr, audio_data = get_tts_wav(ref_wav_path, prompt_text, prompt_language, text, text_language, how_to_cut, top_k, top_p, temperature, ref_free) + + sf.write(current_output_path, audio_data, sr, format="wav") + print("Times:\nref_audio: {:.2f}s text: {:.2f}s phonemes & bert: {:.2f}s gpt: {:.2f}s sovits: {:.2f}s".format(*times)) + return current_output_path + +# Command-line interface def main(): - parser = argparse.ArgumentParser(description="Inference CLI for GPT-SoVITS") - parser.add_argument('--config', type=str, default='GPT_SoVITS/configs/tts_infer.yaml', help='Path to the configuration file') - parser.add_argument('--gpt_model', type=str, help='Path to the GPT model') - parser.add_argument('--sovits_model', type=str, help='Path to the SoVITS model') - parser.add_argument('--ref_audio', type=str, required=True, help='Path to the reference audio file') - parser.add_argument('--ref_text', type=str, required=True, help='Path to the reference text file') - parser.add_argument('--target_language', type=str, required=True, choices=["zh", "en", "ja", "zh_en", "ja_en", "multi"], help='Target language for synthesis') - parser.add_argument('--ref_language', type=str, required=True, choices=["zh", "en", "ja", "zh_en", "ja_en", "multi"], help='Reference language') - parser.add_argument('--target_text', type=str, required=True, help='Path to the target text file') - parser.add_argument('--how_to_cut', type=str, required=True, choices=["cut0", "cut1", "cut2", "cut3", "cut4", "cut5"], help='Text split method') - parser.add_argument('--output_path', type=str, required=True, help='Path to save the output') - parser.add_argument('--top_k', type=int, default=5, help='Top k sampling') - parser.add_argument('--top_p', type=float, default=1.0, help='Top p sampling') - parser.add_argument('--temperature', type=float, default=1.0, help='Temperature for sampling') - parser.add_argument('--batch_size', type=int, default=20, help='Batch size for inference') - parser.add_argument('--speed_factor', type=float, default=1.0, help='Speed factor for synthesis') - parser.add_argument('--split_bucket', action='store_true', help='Enable data bucketing') - parser.add_argument('--fragment_interval', type=float, default=0.3, help='Interval between audio fragments') - parser.add_argument('--seed', type=int, default=-1, help='Random seed') - parser.add_argument('--keep_random', action='store_true', help='Keep random seed') - parser.add_argument('--parallel_infer', action='store_true', help='Enable parallel inference') - parser.add_argument('--repetition_penalty', type=float, default=1.35, help='Repetition penalty for T2S model') - + parser = argparse.ArgumentParser(description="GPT-SoVITS TTS Inference CLI") + parser.add_argument("--use_pretrained_gpt", action="store_true", help="Use pretrained GPT model") + parser.add_argument("--gpt_model", type=str, help="Path to GPT model checkpoint") + parser.add_argument("--sovits_model", type=str, help="Path to SoVITS model checkpoint") + parser.add_argument("--ref_audio", type=str, help="Path to reference wav file") + parser.add_argument("--ref_text", type=str, help="Path to reference text file") + parser.add_argument("--target_language", type=str, choices=dict_language.keys(), help="Language of the target text") + parser.add_argument("--ref_language", type=str, choices=dict_language.keys(), help="Language of the reference text") + parser.add_argument("--target_text", type=str, help="Path to the target text file") + parser.add_argument("--how_to_cut", type=str, default="No slice", help="How to cut the text for synthesis") + parser.add_argument("--top_k", type=int, default=20, help="Top K sampling") + parser.add_argument("--top_p", type=float, default=0.6, help="Top P sampling") + parser.add_argument("--temperature", type=float, default=0.6, help="Sampling temperature") + parser.add_argument("--ref_free", action="store_true", help="Perform reference-free TTS") + parser.add_argument("--output_path", type=str, default="output", help="Path to save the output wav file") + args = parser.parse_args() - # Create TTS configuration - tts_config = create_tts_config(args.config) + if args.use_pretrained_gpt: + args.gpt_model = pretrained_gpt_path + args.sovits_model = pretrained_sovits_path - # Override model paths if provided - if args.gpt_model: - tts_config.t2s_weights_path = args.gpt_model - if args.sovits_model: - tts_config.vits_weights_path = args.sovits_model + # Check required arguments + if not args.ref_audio or not args.ref_text or not args.target_text or not args.target_language: + parser.error("--ref_audio, --ref_text, --target_text, and --target_language are required when --use_pretrained_gpt is enabled") - # Print the TTS configuration - print_tts_config(tts_config) + if not args.use_pretrained_gpt and (not args.gpt_model or not args.sovits_model): + parser.error("--gpt_model and --sovits_model are required when --use_pretrained_gpt is not enabled") - # Read input files - with open(args.ref_text, 'r', encoding='utf-8') as f: - prompt_text = f.read().strip() - with open(args.target_text, 'r', encoding='utf-8') as f: + prompt_text = None + if args.ref_text and os.path.exists(args.ref_text): + with open(args.ref_text, 'r') as f: + prompt_text = f.read().strip() + + with open(args.target_text, 'r') as f: target_text = f.read().strip() - # Call the inference function - inference( - tts_config, - text=target_text, - text_lang=args.target_language, - ref_audio_path=args.ref_audio, + output_path = gptsovits_inference( + gpt_model=args.gpt_model, + sovits_model=args.sovits_model, + ref_wav_path=args.ref_audio, prompt_text=prompt_text, - prompt_lang=args.ref_language, + prompt_language=args.ref_language, + text=target_text, + text_language=args.target_language, + how_to_cut=args.how_to_cut, top_k=args.top_k, top_p=args.top_p, temperature=args.temperature, - text_split_method=args.how_to_cut, - batch_size=args.batch_size, - speed_factor=args.speed_factor, - ref_text_free=False, - split_bucket=args.split_bucket, - fragment_interval=args.fragment_interval, - seed=args.seed, - keep_random=args.keep_random, - parallel_infer=args.parallel_infer, - repetition_penalty=args.repetition_penalty, + ref_free=args.ref_free, output_path=args.output_path ) + if output_path: + print(f"Generated audio saved at: {output_path}") if __name__ == "__main__": main() diff --git a/tts/GPT-SoVITS/tools/asr/fasterwhisper_asr.py b/tts/GPT-SoVITS/tools/asr/fasterwhisper_asr.py index e9fc6a4..669ac3a 100644 --- a/tts/GPT-SoVITS/tools/asr/fasterwhisper_asr.py +++ b/tts/GPT-SoVITS/tools/asr/fasterwhisper_asr.py @@ -78,7 +78,7 @@ def execute_asr(input_folder, output_folder, model_size, language, precision): text += segment.text output.append(f"{file_path}|{output_file_name}|{info.language.upper()}|{text}") except: - print(traceback.format_exc()) + return print(traceback.format_exc()) output_folder = output_folder or "output/asr_opt" os.makedirs(output_folder, exist_ok=True) diff --git a/tts/GPT-SoVITS/tools/cmd-denoise.py b/tts/GPT-SoVITS/tools/cmd-denoise.py index e3d7e98..b23eee2 100644 --- a/tts/GPT-SoVITS/tools/cmd-denoise.py +++ b/tts/GPT-SoVITS/tools/cmd-denoise.py @@ -1,5 +1,5 @@ import os,argparse -import traceback + from modelscope.pipelines import pipeline from modelscope.utils.constant import Tasks from tqdm import tqdm @@ -12,10 +12,7 @@ def execute_denoise(input_folder,output_folder): # print(input_folder) # print(list(os.listdir(input_folder).sort())) for name in tqdm(os.listdir(input_folder)): - try: - ans("%s/%s"%(input_folder,name),output_path='%s/%s'%(output_folder,name)) - except: - traceback.print_exc() + ans("%s/%s"%(input_folder,name),output_path='%s/%s'%(output_folder,name)) if __name__ == '__main__': parser = argparse.ArgumentParser() diff --git a/tts/GPT-SoVITS/tools/slice_per_utterance.py b/tts/GPT-SoVITS/tools/slice_per_utterance.py new file mode 100644 index 0000000..2343e78 --- /dev/null +++ b/tts/GPT-SoVITS/tools/slice_per_utterance.py @@ -0,0 +1,66 @@ +import os +import traceback +import torch + +from transformers import pipeline +import soundfile as sf + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +to_lang_name = { + "en": "english", + "ja": "japanese", + "zh":"chinese" +} + +def slice(input_audio, audio_output_folder, transcript_output_folder, model_size, language, precision): + audio_data, sr = sf.read(input_audio) + + output = [] + os.makedirs(audio_output_folder, exist_ok=True) + os.makedirs(transcript_output_folder, exist_ok=True) + output_base_name = os.path.splitext(os.path.basename(input_audio))[0] + + try: + # hf whisper does better for longer audios + print(f"Loading whisper {model_size} model...") + whisper_model = pipeline( + "automatic-speech-recognition", + model=f"openai/whisper-{model_size}", + chunk_length_s=30, + return_timestamps=True, + device=device, + ) + + print("Trancribing the audio...") + segments = whisper_model(input_audio, return_timestamps=True, generate_kwargs={"language": to_lang_name[language]}) + skipped_segments = 0 + + print("Slicing audio based on transcript...") + for i, segment in enumerate(segments["chunks"]): + output_audio_path = os.path.join(audio_output_folder, f"{output_base_name}_{i}.wav") + start_time = int(segment["timestamp"][0]*sr) + end_time = int(segment["timestamp"][1]*sr) + + if not 54 > (segment["timestamp"][1]-segment["timestamp"][0]) > 0.6: + print(f"Skipped segment {i} due to being under or over the required length") + skipped_segments += 1 + continue + + new_audio_data = audio_data[start_time:end_time] + sf.write( + output_audio_path, + new_audio_data, + sr, + format="wav" + ) + + output.append(f"{output_audio_path}|{output_base_name}|{language.upper()}|{segment['text']}") + print(f"{i}. [{segment['timestamp'][0]:.2f}s -> {segment['timestamp'][1]:.2f}s]{segment['text']}") + except: + return print(traceback.format_exc()) + + print(f"Total skipped segments: {skipped_segments}") + output_transcript_path = os.path.join(transcript_output_folder, f"{output_base_name}.list") + with open(output_transcript_path, "w", encoding="utf-8") as f: + f.write("\n".join(output)) + return audio_output_folder, output_transcript_path \ No newline at end of file diff --git a/tts/GPT-SoVITS/tools/uvr5/mdxnet.py b/tts/GPT-SoVITS/tools/uvr5/mdxnet.py index 372db25..0d609c4 100644 --- a/tts/GPT-SoVITS/tools/uvr5/mdxnet.py +++ b/tts/GPT-SoVITS/tools/uvr5/mdxnet.py @@ -220,7 +220,7 @@ class Predictor: opt_path_other = path_other[:-4] + ".%s" % format if os.path.exists(path_vocal): os.system( - "ffmpeg -i '%s' -vn '%s' -q:a 2 -y" % (path_vocal, opt_path_vocal) + "ffmpeg -i %s -vn %s -q:a 2 -y" % (path_vocal, opt_path_vocal) ) if os.path.exists(opt_path_vocal): try: @@ -229,7 +229,7 @@ class Predictor: pass if os.path.exists(path_other): os.system( - "ffmpeg -i '%s' -vn '%s' -q:a 2 -y" % (path_other, opt_path_other) + "ffmpeg -i %s -vn %s -q:a 2 -y" % (path_other, opt_path_other) ) if os.path.exists(opt_path_other): try: diff --git a/tts/GPT-SoVITS/webui.py b/tts/GPT-SoVITS/webui.py index 88e8eb5..8ac0df2 100644 --- a/tts/GPT-SoVITS/webui.py +++ b/tts/GPT-SoVITS/webui.py @@ -85,7 +85,7 @@ if if_gpu_ok and len(gpu_infos) > 0: else: gpu_info = ("%s\t%s" % ("0", "CPU")) gpu_infos.append("%s\t%s" % ("0", "CPU")) - default_batch_size = int(psutil.virtual_memory().total/ 1024 / 1024 / 1024 / 2) + default_batch_size = psutil.virtual_memory().total/ 1024 / 1024 / 1024 / 2 gpus = "-".join([i[0] for i in gpu_infos]) pretrained_sovits_name="GPT_SoVITS/pretrained_models/s2G488k.pth" @@ -418,10 +418,7 @@ def open1a(inp_text,inp_wav_dir,exp_name,gpu_numbers,bert_pretrained_dir): with open(path_text, "w", encoding="utf8") as f: f.write("\n".join(opt) + "\n") ps1a=[] - if len("".join(opt)) > 0: - yield "文本进程成功", {"__type__": "update", "visible": True}, {"__type__": "update", "visible": False} - else: - yield "文本进程失败", {"__type__": "update", "visible": True}, {"__type__": "update", "visible": False} + yield "文本进程结束",{"__type__":"update","visible":True},{"__type__":"update","visible":False} else: yield "已有正在进行的文本任务，需先终止才能开启下一次任务", {"__type__": "update", "visible": False}, {"__type__": "update", "visible": True} @@ -586,7 +583,7 @@ def open1abc(inp_text,inp_wav_dir,exp_name,gpu_numbers1a,gpu_numbers1Ba,gpu_numb os.remove(txt_path) with open(path_text, "w",encoding="utf8") as f: f.write("\n".join(opt) + "\n") - assert len("".join(opt)) > 0, "1Aa-文本获取进程失败" + yield "进度：1a-done", {"__type__": "update", "visible": False}, {"__type__": "update", "visible": True} ps1abc=[] #############################1b