From a108e4e410642616a049534c3e963e6327013d8c Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 12 Jan 2023 04:19:11 -0500 Subject: [PATCH] End all sentences with punctuation. also Include some forgotten files --- tts/text/pipeline/english_pipeline_v1.py | 7 + tts/text/pipeline/espeak_pipeline.py | 6 + tts/text/pipeline/legacy_fakeyou_pipeline.py | 6 + .../strip_and_add_ending_period.py | 22 +++ tts/utils.py | 6 + voice_conversion/softvc/old_cli_inference.py | 145 ++++++++++++++++++ voice_conversion/softvc/utils.py | 6 + 7 files changed, 198 insertions(+) create mode 100644 tts/text/pipeline_components/normalization/strip_and_add_ending_period.py create mode 100644 tts/utils.py create mode 100755 voice_conversion/softvc/old_cli_inference.py create mode 100644 voice_conversion/softvc/utils.py diff --git a/tts/text/pipeline/english_pipeline_v1.py b/tts/text/pipeline/english_pipeline_v1.py index 9f6ea90..756d157 100644 --- a/tts/text/pipeline/english_pipeline_v1.py +++ b/tts/text/pipeline/english_pipeline_v1.py @@ -2,6 +2,7 @@ from typing import List from text.pipeline.base_pipeline import BasePipeline from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence from text.pipeline_components.normalization.english_normalization import normalize_english +from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period from text.pipeline_components.phonetization.base_phonetization import BasePhonetization from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization @@ -13,6 +14,8 @@ class EnglishPipelineV1(BasePipeline): It introduces Arpabet and individually upgradable components. The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that the model input expectations on both sides match one another. + + NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes". """ def __init__(self, phonetization: BasePhonetization): @@ -21,6 +24,10 @@ class EnglishPipelineV1(BasePipeline): def user_input_to_sequence(self, input_text: str) -> List[int]: """Convert raw user input into a numerical sequence for ML inference.""" + # Force addition of an ending period to prevent "strokes" + input_text = strip_and_add_ending_period(input_text) + + parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text) self._normalize_graphemes(parsed_sequence) diff --git a/tts/text/pipeline/espeak_pipeline.py b/tts/text/pipeline/espeak_pipeline.py index a0d71c7..91ee403 100644 --- a/tts/text/pipeline/espeak_pipeline.py +++ b/tts/text/pipeline/espeak_pipeline.py @@ -2,6 +2,7 @@ from typing import List from text.pipeline.base_pipeline import BasePipeline from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence from text.pipeline_components.normalization.espeak_normalization import normalize +from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period from text.pipeline_components.phonetization.espeak_phonemization import EspeakPhonetization from text.pipeline_components.sequencing.espeak_symbols import Symbols @@ -11,6 +12,8 @@ class EspeakPipeline(BasePipeline): It introduces Arpabet and individually upgradable components. The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that the model input expectations on both sides match one another. + + NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes". """ def __init__(self, lang='en'): @@ -26,6 +29,9 @@ class EspeakPipeline(BasePipeline): def user_input_to_sequence(self, input_text: str) -> List[int]: """Convert raw user input into a numerical sequence for ML inference.""" + # Force addition of an ending period to prevent "strokes" + input_text = strip_and_add_ending_period(input_text) + parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text) self._normalize_graphemes(parsed_sequence) diff --git a/tts/text/pipeline/legacy_fakeyou_pipeline.py b/tts/text/pipeline/legacy_fakeyou_pipeline.py index a99e4db..7a55ad7 100644 --- a/tts/text/pipeline/legacy_fakeyou_pipeline.py +++ b/tts/text/pipeline/legacy_fakeyou_pipeline.py @@ -3,6 +3,7 @@ from typing import List from text.pipeline.base_pipeline import BasePipeline from text.pipeline_components.normalization.common_cleaners import collapse_whitespace, convert_to_ascii, lowercase from text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers +from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period from text.pipeline_components.sequencing.english_symbols_v1 import arpabet_to_sequence, symbols_to_sequence # Regular expression matching text enclosed in curly braces: @@ -15,6 +16,8 @@ class LegacyFakeYouPipeline(BasePipeline): leverage CMUDict to look up arpabet even when models were known to be trained with arpabet support. (It did support curly braces for manual arpabet inclusion.) + + NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes". """ def user_input_to_sequence(self, input_text: str) -> List[int]: @@ -23,6 +26,9 @@ class LegacyFakeYouPipeline(BasePipeline): return self._text_to_sequence(pre_sequence, ['english_cleaners']) def _preprocess_text(self, input_text: str, is_raw_input: bool = True) -> str: + # Force addition of an ending period to prevent "strokes" + input_text = strip_and_add_ending_period(input_text) + pre_sequence = "" for line in input_text.split("\n"): if len(line) < 1: diff --git a/tts/text/pipeline_components/normalization/strip_and_add_ending_period.py b/tts/text/pipeline_components/normalization/strip_and_add_ending_period.py new file mode 100644 index 0000000..524e822 --- /dev/null +++ b/tts/text/pipeline_components/normalization/strip_and_add_ending_period.py @@ -0,0 +1,22 @@ + +# If a sentence doesn't end in expected ending punctuation, we'll want to auto-punctuate. +EXPECTED_ENDING_PUNCTUATIONS = [ + '!', + '.', + '?', + '…', + '\'', + '\"', +] + +PERIOD = '.' + +# Prevent model from having a "stroke" by adding missing ending punctuation. +def strip_and_add_ending_period(input_text: str) -> str: + # TODO: This isn't great for all languages. + input_text = input_text.strip() + ends_with_period = any([input_text.endswith(e) for e in EXPECTED_ENDING_PUNCTUATIONS]) + if not ends_with_period: + input_text += PERIOD + return input_text + \ No newline at end of file diff --git a/tts/utils.py b/tts/utils.py new file mode 100644 index 0000000..435f923 --- /dev/null +++ b/tts/utils.py @@ -0,0 +1,6 @@ + +# TODO: De-duplicate this +def print_title(title, length=40): + print(f'\n{"/"*length}') + print(title.center(length, ' ')) + print(f'{"/"*length}\n', flush=True) diff --git a/voice_conversion/softvc/old_cli_inference.py b/voice_conversion/softvc/old_cli_inference.py new file mode 100755 index 0000000..2147ebd --- /dev/null +++ b/voice_conversion/softvc/old_cli_inference.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python + +import IPython.display as display +import torch, torchaudio +import torch.nn as nn +import sys +import soundfile as sf + +from time import time +import torch.jit +sys.path.insert(0, './hubert/hubert') +from model import HubertSoft + +sys.path.insert(0, './acoustic-model/') +from acoustic.model import AcousticModel + +# =============================== + + +#@title select input wav +#file_name = "/content/drive/MyDrive/VCModels/Source/brandon1.wav"#@param { type: "string"} +file_name = "samples/source/easy/brandon1.wav" + +source, sr = torchaudio.load(f"{file_name}") +source = torchaudio.functional.resample(source, sr, 16000) +source = source.unsqueeze(0).cuda() +print(f"Wav Shape and Type {source.shape} {source.dtype}") +if source.shape[1] == 2: # remove channel + source = torch.mean(source, dim=0).unsqueeze(0) + print(f"Removing Channel to mono Wav Shape and Type {source.shape} {source.dtype}") + +display.Audio(source.squeeze().cpu(), rate=16000) + + +# =============================== + +import datetime +from utils_acoustic import AcousticModelScripted +from utils_hubert import modify_model + +hubert_soft = torch.hub.load("bshall/hubert:main", "hubert_soft").cuda() +hubert_soft.eval() + +script_acoustic = True #@param {type: "boolean"} +print(script_acoustic) + +# Load the acoustic model (either hubert_soft or hubert_discrete) +#input_acoustic_model_file = "/content/drive/MyDrive/VCModels/Padme/model-24000.pt" #@param {type: "string"} +input_acoustic_model_file = "/home/bt/models/voice-conversion/trump-acoustic.pt" +acoustic_dict = torch.load(input_acoustic_model_file) # this used ddp ## REPLACE THIS! <-- ! +if script_acoustic == True: + acoustic = AcousticModelScripted().cuda() +else: + acoustic = AcousticModel().cuda() + +acoustic.eval() + +new_state_dict = modify_model(state_dict=acoustic_dict) +acoustic.load_state_dict(new_state_dict) + +import importlib.util +import sys + +spec = importlib.util.spec_from_file_location("hifigan.utils", "/content/hifigan/hifigan/utils.py") +u = importlib.util.module_from_spec(spec) +sys.modules["hifigan.utils"] = u +spec.loader.exec_module(u) + +spec = importlib.util.spec_from_file_location("hifigan.generator", "/content/hifigan/hifigan/generator.py") +g = importlib.util.module_from_spec(spec) +sys.modules["hifigan.generator"] = g +spec.loader.exec_module(g) + +new_state_dict = modify_model(state_dict=acoustic_dict) +acoustic.load_state_dict(new_state_dict) +input_hifigan_checkpoint_path = "/content/drive/MyDrive/VCModels/Hifigan/Padme.pt" #@param {type: "string"} +if input_hifigan_checkpoint_path: + checkpoint = torch.load(input_hifigan_checkpoint_path)["generator"]["model"] + hifigan = g.HifiganGenerator() + consume_prefix_in_state_dict_if_present(checkpoint, "module.") + hifigan.load_state_dict(checkpoint) + hifigan.remove_weight_norm() +else: + hifigan = torch.hub.load("bshall/hifigan:main", "hifigan_hubert_soft") + +hifigan.cuda() +hifigan.eval() + +#utils.remove_weight_norm(hubert_soft.positional_embedding.conv) +# Convert to the target speaker +units = None +mel = None + +unit_start_time = 0.0 +unit_end_time = 0.0 + +acoustic_start_time = 0.0 +acoustic_end_time = 0.0 + +hifi_start_time = 0.0 +hifi_end_time = 0.0 + +with torch.inference_mode(): + unit_start_time = datetime.datetime.now() + units = hubert_soft.units(source) + print(f"Units and Type {units.shape} {units.dtype}") + unit_end_time = datetime.datetime.now() + + hubert_soft_script = torch.jit.trace_module(hubert_soft,{"units":source}) + torch.jit.save(hubert_soft_script, 'hubert_soft.jit') + + print(f"Unit Generation Time {(unit_end_time - unit_start_time).total_seconds() * 1000}") + + print("Hubert Soft Exported") + + acoustic_start_time = datetime.datetime.now() + mel = acoustic.generate(units).transpose(1, 2) + print(f"Mel and Type {mel.shape} {mel.dtype}") + acoustic_end_time = datetime.datetime.now() + print(f"Acoustic Time {(acoustic_end_time - acoustic_start_time).total_seconds() * 1000}") + if script_acoustic == True: + output_acoustic_scripted = '/content/drive/MyDrive/VCModels/FinishedModels/Padme.recast' #@param {type: "string"} + print("Exporting Scripted") + acoustic_model_script = torch.jit.script(acoustic) + torch.jit.save(acoustic_model_script, output_acoustic_scripted) + else: + print("Exporting Traced") + acoustic_model_script = torch.jit.trace_module(acoustic,{"generate":units}) + output_acoustic_traced = '' #@param {type: "string"} + torch.jit.save(acoustic_model_script, output_acoustic_traced) + + print("Acoustic Model Exported") + hifi_start_time = datetime.datetime.now() + + target = hifigan(mel) + print(f"Wavform and Type {target.shape} {target.dtype}") + hifi_end_time = datetime.datetime.now() + print(f"MEL {target.shape}") + print(f"Hifi Gan Time {(hifi_end_time - hifi_start_time).total_seconds() * 1000}") + + hifi_gan_script = torch.jit.trace_module(hifigan,{"forward":mel}) + output_hifigan_jit = '/content/drive/MyDrive/VCModels/FinishedModels/Padme-hifigan-jit.pt' #@param {type: "string"} + torch.jit.save(hifi_gan_script, output_hifigan_jit) + print("Hifi Gan Model Exported") + diff --git a/voice_conversion/softvc/utils.py b/voice_conversion/softvc/utils.py new file mode 100644 index 0000000..435f923 --- /dev/null +++ b/voice_conversion/softvc/utils.py @@ -0,0 +1,6 @@ + +# TODO: De-duplicate this +def print_title(title, length=40): + print(f'\n{"/"*length}') + print(title.center(length, ' ')) + print(f'{"/"*length}\n', flush=True)