End all sentences with punctuation. also Include some forgotten files

tts/text/pipeline/english_pipeline_v1.py

@@ -2,6 +2,7 @@ from typing import List
 from text.pipeline.base_pipeline import BasePipeline
 from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
 from text.pipeline_components.normalization.english_normalization import normalize_english
+from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period
 from text.pipeline_components.phonetization.base_phonetization import BasePhonetization
 from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
 from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
@@ -13,6 +14,8 @@ class EnglishPipelineV1(BasePipeline):
     It introduces Arpabet and individually upgradable components.
     The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that 
     the model input expectations on both sides match one another.
+
+    NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes".
     """
 
     def __init__(self, phonetization: BasePhonetization):
@@ -21,6 +24,10 @@ class EnglishPipelineV1(BasePipeline):
     def user_input_to_sequence(self, input_text: str) -> List[int]:
         """Convert raw user input into a numerical sequence for ML inference."""
 
+        # Force addition of an ending period to prevent "strokes"
+        input_text = strip_and_add_ending_period(input_text)
+
+
         parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text)
         self._normalize_graphemes(parsed_sequence)
 

tts/text/pipeline/espeak_pipeline.py

@@ -2,6 +2,7 @@ from typing import List
 from text.pipeline.base_pipeline import BasePipeline
 from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
 from text.pipeline_components.normalization.espeak_normalization import normalize
+from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period
 from text.pipeline_components.phonetization.espeak_phonemization import EspeakPhonetization
 from text.pipeline_components.sequencing.espeak_symbols import Symbols
 
@@ -11,6 +12,8 @@ class EspeakPipeline(BasePipeline):
     It introduces Arpabet and individually upgradable components.
     The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that 
     the model input expectations on both sides match one another.
+
+    NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes".
     """
 
     def __init__(self, lang='en'):
@@ -26,6 +29,9 @@ class EspeakPipeline(BasePipeline):
     def user_input_to_sequence(self, input_text: str) -> List[int]:
         """Convert raw user input into a numerical sequence for ML inference."""
 
+        # Force addition of an ending period to prevent "strokes"
+        input_text = strip_and_add_ending_period(input_text)
+
         parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text)
         self._normalize_graphemes(parsed_sequence)
 

tts/text/pipeline/legacy_fakeyou_pipeline.py

@@ -3,6 +3,7 @@ from typing import List
 from text.pipeline.base_pipeline import BasePipeline
 from text.pipeline_components.normalization.common_cleaners import collapse_whitespace, convert_to_ascii, lowercase
 from text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers
+from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period
 from text.pipeline_components.sequencing.english_symbols_v1 import arpabet_to_sequence, symbols_to_sequence
 
 # Regular expression matching text enclosed in curly braces:
@@ -15,6 +16,8 @@ class LegacyFakeYouPipeline(BasePipeline):
     leverage CMUDict to look up arpabet even when models were known to
     be trained with arpabet support. (It did support curly braces for 
     manual arpabet inclusion.)
+
+    NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes".
     """
 
     def user_input_to_sequence(self, input_text: str) -> List[int]:
@@ -23,6 +26,9 @@ class LegacyFakeYouPipeline(BasePipeline):
         return self._text_to_sequence(pre_sequence, ['english_cleaners'])
 
     def _preprocess_text(self, input_text: str, is_raw_input: bool = True) -> str:
+        # Force addition of an ending period to prevent "strokes"
+        input_text = strip_and_add_ending_period(input_text)
+
         pre_sequence = ""
         for line in input_text.split("\n"):
             if len(line) < 1:

tts/text/pipeline_components/normalization/strip_and_add_ending_period.py

@@ -0,0 +1,22 @@
+
+# If a sentence doesn't end in expected ending punctuation, we'll want to auto-punctuate.
+EXPECTED_ENDING_PUNCTUATIONS = [
+    '!',
+    '.', 
+    '?',
+    '…',
+    '\'',
+    '\"',
+]
+
+PERIOD = '.'
+
+# Prevent model from having a "stroke" by adding missing ending punctuation.
+def strip_and_add_ending_period(input_text: str) -> str:
+    # TODO: This isn't  great for all languages.
+    input_text = input_text.strip()
+    ends_with_period = any([input_text.endswith(e) for e in EXPECTED_ENDING_PUNCTUATIONS])
+    if not ends_with_period: 
+        input_text += PERIOD
+    return input_text
+    

tts/utils.py

@@ -0,0 +1,6 @@
+
+# TODO: De-duplicate this
+def print_title(title, length=40):
+    print(f'\n{"/"*length}')
+    print(title.center(length, ' '))
+    print(f'{"/"*length}\n', flush=True)

voice_conversion/softvc/old_cli_inference.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+
+import IPython.display as display
+import torch, torchaudio
+import torch.nn as nn
+import sys  
+import soundfile as sf
+
+from time import time
+import torch.jit
+sys.path.insert(0, './hubert/hubert')
+from model import HubertSoft
+
+sys.path.insert(0, './acoustic-model/')
+from acoustic.model import AcousticModel
+
+# ===============================
+
+
+#@title select input wav
+#file_name = "/content/drive/MyDrive/VCModels/Source/brandon1.wav"#@param { type: "string"}
+file_name = "samples/source/easy/brandon1.wav"
+
+source, sr = torchaudio.load(f"{file_name}")
+source = torchaudio.functional.resample(source, sr, 16000)
+source = source.unsqueeze(0).cuda()
+print(f"Wav Shape and Type {source.shape} {source.dtype}")
+if source.shape[1] == 2: # remove channel
+    source = torch.mean(source, dim=0).unsqueeze(0)
+    print(f"Removing Channel to mono Wav Shape and Type {source.shape} {source.dtype}")
+
+display.Audio(source.squeeze().cpu(), rate=16000)
+
+
+# ===============================
+
+import datetime
+from utils_acoustic import AcousticModelScripted
+from utils_hubert import modify_model
+
+hubert_soft = torch.hub.load("bshall/hubert:main", "hubert_soft").cuda()
+hubert_soft.eval()
+
+script_acoustic = True #@param {type: "boolean"}
+print(script_acoustic)
+
+# Load the acoustic model (either hubert_soft or hubert_discrete) 
+#input_acoustic_model_file = "/content/drive/MyDrive/VCModels/Padme/model-24000.pt" #@param {type: "string"}
+input_acoustic_model_file = "/home/bt/models/voice-conversion/trump-acoustic.pt"
+acoustic_dict = torch.load(input_acoustic_model_file) # this used ddp ## REPLACE THIS! <-- !
+if script_acoustic == True:
+    acoustic = AcousticModelScripted().cuda()
+else:
+    acoustic = AcousticModel().cuda()
+
+acoustic.eval()
+
+new_state_dict = modify_model(state_dict=acoustic_dict)
+acoustic.load_state_dict(new_state_dict)
+
+import importlib.util
+import sys
+
+spec = importlib.util.spec_from_file_location("hifigan.utils", "/content/hifigan/hifigan/utils.py")
+u = importlib.util.module_from_spec(spec)
+sys.modules["hifigan.utils"] = u
+spec.loader.exec_module(u)
+
+spec = importlib.util.spec_from_file_location("hifigan.generator", "/content/hifigan/hifigan/generator.py")
+g = importlib.util.module_from_spec(spec)
+sys.modules["hifigan.generator"] = g
+spec.loader.exec_module(g)
+
+new_state_dict = modify_model(state_dict=acoustic_dict)
+acoustic.load_state_dict(new_state_dict)
+input_hifigan_checkpoint_path = "/content/drive/MyDrive/VCModels/Hifigan/Padme.pt" #@param {type: "string"}
+if input_hifigan_checkpoint_path:
+  checkpoint = torch.load(input_hifigan_checkpoint_path)["generator"]["model"]
+  hifigan = g.HifiganGenerator()
+  consume_prefix_in_state_dict_if_present(checkpoint, "module.")
+  hifigan.load_state_dict(checkpoint)
+  hifigan.remove_weight_norm()
+else:
+  hifigan = torch.hub.load("bshall/hifigan:main", "hifigan_hubert_soft")
+
+hifigan.cuda()
+hifigan.eval()
+
+#utils.remove_weight_norm(hubert_soft.positional_embedding.conv)
+# Convert to the target speaker
+units = None
+mel = None 
+
+unit_start_time = 0.0
+unit_end_time = 0.0
+
+acoustic_start_time = 0.0
+acoustic_end_time = 0.0 
+
+hifi_start_time = 0.0
+hifi_end_time = 0.0
+
+with torch.inference_mode():
+    unit_start_time = datetime.datetime.now()
+    units = hubert_soft.units(source)
+    print(f"Units and Type {units.shape} {units.dtype}")
+    unit_end_time = datetime.datetime.now()
+
+    hubert_soft_script = torch.jit.trace_module(hubert_soft,{"units":source})
+    torch.jit.save(hubert_soft_script, 'hubert_soft.jit')
+
+    print(f"Unit Generation Time {(unit_end_time - unit_start_time).total_seconds() * 1000}")
+
+    print("Hubert Soft Exported")
+
+    acoustic_start_time = datetime.datetime.now()
+    mel = acoustic.generate(units).transpose(1, 2)
+    print(f"Mel and Type {mel.shape} {mel.dtype}")
+    acoustic_end_time = datetime.datetime.now()
+    print(f"Acoustic Time {(acoustic_end_time - acoustic_start_time).total_seconds() * 1000}")
+    if script_acoustic == True:
+        output_acoustic_scripted = '/content/drive/MyDrive/VCModels/FinishedModels/Padme.recast' #@param {type: "string"}
+        print("Exporting Scripted")
+        acoustic_model_script = torch.jit.script(acoustic)
+        torch.jit.save(acoustic_model_script, output_acoustic_scripted)
+    else:
+        print("Exporting Traced")
+        acoustic_model_script = torch.jit.trace_module(acoustic,{"generate":units})
+        output_acoustic_traced = '' #@param {type: "string"}
+        torch.jit.save(acoustic_model_script, output_acoustic_traced)
+
+    print("Acoustic Model Exported")
+    hifi_start_time = datetime.datetime.now()
+
+    target = hifigan(mel)
+    print(f"Wavform and Type {target.shape} {target.dtype}")
+    hifi_end_time = datetime.datetime.now()
+    print(f"MEL {target.shape}")
+    print(f"Hifi Gan Time {(hifi_end_time - hifi_start_time).total_seconds() * 1000}")
+
+    hifi_gan_script = torch.jit.trace_module(hifigan,{"forward":mel})
+    output_hifigan_jit = '/content/drive/MyDrive/VCModels/FinishedModels/Padme-hifigan-jit.pt' #@param {type: "string"}
+    torch.jit.save(hifi_gan_script, output_hifigan_jit)
+    print("Hifi Gan Model Exported")
+

voice_conversion/softvc/utils.py

@@ -0,0 +1,6 @@
+
+# TODO: De-duplicate this
+def print_title(title, length=40):
+    print(f'\n{"/"*length}')
+    print(title.center(length, ' '))
+    print(f'{"/"*length}\n', flush=True)