End all sentences with punctuation. also Include some forgotten files

tts/text/pipeline/english_pipeline_v1.py

@@ -2,6 +2,7 @@ from typing import List
 from text.pipeline.base_pipeline import BasePipeline
 from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
 from text.pipeline_components.normalization.english_normalization import normalize_english
+from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period
 from text.pipeline_components.phonetization.base_phonetization import BasePhonetization
 from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
 from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
@@ -13,6 +14,8 @@ class EnglishPipelineV1(BasePipeline):
     It introduces Arpabet and individually upgradable components.
     The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that 
     the model input expectations on both sides match one another.
+
+    NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes".
     """
 
     def __init__(self, phonetization: BasePhonetization):
@@ -21,6 +24,10 @@ class EnglishPipelineV1(BasePipeline):
     def user_input_to_sequence(self, input_text: str) -> List[int]:
         """Convert raw user input into a numerical sequence for ML inference."""
 
+        # Force addition of an ending period to prevent "strokes"
+        input_text = strip_and_add_ending_period(input_text)
+
+
         parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text)
         self._normalize_graphemes(parsed_sequence)
 

tts/text/pipeline/espeak_pipeline.py

@@ -2,6 +2,7 @@ from typing import List
 from text.pipeline.base_pipeline import BasePipeline
 from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
 from text.pipeline_components.normalization.espeak_normalization import normalize
+from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period
 from text.pipeline_components.phonetization.espeak_phonemization import EspeakPhonetization
 from text.pipeline_components.sequencing.espeak_symbols import Symbols
 
@@ -11,6 +12,8 @@ class EspeakPipeline(BasePipeline):
     It introduces Arpabet and individually upgradable components.
     The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that 
     the model input expectations on both sides match one another.
+
+    NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes".
     """
 
     def __init__(self, lang='en'):
@@ -26,6 +29,9 @@ class EspeakPipeline(BasePipeline):
     def user_input_to_sequence(self, input_text: str) -> List[int]:
         """Convert raw user input into a numerical sequence for ML inference."""
 
+        # Force addition of an ending period to prevent "strokes"
+        input_text = strip_and_add_ending_period(input_text)
+
         parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text)
         self._normalize_graphemes(parsed_sequence)
 

tts/text/pipeline/legacy_fakeyou_pipeline.py

@@ -3,6 +3,7 @@ from typing import List
 from text.pipeline.base_pipeline import BasePipeline
 from text.pipeline_components.normalization.common_cleaners import collapse_whitespace, convert_to_ascii, lowercase
 from text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers
+from text.pipeline_components.normalization.strip_and_add_ending_period import strip_and_add_ending_period
 from text.pipeline_components.sequencing.english_symbols_v1 import arpabet_to_sequence, symbols_to_sequence
 
 # Regular expression matching text enclosed in curly braces:
@@ -15,6 +16,8 @@ class LegacyFakeYouPipeline(BasePipeline):
     leverage CMUDict to look up arpabet even when models were known to
     be trained with arpabet support. (It did support curly braces for 
     manual arpabet inclusion.)
+
+    NB: Updated 2023-01-12 to force addition of ending periods to prevent "strokes".
     """
 
     def user_input_to_sequence(self, input_text: str) -> List[int]:
@@ -23,6 +26,9 @@ class LegacyFakeYouPipeline(BasePipeline):
         return self._text_to_sequence(pre_sequence, ['english_cleaners'])
 
     def _preprocess_text(self, input_text: str, is_raw_input: bool = True) -> str:
+        # Force addition of an ending period to prevent "strokes"
+        input_text = strip_and_add_ending_period(input_text)
+
         pre_sequence = ""
         for line in input_text.split("\n"):
             if len(line) < 1:

tts/text/pipeline_components/normalization/strip_and_add_ending_period.py

@@ -0,0 +1,22 @@
+
+# If a sentence doesn't end in expected ending punctuation, we'll want to auto-punctuate.
+EXPECTED_ENDING_PUNCTUATIONS = [
+    '!',
+    '.', 
+    '?',
+    '…',
+    '\'',
+    '\"',
+]
+
+PERIOD = '.'
+
+# Prevent model from having a "stroke" by adding missing ending punctuation.
+def strip_and_add_ending_period(input_text: str) -> str:
+    # TODO: This isn't  great for all languages.
+    input_text = input_text.strip()
+    ends_with_period = any([input_text.endswith(e) for e in EXPECTED_ENDING_PUNCTUATIONS])
+    if not ends_with_period: 
+        input_text += PERIOD
+    return input_text
+