Add english cleaners, begin work on legacy pipeline inclusion

tts/text/pipeline/english_pipeline_v1.py

@@ -10,6 +10,12 @@ _ENGLISH_DICTIONARY = CMUDict(DEFAULT_CMUDICT_PATH)
 _ENGLISH_PHONETIZER = EnglishPhonetization(_ENGLISH_DICTIONARY)
 
 class EnglishPipelineV1(BasePipeline):
+    """
+    This is a new text pipeline for FakeYou that will be introduced in July 2022.
+    It introduces Arpabet and individually upgradable components.
+    The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that 
+    the model input expectations on both sides match one another.
+    """
 
     def user_input_to_sequence(self, input_text: str) -> List[int]:
         """Convert raw user input into a numerical sequence for ML inference."""

tts/text/pipeline/legacy_fakeyou_pipeline.py

@@ -0,0 +1,28 @@
+from typing import List
+from tts.text.pipeline.base_pipeline import BasePipeline
+
+class LegacyFakeYouPipeline(BasePipeline):
+    """
+    This pipeline was in use from June 2021 to July 2022.
+    In contrast to legacy "vo.codes" (2020 - 2021), it does not 
+    leverage Arpabet even when models were trained with Arpabet support.
+    """
+
+
+    def user_input_to_sequence(self, input_text: str) -> List[int]:
+        """Convert raw user input into a numerical sequence for ML inference."""
+        pre_sequence = self._preprocess_text(input_text)
+        return text_to_sequence(pre_sequence, ['english_cleaners'])
+
+    def _preprocess_text(self, input_text: str, is_raw_input: bool = True) -> str:
+        pre_sequence = ""
+        for line in input_text.split("\n"):
+            if len(line) < 1:
+                continue
+            if is_raw_input:
+                if line[-1] != ";":
+                    line = line + ";"
+            else:
+                pass
+            pre_sequence += line
+        return pre_sequence

tts/text/pipeline_components/normalization/english_cleaners.py

@@ -0,0 +1,102 @@
+""" from https://github.com/keithito/tacotron """
+
+import inflect
+import re
+
+# List of (regular expression, replacement) pairs for abbreviations:
+_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
+  ('mrs', 'misess'),
+  ('mr', 'mister'),
+  ('dr', 'doctor'),
+  ('st', 'saint'),
+  ('co', 'company'),
+  ('jr', 'junior'),
+  ('maj', 'major'),
+  ('gen', 'general'),
+  ('drs', 'doctors'),
+  ('rev', 'reverend'),
+  ('lt', 'lieutenant'),
+  ('hon', 'honorable'),
+  ('sgt', 'sergeant'),
+  ('capt', 'captain'),
+  ('esq', 'esquire'),
+  ('ltd', 'limited'),
+  ('col', 'colonel'),
+  ('ft', 'fort'),
+]]
+
+
+def expand_abbreviations(text):
+  for regex, replacement in _abbreviations:
+    text = re.sub(regex, replacement, text)
+  return text
+
+
+_inflect = inflect.engine()
+_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
+_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
+_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
+_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
+_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
+_number_re = re.compile(r'[0-9]+')
+
+
+def _remove_commas(m):
+  return m.group(1).replace(',', '')
+
+
+def _expand_decimal_point(m):
+  return m.group(1).replace('.', ' point ')
+
+
+def _expand_dollars(m):
+  match = m.group(1)
+  parts = match.split('.')
+  if len(parts) > 2:
+    return match + ' dollars'  # Unexpected format
+  dollars = int(parts[0]) if parts[0] else 0
+  cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+  if dollars and cents:
+    dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+    cent_unit = 'cent' if cents == 1 else 'cents'
+    return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
+  elif dollars:
+    dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+    return '%s %s' % (dollars, dollar_unit)
+  elif cents:
+    cent_unit = 'cent' if cents == 1 else 'cents'
+    return '%s %s' % (cents, cent_unit)
+  else:
+    return 'zero dollars'
+
+
+def _expand_ordinal(m):
+  return _inflect.number_to_words(m.group(0))
+
+
+def _expand_number(m):
+  num = int(m.group(0))
+  if num > 1000 and num < 3000:
+    if num == 2000:
+      return 'two thousand'
+    elif num > 2000 and num < 2010:
+      return 'two thousand ' + _inflect.number_to_words(num % 100)
+    elif num % 100 == 0:
+      return _inflect.number_to_words(num // 100) + ' hundred'
+    else:
+      return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
+  else:
+    return _inflect.number_to_words(num, andword='')
+
+
+def normalize_numbers(text):
+  text = re.sub(_comma_number_re, _remove_commas, text)
+  text = re.sub(_pounds_re, r'\1 pounds', text)
+  text = re.sub(_dollars_re, _expand_dollars, text)
+  text = re.sub(_decimal_number_re, _expand_decimal_point, text)
+  text = re.sub(_ordinal_re, _expand_ordinal, text)
+  text = re.sub(_number_re, _expand_number, text)
+  return text
+
+def expand_numbers(text):
+  return normalize_numbers(text)

tts/text/pipeline_components/normalization/english_normalization.py

@@ -1,14 +1,13 @@
 from tts.text.pipeline_components.normalization.common_cleaners import collapse_whitespace
 from tts.text.pipeline_components.normalization.common_cleaners import convert_to_ascii
 from tts.text.pipeline_components.normalization.common_cleaners import lowercase
+from tts.text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers
 
 def normalize_english(raw_input_text: str) -> str:
     text = convert_to_ascii(raw_input_text)
     text = lowercase(text)
-
-    # TODO: Expand abbreviations
-    # TODO: Expand numbers
-
+    text = expand_numbers(text)
+    text = expand_abbreviations(text)
     text = collapse_whitespace(text)
 
     return text

tts/text/pipeline_components/normalization/test_english_normalization.py

@@ -9,3 +9,11 @@ class TestEnglishNormalization:
     def test_lowercase(self):
         output = normalize_english("This is a test of Normalization.")
         assert output == "this is a test of normalization."
+
+    def test_numbers(self):
+        output = normalize_english("1,000")
+        assert output == "one thousand"
+
+    def test_abbreviations(self):
+        output = normalize_english("mr. ed")
+        assert output == "mister ed"