Entirely reworked text handling pipeline

This breaks the text pipeline into the following pieces:

- input parsing (split known bracketed phoneme sections out first)

- normalization (fix spacing, numbers, abbreviations, etc.)

- phonetization (do CMUdict lookup or g2p prediction)

- sequencing (convert the symbolic representation to integer
  representation)

Each "pipeline" can have a unique combination of operations for each of
these pipeline steps.

This will grow over time.

tts/text/pipeline_components/normalization/common_cleaners.py

@@ -0,0 +1,32 @@
+"""
+Copied or adapted from Keith Ito's Tacotron2 code.
+https://github.com/keithito/tacotron
+"""
+
+import re
+from unidecode import unidecode
+
+# Regular expression matching whitespace:
+_whitespace_re = re.compile(r'\s+')
+
+def lowercase(text: str) -> str:
+  return text.lower()
+
+def collapse_whitespace(text) -> str:
+  return re.sub(_whitespace_re, ' ', text)
+
+def convert_to_ascii(text: str) -> str:
+  return unidecode(text)
+
+def basic_cleaners(text: str) -> str:
+  '''Basic pipeline that lowercases and collapses whitespace without transliteration.'''
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text
+
+def transliteration_cleaners(text: str) -> str:
+  '''Pipeline for non-English text that transliterates to ASCII.'''
+  text = convert_to_ascii(text)
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text

tts/text/pipeline_components/normalization/english_cleaners.py

@@ -0,0 +1,102 @@
+""" from https://github.com/keithito/tacotron """
+
+import inflect
+import re
+
+# List of (regular expression, replacement) pairs for abbreviations:
+_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
+  ('mrs', 'misess'),
+  ('mr', 'mister'),
+  ('dr', 'doctor'),
+  ('st', 'saint'),
+  ('co', 'company'),
+  ('jr', 'junior'),
+  ('maj', 'major'),
+  ('gen', 'general'),
+  ('drs', 'doctors'),
+  ('rev', 'reverend'),
+  ('lt', 'lieutenant'),
+  ('hon', 'honorable'),
+  ('sgt', 'sergeant'),
+  ('capt', 'captain'),
+  ('esq', 'esquire'),
+  ('ltd', 'limited'),
+  ('col', 'colonel'),
+  ('ft', 'fort'),
+]]
+
+
+def expand_abbreviations(text):
+  for regex, replacement in _abbreviations:
+    text = re.sub(regex, replacement, text)
+  return text
+
+
+_inflect = inflect.engine()
+_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
+_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
+_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
+_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
+_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
+_number_re = re.compile(r'[0-9]+')
+
+
+def _remove_commas(m):
+  return m.group(1).replace(',', '')
+
+
+def _expand_decimal_point(m):
+  return m.group(1).replace('.', ' point ')
+
+
+def _expand_dollars(m):
+  match = m.group(1)
+  parts = match.split('.')
+  if len(parts) > 2:
+    return match + ' dollars'  # Unexpected format
+  dollars = int(parts[0]) if parts[0] else 0
+  cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+  if dollars and cents:
+    dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+    cent_unit = 'cent' if cents == 1 else 'cents'
+    return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
+  elif dollars:
+    dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+    return '%s %s' % (dollars, dollar_unit)
+  elif cents:
+    cent_unit = 'cent' if cents == 1 else 'cents'
+    return '%s %s' % (cents, cent_unit)
+  else:
+    return 'zero dollars'
+
+
+def _expand_ordinal(m):
+  return _inflect.number_to_words(m.group(0))
+
+
+def _expand_number(m):
+  num = int(m.group(0))
+  if num > 1000 and num < 3000:
+    if num == 2000:
+      return 'two thousand'
+    elif num > 2000 and num < 2010:
+      return 'two thousand ' + _inflect.number_to_words(num % 100)
+    elif num % 100 == 0:
+      return _inflect.number_to_words(num // 100) + ' hundred'
+    else:
+      return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
+  else:
+    return _inflect.number_to_words(num, andword='')
+
+
+def normalize_numbers(text):
+  text = re.sub(_comma_number_re, _remove_commas, text)
+  text = re.sub(_pounds_re, r'\1 pounds', text)
+  text = re.sub(_dollars_re, _expand_dollars, text)
+  text = re.sub(_decimal_number_re, _expand_decimal_point, text)
+  text = re.sub(_ordinal_re, _expand_ordinal, text)
+  text = re.sub(_number_re, _expand_number, text)
+  return text
+
+def expand_numbers(text):
+  return normalize_numbers(text)

tts/text/pipeline_components/normalization/english_normalization.py

@@ -0,0 +1,13 @@
+from text.pipeline_components.normalization.common_cleaners import collapse_whitespace
+from text.pipeline_components.normalization.common_cleaners import convert_to_ascii
+from text.pipeline_components.normalization.common_cleaners import lowercase
+from text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers
+
+def normalize_english(raw_input_text: str) -> str:
+    text = convert_to_ascii(raw_input_text)
+    text = lowercase(text)
+    text = expand_numbers(text)
+    text = expand_abbreviations(text)
+    text = collapse_whitespace(text)
+
+    return text

tts/text/pipeline_components/normalization/spanish_normalization.py

@@ -0,0 +1,14 @@
+from text.pipeline_components.normalization.common_cleaners import collapse_whitespace
+from text.pipeline_components.normalization.common_cleaners import convert_to_ascii
+from text.pipeline_components.normalization.common_cleaners import lowercase
+
+def normalize_spanish(raw_input_text: str) -> str:
+    text = convert_to_ascii(raw_input_text)
+    text = lowercase(text)
+
+    # TODO: Expand abbreviations (Spanish)
+    # TODO: Expand numbers (Spanish)
+
+    text = collapse_whitespace(text)
+
+    return text

tts/text/pipeline_components/normalization/test_english_normalization.py

@@ -0,0 +1,19 @@
+from text.pipeline_components.normalization.english_normalization import normalize_english
+
+class TestEnglishNormalization:
+
+    def test_whitespace_collapse(self):
+        output = normalize_english(" \t \n   normalized   \n \t ")
+        assert output == " normalized "
+
+    def test_lowercase(self):
+        output = normalize_english("This is a test of Normalization.")
+        assert output == "this is a test of normalization."
+
+    def test_numbers(self):
+        output = normalize_english("1,000")
+        assert output == "one thousand"
+
+    def test_abbreviations(self):
+        output = normalize_english("mr. ed")
+        assert output == "mister ed"