Entirely reworked text handling pipeline

This breaks the text pipeline into the following pieces:

- input parsing (split known bracketed phoneme sections out first)

- normalization (fix spacing, numbers, abbreviations, etc.)

- phonetization (do CMUdict lookup or g2p prediction)

- sequencing (convert the symbolic representation to integer
  representation)

Each "pipeline" can have a unique combination of operations for each of
these pipeline steps.

This will grow over time.

tts/text/pipeline_components/phonetization/base_phonetization.py

@@ -0,0 +1,19 @@
+
+import re
+from typing import List
+from text.pipeline_components.phonetization.cmudict import CMUDict
+
+# Regex taken from open source tacotron2 extensions
+# NB: This was broken and couldn't handle apostrophes
+#_arpabet_split_re = re.compile(r'[^\s]*{.*?}[^\s]*|[\w]+|[^\w\s]+')
+
+# Modified regex to not split apostrophe (eg. "you're", "I'm", etc.) or hyphenation (eg "avant-garde")
+_arpabet_split_re = re.compile(r'[^\s]*{.*?}[^\s]*|[\w]+\'[\w]+|[\w]+\-[\w]+|[\w]+|[^\w\s]+')
+
+class BasePhonetization:
+    def __init__(self, phoneme_dictionary: CMUDict):
+        self.phoneme_dictionary = phoneme_dictionary
+
+    def _split_grapheme_sentence_to_words(self, text_sequence: str) -> List[str]:
+        """Split a grapheme word sequence to individual grapheme words"""
+        return [word for word in _arpabet_split_re.findall(text_sequence)]

tts/text/pipeline_components/phonetization/cmudict.py

@@ -0,0 +1,72 @@
+""" 
+Contents of this file are mostly from https://github.com/keithito/tacotron
+"""
+
+import re
+
+# TODO: Don't use strings for arpabet phoneme representation, and share the memory.
+
+# Relative to TTS codebase root
+# This does not include any CMUDict extensions or alternatives (eg. for other languages).
+DEFAULT_CMUDICT_PATH = 'dictionaries/cmudict-0.7b'
+
+valid_symbols = [
+  'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2',
+  'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2',
+  'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY',
+  'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1',
+  'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0',
+  'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW',
+  'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH'
+]
+
+_valid_symbol_set = set(valid_symbols)
+
+
+class CMUDict:
+  '''Thin wrapper around CMUDict data. http://www.speech.cs.cmu.edu/cgi-bin/cmudict'''
+  def __init__(self, file_or_path, keep_ambiguous=True):
+    if isinstance(file_or_path, str):
+      with open(file_or_path, encoding='latin-1') as f:
+        entries = _parse_cmudict(f)
+    else:
+      entries = _parse_cmudict(file_or_path)
+    if not keep_ambiguous:
+      entries = {word: pron for word, pron in entries.items() if len(pron) == 1}
+    self._entries = entries
+
+
+  def __len__(self):
+    return len(self._entries)
+
+
+  def lookup(self, word):
+    '''Returns list of ARPAbet pronunciations of the given word.'''
+    return self._entries.get(word.upper())
+
+
+
+_alt_re = re.compile(r'\([0-9]+\)')
+
+
+def _parse_cmudict(file):
+  cmudict = {}
+  for line in file:
+    if len(line) and (line[0] >= 'A' and line[0] <= 'Z' or line[0] == "'"):
+      parts = line.split('  ')
+      word = re.sub(_alt_re, '', parts[0])
+      pronunciation = _get_pronunciation(parts[1])
+      if pronunciation:
+        if word in cmudict:
+          cmudict[word].append(pronunciation)
+        else:
+          cmudict[word] = [pronunciation]
+  return cmudict
+
+
+def _get_pronunciation(s):
+  parts = s.strip().split(' ')
+  for part in parts:
+    if part not in _valid_symbol_set:
+      return None
+  return ' '.join(parts)

tts/text/pipeline_components/phonetization/english_phonetization.py

@@ -0,0 +1,30 @@
+from typing import List
+from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence
+from text.pipeline_components.phonetization.base_phonetization import BasePhonetization
+from text.pipeline_components.phonetization.cmudict import CMUDict
+
+class EnglishPhonetization(BasePhonetization):
+
+    def __init__(self, phoneme_dictionary: CMUDict):
+        super().__init__(phoneme_dictionary)
+
+    def try_convert_to_phonemes(self, grapheme_sequence: str) -> ParsedSequence:
+        """
+        Attempt to enrich a grapheme sequence with phonemes where word lookup succeeds.
+        Grapheme symbols will be left in place.
+        The output list will need to be space-separated before inference.
+        """
+        words = self._split_grapheme_sentence_to_words(grapheme_sequence)
+        converted_words = []
+
+        for word in words:
+            maybe_phonemes = self.phoneme_dictionary.lookup(word)
+            if maybe_phonemes:
+                # NB: Dictionary returns a list of possible matches. 
+                # We'll just use the first.
+                converted_words.append(ArpabetSequence(maybe_phonemes[0]))
+            else:
+                # TODO: Use g2p prediction to predict the phonemes.
+                converted_words.append(GraphemeSequence(word))
+
+        return converted_words

tts/text/pipeline_components/phonetization/test_cmudict.py

@@ -0,0 +1,15 @@
+from text.pipeline_components.phonetization.cmudict import CMUDict
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH
+
+
+class TestCmuDict:
+
+    def test_lookup_success(self):
+        cmudict = CMUDict(DEFAULT_CMUDICT_PATH)
+        arpabet_sequence = cmudict.lookup('word')
+        assert arpabet_sequence == ['W ER1 D'] # TODO: Split the phonemes!
+
+    def test_lookup_failure(self):
+        cmudict = CMUDict(DEFAULT_CMUDICT_PATH)
+        arpabet_sequence = cmudict.lookup('fakeword')
+        assert arpabet_sequence == None

tts/text/pipeline_components/phonetization/test_english_phonetization.py

@@ -0,0 +1,94 @@
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
+from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
+from text.pipeline_components.input_parsing.parser import ArpabetSequence
+from text.pipeline_components.input_parsing.parser import GraphemeSequence
+
+class TestEnglishPhonetization:
+
+    def test_no_input(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("")
+        assert output == []
+
+    def test_all_words_in_dictionary(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("this is a test")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            ArpabetSequence("IH1 Z"),
+            ArpabetSequence("AH0"), 
+            ArpabetSequence("T EH1 S T"),
+        ]
+
+    def test_most_words_in_dictionary(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("nonword is a test")
+        assert output == [
+            GraphemeSequence("nonword"), 
+            ArpabetSequence("IH1 Z"), 
+            ArpabetSequence("AH0"), 
+            ArpabetSequence("T EH1 S T"),
+        ]
+
+    def test_words_with_apostrophe(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("I'm can't you're we're ain't")
+        assert output == [
+            ArpabetSequence("AY1 M"), 
+            ArpabetSequence("K AE1 N T"), 
+            ArpabetSequence("Y UH1 R"),
+            ArpabetSequence("W IY1 R"),
+            ArpabetSequence("EY1 N T"),
+        ]
+
+    def test_words_with_hyphenation(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("avant-garde bake-off")
+        assert output == [
+            ArpabetSequence("AH0 V AA1 N T G AA1 R D"), 
+            ArpabetSequence("B EY1 K AO1 F"), 
+        ]
+
+    def test_punctuation(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("this...")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            GraphemeSequence("..."),
+        ]
+        output = phonetization.try_convert_to_phonemes("this!")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            GraphemeSequence("!"),
+        ]
+        output = phonetization.try_convert_to_phonemes("this?")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            GraphemeSequence("?"),
+        ]
+
+    def test_extra_spaces(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("    this    is    a     test   ")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            ArpabetSequence("IH1 Z"),
+            ArpabetSequence("AH0"), 
+            ArpabetSequence("T EH1 S T"),
+        ]
+
+    def test_multi_non_word_extra_spaces(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("    nonword    anothernonword   ")
+        assert output == [
+            GraphemeSequence("nonword"), 
+            GraphemeSequence("anothernonword"), 
+        ]