Entirely reworked text handling pipeline

This breaks the text pipeline into the following pieces:

- input parsing (split known bracketed phoneme sections out first)

- normalization (fix spacing, numbers, abbreviations, etc.)

- phonetization (do CMUdict lookup or g2p prediction)

- sequencing (convert the symbolic representation to integer
  representation)

Each "pipeline" can have a unique combination of operations for each of
these pipeline steps.

This will grow over time.

tts/text/pipeline_components/sequencing/english_symbols_v1.py

@@ -0,0 +1,50 @@
+"""
+Originally adapted from https://github.com/keithito/tacotron
+
+This defines the set of symbols used in text input to the model.
+
+This is the set of character encodings that FakeYou and UberDuck models have
+been trained on. We freeze it here as "V1", because these may change in future
+notebooks.
+
+Original documentation: 
+    The default is a set of ASCII characters that works well for English or 
+    text that has been run through Unidecode. For other data, you can modify 
+    _characters. See TRAINING_DATA.md for details.
+"""
+
+from text.pipeline_components.phonetization import cmudict
+from typing import List
+
+# NB: DO NOT CHANGE THE ORDER OF SYMBOLS! 
+# Models are trained on and depend upon the order.
+_pad        = '_'
+_punctuation = '!\'(),.:;? ' #NB: Includes space
+_special = '-'
+_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+
+# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
+_arpabet = ['@' + s for s in cmudict.valid_symbols]
+
+# Export all symbols:
+symbols = [_pad] + list(_special) + list(_punctuation) + list(_letters) + _arpabet
+
+# Mappings from symbol to numeric ID and vice versa:
+# The ordering **CANNOT** change.
+_symbol_to_id = {s: i for i, s in enumerate(symbols)}
+_id_to_symbol = {i: s for i, s in enumerate(symbols)}
+
+# Spaces can be injected when joining words.
+SPACE_INTEGER_ENCODING = _symbol_to_id[' ']
+
+
+def symbols_to_sequence(symbols: str) -> List[int]:
+    return [_symbol_to_id[s] for s in symbols if _should_keep_symbol(s)]
+
+
+def arpabet_to_sequence(arpabet_string: str) -> List[int]:
+    return symbols_to_sequence(['@' + s for s in arpabet_string.split()])
+
+
+def _should_keep_symbol(s):
+    return s in _symbol_to_id and s is not '_' and s is not '~'

tts/text/pipeline_components/sequencing/test_english_symbols_v1.py

@@ -0,0 +1,29 @@
+from text.pipeline_components.sequencing.english_symbols_v1 import arpabet_to_sequence, symbols_to_sequence
+
+class TestEnglishSymbols:
+
+    def test_no_input_arpabet(self):
+        output = arpabet_to_sequence('')
+        assert output == []
+
+    def test_no_input_symbols(self):
+        output = symbols_to_sequence('')
+        assert output == []
+
+    def test_arpabet_word(self):
+        # NB: Models are trained on these integer encodings. 
+        # If these tests break, the order of symbols may have changed. 
+        output = arpabet_to_sequence('T EH1 S T')
+        assert output == [133, 94, 131, 133]
+
+    def test_grapheme_word(self):
+        # NB: Models are trained on these integer encodings. 
+        # If these tests break, the order of symbols may have changed. 
+        output = symbols_to_sequence('test')
+        assert output == [57, 42, 56, 57]
+
+    def test_grapheme_sentence(self):
+        # NB: Models are trained on these integer encodings. 
+        # If these tests break, the order of symbols may have changed. 
+        output = symbols_to_sequence('this is a test.')
+        assert output == [57, 45, 46, 56, 11, 46, 56, 11, 38, 11, 57, 42, 56, 57, 7]