Entirely reworked text handling pipeline

This breaks the text pipeline into the following pieces:

- input parsing (split known bracketed phoneme sections out first)

- normalization (fix spacing, numbers, abbreviations, etc.)

- phonetization (do CMUdict lookup or g2p prediction)

- sequencing (convert the symbolic representation to integer
  representation)

Each "pipeline" can have a unique combination of operations for each of
these pipeline steps.

This will grow over time.

tts/README.md

@@ -13,6 +13,13 @@ The following files are entrypoints and an essential part of FakeYou.com functio
    files).
 
 
+To Run Tests
+------------
+
+```
+pytest text/
+```
+
 Vocodes Server Parameters
 -------------------------
 

tts/pyproject.toml

@@ -0,0 +1,10 @@
+# For pytest
+
+[build-system]
+requires = ["setuptools>=42", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+pythonpath = [
+  "."
+]

tts/pyrightconfig.json

@@ -0,0 +1,39 @@
+{
+    "include": [
+      "text/"
+    ],
+    
+    "exclude": [
+      ".git/",
+      "**/*test_*.py",
+      "**/node_modules",
+      "**/__pycache__",
+      "NeMo/",
+      "hifi-gan/",
+      "hifigan/",
+      "nemo/",
+      "phonetics",
+      "python/",
+      "tacotron2/",
+      "temp/",
+      "waveglow/"
+    ],
+  
+    "ignore": [
+      "tacotron2/",
+      "**/*test_*.py"
+    ],
+  
+    "defineConstant": {
+    },
+  
+    "venvPath": "python/",
+  
+    "reportMissingImports": true,
+    "reportMissingTypeStubs": true,
+  
+    "pythonVersion": "3.6",
+    "pythonPlatform": "Linux",
+
+    "typeCheckingMode": "strict"
+  }

tts/setup.cfg

@@ -0,0 +1,7 @@
+# For pytest
+
+[metadata]
+name = tts
+
+[options]
+packages = find:

tts/text/pipeline/base_pipeline.py

@@ -0,0 +1,7 @@
+from typing import List
+
+class BasePipeline:
+
+    def user_input_to_sequence(self, input_text: str) -> List[int]:
+        """Convert raw user input into a numerical sequence for ML inference."""
+        return []

tts/text/pipeline/english_pipeline_v1.py

@@ -0,0 +1,74 @@
+from typing import List
+from text.pipeline.base_pipeline import BasePipeline
+from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
+from text.pipeline_components.normalization.english_normalization import normalize_english
+from text.pipeline_components.phonetization.base_phonetization import BasePhonetization
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
+from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
+from text.pipeline_components.sequencing.english_symbols_v1 import SPACE_INTEGER_ENCODING, arpabet_to_sequence, symbols_to_sequence
+
+class EnglishPipelineV1(BasePipeline):
+    """
+    This is a new text pipeline for FakeYou that will be introduced in July 2022.
+    It introduces Arpabet and individually upgradable components.
+    The code is not 1:1 with our notebooks, but we'll go to efforts to make sure that 
+    the model input expectations on both sides match one another.
+    """
+
+    def __init__(self, phonetization: BasePhonetization):
+        self.phonetization = phonetization
+
+    def user_input_to_sequence(self, input_text: str) -> List[int]:
+        """Convert raw user input into a numerical sequence for ML inference."""
+
+        parsed_sequence = parse_grapheme_and_arpabet_sequence(input_text)
+        self._normalize_graphemes(parsed_sequence)
+
+        parsed_sequence = self._convert_phonemes(parsed_sequence)
+        
+        return self._convert_to_integer_sequence(parsed_sequence)
+
+    def _normalize_graphemes(self, parsed_sequence: ParsedSequence):
+        for i in range(len(parsed_sequence)):
+            subsequence = parsed_sequence[i]
+            if isinstance(subsequence, GraphemeSequence):
+                normalized = normalize_english(subsequence.sequence)
+                parsed_sequence[i] = GraphemeSequence(normalized)
+
+
+    def _convert_phonemes(self, parsed_sequence: ParsedSequence) -> ParsedSequence:
+        # NB: We will likely break grapheme runs into multiple arpabet sequences, extending the 
+        # output list length.
+        updated_sequence = []
+
+        for subsequence in parsed_sequence:
+            if isinstance(subsequence, ArpabetSequence):
+                updated_sequence.append(subsequence)
+
+            elif isinstance(subsequence, GraphemeSequence):
+                converted_sequence = self.phonetization.try_convert_to_phonemes(subsequence.sequence)
+                updated_sequence.extend(converted_sequence)
+
+        return updated_sequence
+
+    def _convert_to_integer_sequence(self, parsed_sequence: ParsedSequence) -> List[int]:
+        integer_sequence = []
+        last_index = len(parsed_sequence) - 1
+
+        for i, subsequence in enumerate(parsed_sequence):
+            integer_subsequence = []
+
+            if isinstance(subsequence, ArpabetSequence):
+                integer_subsequence = arpabet_to_sequence(subsequence.sequence)
+            elif isinstance(subsequence, GraphemeSequence):
+                integer_subsequence = symbols_to_sequence(subsequence.sequence)
+                
+            if integer_subsequence:
+                integer_sequence.extend(integer_subsequence)
+
+                if i < last_index and integer_sequence[-1] != SPACE_INTEGER_ENCODING:
+                    # Separate words with single spaces if not already present.
+                    integer_sequence.append(SPACE_INTEGER_ENCODING)
+
+        return integer_sequence
+            

tts/text/pipeline/legacy_fakeyou_pipeline.py

@@ -0,0 +1,74 @@
+import re
+from typing import List
+from text.pipeline.base_pipeline import BasePipeline
+from text.pipeline_components.normalization.common_cleaners import collapse_whitespace, convert_to_ascii, lowercase
+from text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers
+from text.pipeline_components.sequencing.english_symbols_v1 import arpabet_to_sequence, symbols_to_sequence
+
+# Regular expression matching text enclosed in curly braces:
+_curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)')
+
+class LegacyFakeYouPipeline(BasePipeline):
+    """
+    This pipeline was in use from June 2021 to July 2022.
+    In contrast to legacy "vo.codes" (2020 - 2021), it does not 
+    leverage CMUDict to look up arpabet even when models were known to
+    be trained with arpabet support. (It did support curly braces for 
+    manual arpabet inclusion.)
+    """
+
+    def user_input_to_sequence(self, input_text: str) -> List[int]:
+        """Convert raw user input into a numerical sequence for ML inference."""
+        pre_sequence = self._preprocess_text(input_text)
+        return self._text_to_sequence(pre_sequence, ['english_cleaners'])
+
+    def _preprocess_text(self, input_text: str, is_raw_input: bool = True) -> str:
+        pre_sequence = ""
+        for line in input_text.split("\n"):
+            if len(line) < 1:
+                continue
+            if is_raw_input:
+                if line[-1] != ";":
+                    line = line + ";"
+            else:
+                pass
+            pre_sequence += line
+        return pre_sequence
+
+
+    def _text_to_sequence(self, text, cleaner_names):
+        '''Converts a string of text to a sequence of IDs corresponding to the symbols in the text.
+
+            The text can optionally have ARPAbet sequences enclosed in curly braces embedded
+            in it. For example, "Turn left on {HH AW1 S S T AH0 N} Street."
+
+            Args:
+            text: string to convert to a sequence
+            cleaner_names: names of the cleaner functions to run the text through
+
+            Returns:
+            List of integers corresponding to the symbols in the text
+        '''
+        sequence = []
+
+        # Check for curly braces and treat their contents as ARPAbet:
+        while len(text):
+            m = _curly_re.match(text)
+            if not m:
+                sequence += symbols_to_sequence(english_cleaners(text))
+                break
+            sequence += symbols_to_sequence(english_cleaners(m.group(1)))
+            sequence += arpabet_to_sequence(m.group(2))
+            text = m.group(3)
+
+        return sequence
+
+
+def english_cleaners(text):
+  '''Pipeline for English text, including number and abbreviation expansion.'''
+  text = convert_to_ascii(text)
+  text = lowercase(text)
+  text = expand_numbers(text)
+  text = expand_abbreviations(text)
+  text = collapse_whitespace(text)
+  return text

tts/text/pipeline/legacy_vocodes_pipeline.py

@@ -0,0 +1,2 @@
+# TODO: In the future we may allow uploads of legacy vocodes models.
+# These models utilize a very unique encoding scheme.

tts/text/pipeline/test_english_pipeline_v1.py

@@ -0,0 +1,65 @@
+from text.pipeline.english_pipeline_v1 import EnglishPipelineV1
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
+from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
+
+
+class TestEnglishPipelineV1:
+
+    def build_pipeline_instance(self):
+        # NB: CMUDict used in production may differ
+        cmudict = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(cmudict)
+        return EnglishPipelineV1(phonetization)
+
+    def test_no_input(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("")
+        assert output == []
+
+    def test_whitespace(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("   \n    \t ")
+        assert output == []
+
+    def test_single_word(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("hello")
+        assert output == [106, 73, 117, 123] # NB: Arpabet encoded
+
+    def test_two_words(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("hello hello")
+        assert output == [
+            106, 73, 117, 123, # NB: Arpabet encoded
+            11, # Space
+            106, 73, 117, 123, # Arpabet encoded
+        ]
+
+    def test_two_words_with_arpabet_segment(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("hello {HH AH0 L OW1}")
+        assert output == [
+            106, 73, 117, 123, # NB: Arpabet encoded
+            11, # Space
+            106, 73, 117, 123, # Manually arpabet encoded
+        ]
+
+    def test_words_and_punctuation(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("hello friends, welcome!")
+        assert output == [
+            106, 73, 117, 123, # NB: Arpabet encoded
+            11, # Space
+            104, 130, 94, 119, 90, 146,
+            11, # Space (TODO: Get rid of extraneous spaces with richer parsing)
+            6,  # Comma
+            11, # Space
+            144, 94, 117, 116, 73, 118, 
+            11, # Space
+            2, # Exclamation mark
+        ]
+
+    def test_apostrophe_word(self):
+        pipeline = self.build_pipeline_instance()
+        output = pipeline.user_input_to_sequence("you're") 
+        assert output == [145, 137, 130] # YOU'RE  Y UH1 R   ~   145  137  130

tts/text/pipeline/test_legacy_fakeyou_pipeline.py

@@ -0,0 +1,42 @@
+
+from text.pipeline.legacy_fakeyou_pipeline import LegacyFakeYouPipeline
+
+
+class TestLegacyFakeYouPipeline():
+
+    def test_no_input(self):
+        pipeline = LegacyFakeYouPipeline()
+        output = pipeline.user_input_to_sequence("")
+        assert output == []
+
+    def test_single_word(self):
+        pipeline = LegacyFakeYouPipeline()
+        output = pipeline.user_input_to_sequence("hello")
+        # NB: Graphemes remains ASCII! No CMUDict lookup or phoneme prediction.
+        # Also note the ending ";".
+        assert output == [
+            45, 42, 49, 49, 52, # "h e l l o"
+            9, # Semicolon line ender
+        ] 
+
+    def test_two_words(self):
+        pipeline = LegacyFakeYouPipeline()
+        output = pipeline.user_input_to_sequence("hello hello")
+        # NB: Graphemes remains ASCII! No CMUDict lookup or phoneme prediction.
+        assert output == [
+            45, 42, 49, 49, 52, # "h e l l o"
+            11, # Space
+            45, 42, 49, 49, 52, # "h e l l o"
+            9, # Semicolon line ender
+        ]
+
+    def test_two_words_with_arpabet_segment(self):
+        pipeline = LegacyFakeYouPipeline()
+        output = pipeline.user_input_to_sequence("hello {HH AH0 L OW1}")
+        # NB: Graphemes remains ASCII! No CMUDict lookup or phoneme prediction.
+        assert output == [
+            45, 42, 49, 49, 52, # "h e l l o"
+            11, # Space
+            106, 73, 117, 123, # Legacy FakeYou supports manual arpabet inclusion
+            9, # Semicolon line ender
+        ]

tts/text/pipeline_components/input_parsing/parser.py

@@ -0,0 +1,72 @@
+"""
+Parse raw text sequences into Arpabet and Grapheme sequences so that they can be 
+treated independently.
+
+Inspired by and including code from original Tacotron2 by Keith Ito.
+"""
+
+# pyright: strict
+
+import re
+from typing import List, Union, Any
+
+
+# Regular expression matching text enclosed in curly braces:
+_curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)')
+
+
+class ArpabetSequence:
+    """
+    Denotes an Arpabet phoneme sequence corresponding to a single word.
+    """
+    def __init__(self, sequence: str):
+        self.sequence = sequence
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, ArpabetSequence) and self.sequence == other.sequence
+    def __repr__(self):
+        return f"<ArpabetSequence: {self.sequence}>"
+
+
+class GraphemeSequence:
+    """
+    Denotes any string of non-Arpabet text, which may include multiple words,
+    spacing, and punctuation characters.
+    """
+    def __init__(self, sequence: str):
+        self.sequence = sequence
+    def __eq__(self, other: Any) -> bool:
+        return isinstance(other, GraphemeSequence) and self.sequence == other.sequence
+    def __repr__(self):
+        return f"<GraphemeSequence: {self.sequence}>"
+
+
+ParsedSequence = List[Union[ArpabetSequence, GraphemeSequence]]
+
+
+def parse_grapheme_and_arpabet_sequence(input_text: str) -> ParsedSequence:
+    """
+    Parse a string into a list of ArpabetSequence and GraphemeSequence so that each 
+    subsequence can be treated independently.
+    """
+    sequence : ParsedSequence = []
+
+    # Check for curly braces and treat their contents as ARPAbet:
+    while len(input_text):
+        input_text = input_text.strip()
+        m = _curly_re.match(input_text)
+        if not m:
+            sequence.append(GraphemeSequence(input_text))
+            break
+
+        before = m.group(1).strip()
+        bracketed = m.group(2).strip()
+        after = m.group(3).strip()
+
+        if before:
+            sequence.append(GraphemeSequence(before))
+        if bracketed:
+            sequence.append(ArpabetSequence(bracketed))
+
+        input_text = after
+
+    return sequence

tts/text/pipeline_components/input_parsing/test_parser.py

@@ -0,0 +1,45 @@
+from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, parse_grapheme_and_arpabet_sequence
+
+class TestParser:
+
+    def test_only_text(self):
+        parsed = parse_grapheme_and_arpabet_sequence("This is a test")
+        assert len(parsed) == 1
+        assert isinstance(parsed[0], GraphemeSequence)
+
+    def test_only_arpabet(self):
+        parsed = parse_grapheme_and_arpabet_sequence("{T EH1 S T IH0 NG}")
+        assert len(parsed) == 1
+        assert isinstance(parsed[0], ArpabetSequence)
+
+    def test_text_and_arpabet(self):
+        parsed = parse_grapheme_and_arpabet_sequence("This is a {T EH1 S T}")
+        assert len(parsed) == 2
+        assert isinstance(parsed[0], GraphemeSequence)
+        assert isinstance(parsed[1], ArpabetSequence)
+
+    def test_text_and_arpabet_interleved_1(self):
+        parsed = parse_grapheme_and_arpabet_sequence("{DH IH1 S} is a {T EH1 S T}")
+        assert len(parsed) == 3
+        assert isinstance(parsed[0], ArpabetSequence)
+        assert isinstance(parsed[1], GraphemeSequence)
+        assert isinstance(parsed[2], ArpabetSequence)
+
+    def test_text_and_arpabet_interleved_2(self):
+        parsed = parse_grapheme_and_arpabet_sequence("Check out {DH IH1 S} {T EH1 S T} of the {S IH1 S T AH0 M}")
+        assert len(parsed) == 5
+        assert isinstance(parsed[0], GraphemeSequence)
+        assert isinstance(parsed[1], ArpabetSequence)
+        assert isinstance(parsed[2], ArpabetSequence)
+        assert isinstance(parsed[3], GraphemeSequence)
+        assert isinstance(parsed[4], ArpabetSequence)
+
+    def test_text_and_arpabet_interleved_2(self):
+        parsed = parse_grapheme_and_arpabet_sequence("Check out {DH IH1 S} {T EH1 S T} of the {S IH1 S T AH0 M}.")
+        assert len(parsed) == 6
+        assert isinstance(parsed[0], GraphemeSequence)
+        assert isinstance(parsed[1], ArpabetSequence)
+        assert isinstance(parsed[2], ArpabetSequence)
+        assert isinstance(parsed[3], GraphemeSequence)
+        assert isinstance(parsed[4], ArpabetSequence)
+        assert isinstance(parsed[5], GraphemeSequence)

tts/text/pipeline_components/normalization/common_cleaners.py

@@ -0,0 +1,32 @@
+"""
+Copied or adapted from Keith Ito's Tacotron2 code.
+https://github.com/keithito/tacotron
+"""
+
+import re
+from unidecode import unidecode
+
+# Regular expression matching whitespace:
+_whitespace_re = re.compile(r'\s+')
+
+def lowercase(text: str) -> str:
+  return text.lower()
+
+def collapse_whitespace(text) -> str:
+  return re.sub(_whitespace_re, ' ', text)
+
+def convert_to_ascii(text: str) -> str:
+  return unidecode(text)
+
+def basic_cleaners(text: str) -> str:
+  '''Basic pipeline that lowercases and collapses whitespace without transliteration.'''
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text
+
+def transliteration_cleaners(text: str) -> str:
+  '''Pipeline for non-English text that transliterates to ASCII.'''
+  text = convert_to_ascii(text)
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text

tts/text/pipeline_components/normalization/english_cleaners.py

@@ -0,0 +1,102 @@
+""" from https://github.com/keithito/tacotron """
+
+import inflect
+import re
+
+# List of (regular expression, replacement) pairs for abbreviations:
+_abbreviations = [(re.compile('\\b%s\\.' % x[0], re.IGNORECASE), x[1]) for x in [
+  ('mrs', 'misess'),
+  ('mr', 'mister'),
+  ('dr', 'doctor'),
+  ('st', 'saint'),
+  ('co', 'company'),
+  ('jr', 'junior'),
+  ('maj', 'major'),
+  ('gen', 'general'),
+  ('drs', 'doctors'),
+  ('rev', 'reverend'),
+  ('lt', 'lieutenant'),
+  ('hon', 'honorable'),
+  ('sgt', 'sergeant'),
+  ('capt', 'captain'),
+  ('esq', 'esquire'),
+  ('ltd', 'limited'),
+  ('col', 'colonel'),
+  ('ft', 'fort'),
+]]
+
+
+def expand_abbreviations(text):
+  for regex, replacement in _abbreviations:
+    text = re.sub(regex, replacement, text)
+  return text
+
+
+_inflect = inflect.engine()
+_comma_number_re = re.compile(r'([0-9][0-9\,]+[0-9])')
+_decimal_number_re = re.compile(r'([0-9]+\.[0-9]+)')
+_pounds_re = re.compile(r'£([0-9\,]*[0-9]+)')
+_dollars_re = re.compile(r'\$([0-9\.\,]*[0-9]+)')
+_ordinal_re = re.compile(r'[0-9]+(st|nd|rd|th)')
+_number_re = re.compile(r'[0-9]+')
+
+
+def _remove_commas(m):
+  return m.group(1).replace(',', '')
+
+
+def _expand_decimal_point(m):
+  return m.group(1).replace('.', ' point ')
+
+
+def _expand_dollars(m):
+  match = m.group(1)
+  parts = match.split('.')
+  if len(parts) > 2:
+    return match + ' dollars'  # Unexpected format
+  dollars = int(parts[0]) if parts[0] else 0
+  cents = int(parts[1]) if len(parts) > 1 and parts[1] else 0
+  if dollars and cents:
+    dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+    cent_unit = 'cent' if cents == 1 else 'cents'
+    return '%s %s, %s %s' % (dollars, dollar_unit, cents, cent_unit)
+  elif dollars:
+    dollar_unit = 'dollar' if dollars == 1 else 'dollars'
+    return '%s %s' % (dollars, dollar_unit)
+  elif cents:
+    cent_unit = 'cent' if cents == 1 else 'cents'
+    return '%s %s' % (cents, cent_unit)
+  else:
+    return 'zero dollars'
+
+
+def _expand_ordinal(m):
+  return _inflect.number_to_words(m.group(0))
+
+
+def _expand_number(m):
+  num = int(m.group(0))
+  if num > 1000 and num < 3000:
+    if num == 2000:
+      return 'two thousand'
+    elif num > 2000 and num < 2010:
+      return 'two thousand ' + _inflect.number_to_words(num % 100)
+    elif num % 100 == 0:
+      return _inflect.number_to_words(num // 100) + ' hundred'
+    else:
+      return _inflect.number_to_words(num, andword='', zero='oh', group=2).replace(', ', ' ')
+  else:
+    return _inflect.number_to_words(num, andword='')
+
+
+def normalize_numbers(text):
+  text = re.sub(_comma_number_re, _remove_commas, text)
+  text = re.sub(_pounds_re, r'\1 pounds', text)
+  text = re.sub(_dollars_re, _expand_dollars, text)
+  text = re.sub(_decimal_number_re, _expand_decimal_point, text)
+  text = re.sub(_ordinal_re, _expand_ordinal, text)
+  text = re.sub(_number_re, _expand_number, text)
+  return text
+
+def expand_numbers(text):
+  return normalize_numbers(text)

tts/text/pipeline_components/normalization/english_normalization.py

@@ -0,0 +1,13 @@
+from text.pipeline_components.normalization.common_cleaners import collapse_whitespace
+from text.pipeline_components.normalization.common_cleaners import convert_to_ascii
+from text.pipeline_components.normalization.common_cleaners import lowercase
+from text.pipeline_components.normalization.english_cleaners import expand_abbreviations, expand_numbers
+
+def normalize_english(raw_input_text: str) -> str:
+    text = convert_to_ascii(raw_input_text)
+    text = lowercase(text)
+    text = expand_numbers(text)
+    text = expand_abbreviations(text)
+    text = collapse_whitespace(text)
+
+    return text

tts/text/pipeline_components/normalization/spanish_normalization.py

@@ -0,0 +1,14 @@
+from text.pipeline_components.normalization.common_cleaners import collapse_whitespace
+from text.pipeline_components.normalization.common_cleaners import convert_to_ascii
+from text.pipeline_components.normalization.common_cleaners import lowercase
+
+def normalize_spanish(raw_input_text: str) -> str:
+    text = convert_to_ascii(raw_input_text)
+    text = lowercase(text)
+
+    # TODO: Expand abbreviations (Spanish)
+    # TODO: Expand numbers (Spanish)
+
+    text = collapse_whitespace(text)
+
+    return text

tts/text/pipeline_components/normalization/test_english_normalization.py

@@ -0,0 +1,19 @@
+from text.pipeline_components.normalization.english_normalization import normalize_english
+
+class TestEnglishNormalization:
+
+    def test_whitespace_collapse(self):
+        output = normalize_english(" \t \n   normalized   \n \t ")
+        assert output == " normalized "
+
+    def test_lowercase(self):
+        output = normalize_english("This is a test of Normalization.")
+        assert output == "this is a test of normalization."
+
+    def test_numbers(self):
+        output = normalize_english("1,000")
+        assert output == "one thousand"
+
+    def test_abbreviations(self):
+        output = normalize_english("mr. ed")
+        assert output == "mister ed"

tts/text/pipeline_components/phonetization/base_phonetization.py

@@ -0,0 +1,19 @@
+
+import re
+from typing import List
+from text.pipeline_components.phonetization.cmudict import CMUDict
+
+# Regex taken from open source tacotron2 extensions
+# NB: This was broken and couldn't handle apostrophes
+#_arpabet_split_re = re.compile(r'[^\s]*{.*?}[^\s]*|[\w]+|[^\w\s]+')
+
+# Modified regex to not split apostrophe (eg. "you're", "I'm", etc.) or hyphenation (eg "avant-garde")
+_arpabet_split_re = re.compile(r'[^\s]*{.*?}[^\s]*|[\w]+\'[\w]+|[\w]+\-[\w]+|[\w]+|[^\w\s]+')
+
+class BasePhonetization:
+    def __init__(self, phoneme_dictionary: CMUDict):
+        self.phoneme_dictionary = phoneme_dictionary
+
+    def _split_grapheme_sentence_to_words(self, text_sequence: str) -> List[str]:
+        """Split a grapheme word sequence to individual grapheme words"""
+        return [word for word in _arpabet_split_re.findall(text_sequence)]

tts/text/pipeline_components/phonetization/cmudict.py

@@ -0,0 +1,72 @@
+""" 
+Contents of this file are mostly from https://github.com/keithito/tacotron
+"""
+
+import re
+
+# TODO: Don't use strings for arpabet phoneme representation, and share the memory.
+
+# Relative to TTS codebase root
+# This does not include any CMUDict extensions or alternatives (eg. for other languages).
+DEFAULT_CMUDICT_PATH = 'dictionaries/cmudict-0.7b'
+
+valid_symbols = [
+  'AA', 'AA0', 'AA1', 'AA2', 'AE', 'AE0', 'AE1', 'AE2', 'AH', 'AH0', 'AH1', 'AH2',
+  'AO', 'AO0', 'AO1', 'AO2', 'AW', 'AW0', 'AW1', 'AW2', 'AY', 'AY0', 'AY1', 'AY2',
+  'B', 'CH', 'D', 'DH', 'EH', 'EH0', 'EH1', 'EH2', 'ER', 'ER0', 'ER1', 'ER2', 'EY',
+  'EY0', 'EY1', 'EY2', 'F', 'G', 'HH', 'IH', 'IH0', 'IH1', 'IH2', 'IY', 'IY0', 'IY1',
+  'IY2', 'JH', 'K', 'L', 'M', 'N', 'NG', 'OW', 'OW0', 'OW1', 'OW2', 'OY', 'OY0',
+  'OY1', 'OY2', 'P', 'R', 'S', 'SH', 'T', 'TH', 'UH', 'UH0', 'UH1', 'UH2', 'UW',
+  'UW0', 'UW1', 'UW2', 'V', 'W', 'Y', 'Z', 'ZH'
+]
+
+_valid_symbol_set = set(valid_symbols)
+
+
+class CMUDict:
+  '''Thin wrapper around CMUDict data. http://www.speech.cs.cmu.edu/cgi-bin/cmudict'''
+  def __init__(self, file_or_path, keep_ambiguous=True):
+    if isinstance(file_or_path, str):
+      with open(file_or_path, encoding='latin-1') as f:
+        entries = _parse_cmudict(f)
+    else:
+      entries = _parse_cmudict(file_or_path)
+    if not keep_ambiguous:
+      entries = {word: pron for word, pron in entries.items() if len(pron) == 1}
+    self._entries = entries
+
+
+  def __len__(self):
+    return len(self._entries)
+
+
+  def lookup(self, word):
+    '''Returns list of ARPAbet pronunciations of the given word.'''
+    return self._entries.get(word.upper())
+
+
+
+_alt_re = re.compile(r'\([0-9]+\)')
+
+
+def _parse_cmudict(file):
+  cmudict = {}
+  for line in file:
+    if len(line) and (line[0] >= 'A' and line[0] <= 'Z' or line[0] == "'"):
+      parts = line.split('  ')
+      word = re.sub(_alt_re, '', parts[0])
+      pronunciation = _get_pronunciation(parts[1])
+      if pronunciation:
+        if word in cmudict:
+          cmudict[word].append(pronunciation)
+        else:
+          cmudict[word] = [pronunciation]
+  return cmudict
+
+
+def _get_pronunciation(s):
+  parts = s.strip().split(' ')
+  for part in parts:
+    if part not in _valid_symbol_set:
+      return None
+  return ' '.join(parts)

tts/text/pipeline_components/phonetization/english_phonetization.py

@@ -0,0 +1,30 @@
+from typing import List
+from text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence
+from text.pipeline_components.phonetization.base_phonetization import BasePhonetization
+from text.pipeline_components.phonetization.cmudict import CMUDict
+
+class EnglishPhonetization(BasePhonetization):
+
+    def __init__(self, phoneme_dictionary: CMUDict):
+        super().__init__(phoneme_dictionary)
+
+    def try_convert_to_phonemes(self, grapheme_sequence: str) -> ParsedSequence:
+        """
+        Attempt to enrich a grapheme sequence with phonemes where word lookup succeeds.
+        Grapheme symbols will be left in place.
+        The output list will need to be space-separated before inference.
+        """
+        words = self._split_grapheme_sentence_to_words(grapheme_sequence)
+        converted_words = []
+
+        for word in words:
+            maybe_phonemes = self.phoneme_dictionary.lookup(word)
+            if maybe_phonemes:
+                # NB: Dictionary returns a list of possible matches. 
+                # We'll just use the first.
+                converted_words.append(ArpabetSequence(maybe_phonemes[0]))
+            else:
+                # TODO: Use g2p prediction to predict the phonemes.
+                converted_words.append(GraphemeSequence(word))
+
+        return converted_words

tts/text/pipeline_components/phonetization/test_cmudict.py

@@ -0,0 +1,15 @@
+from text.pipeline_components.phonetization.cmudict import CMUDict
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH
+
+
+class TestCmuDict:
+
+    def test_lookup_success(self):
+        cmudict = CMUDict(DEFAULT_CMUDICT_PATH)
+        arpabet_sequence = cmudict.lookup('word')
+        assert arpabet_sequence == ['W ER1 D'] # TODO: Split the phonemes!
+
+    def test_lookup_failure(self):
+        cmudict = CMUDict(DEFAULT_CMUDICT_PATH)
+        arpabet_sequence = cmudict.lookup('fakeword')
+        assert arpabet_sequence == None

tts/text/pipeline_components/phonetization/test_english_phonetization.py

@@ -0,0 +1,94 @@
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
+from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
+from text.pipeline_components.input_parsing.parser import ArpabetSequence
+from text.pipeline_components.input_parsing.parser import GraphemeSequence
+
+class TestEnglishPhonetization:
+
+    def test_no_input(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("")
+        assert output == []
+
+    def test_all_words_in_dictionary(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("this is a test")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            ArpabetSequence("IH1 Z"),
+            ArpabetSequence("AH0"), 
+            ArpabetSequence("T EH1 S T"),
+        ]
+
+    def test_most_words_in_dictionary(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("nonword is a test")
+        assert output == [
+            GraphemeSequence("nonword"), 
+            ArpabetSequence("IH1 Z"), 
+            ArpabetSequence("AH0"), 
+            ArpabetSequence("T EH1 S T"),
+        ]
+
+    def test_words_with_apostrophe(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("I'm can't you're we're ain't")
+        assert output == [
+            ArpabetSequence("AY1 M"), 
+            ArpabetSequence("K AE1 N T"), 
+            ArpabetSequence("Y UH1 R"),
+            ArpabetSequence("W IY1 R"),
+            ArpabetSequence("EY1 N T"),
+        ]
+
+    def test_words_with_hyphenation(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("avant-garde bake-off")
+        assert output == [
+            ArpabetSequence("AH0 V AA1 N T G AA1 R D"), 
+            ArpabetSequence("B EY1 K AO1 F"), 
+        ]
+
+    def test_punctuation(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("this...")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            GraphemeSequence("..."),
+        ]
+        output = phonetization.try_convert_to_phonemes("this!")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            GraphemeSequence("!"),
+        ]
+        output = phonetization.try_convert_to_phonemes("this?")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            GraphemeSequence("?"),
+        ]
+
+    def test_extra_spaces(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("    this    is    a     test   ")
+        assert output == [
+            ArpabetSequence("DH IH1 S"), 
+            ArpabetSequence("IH1 Z"),
+            ArpabetSequence("AH0"), 
+            ArpabetSequence("T EH1 S T"),
+        ]
+
+    def test_multi_non_word_extra_spaces(self):
+        phoneme_dictionary = CMUDict(DEFAULT_CMUDICT_PATH)
+        phonetization = EnglishPhonetization(phoneme_dictionary)
+        output = phonetization.try_convert_to_phonemes("    nonword    anothernonword   ")
+        assert output == [
+            GraphemeSequence("nonword"), 
+            GraphemeSequence("anothernonword"), 
+        ]

tts/text/pipeline_components/sequencing/english_symbols_v1.py

@@ -0,0 +1,50 @@
+"""
+Originally adapted from https://github.com/keithito/tacotron
+
+This defines the set of symbols used in text input to the model.
+
+This is the set of character encodings that FakeYou and UberDuck models have
+been trained on. We freeze it here as "V1", because these may change in future
+notebooks.
+
+Original documentation: 
+    The default is a set of ASCII characters that works well for English or 
+    text that has been run through Unidecode. For other data, you can modify 
+    _characters. See TRAINING_DATA.md for details.
+"""
+
+from text.pipeline_components.phonetization import cmudict
+from typing import List
+
+# NB: DO NOT CHANGE THE ORDER OF SYMBOLS! 
+# Models are trained on and depend upon the order.
+_pad        = '_'
+_punctuation = '!\'(),.:;? ' #NB: Includes space
+_special = '-'
+_letters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz'
+
+# Prepend "@" to ARPAbet symbols to ensure uniqueness (some are the same as uppercase letters):
+_arpabet = ['@' + s for s in cmudict.valid_symbols]
+
+# Export all symbols:
+symbols = [_pad] + list(_special) + list(_punctuation) + list(_letters) + _arpabet
+
+# Mappings from symbol to numeric ID and vice versa:
+# The ordering **CANNOT** change.
+_symbol_to_id = {s: i for i, s in enumerate(symbols)}
+_id_to_symbol = {i: s for i, s in enumerate(symbols)}
+
+# Spaces can be injected when joining words.
+SPACE_INTEGER_ENCODING = _symbol_to_id[' ']
+
+
+def symbols_to_sequence(symbols: str) -> List[int]:
+    return [_symbol_to_id[s] for s in symbols if _should_keep_symbol(s)]
+
+
+def arpabet_to_sequence(arpabet_string: str) -> List[int]:
+    return symbols_to_sequence(['@' + s for s in arpabet_string.split()])
+
+
+def _should_keep_symbol(s):
+    return s in _symbol_to_id and s is not '_' and s is not '~'

tts/text/pipeline_components/sequencing/test_english_symbols_v1.py

@@ -0,0 +1,29 @@
+from text.pipeline_components.sequencing.english_symbols_v1 import arpabet_to_sequence, symbols_to_sequence
+
+class TestEnglishSymbols:
+
+    def test_no_input_arpabet(self):
+        output = arpabet_to_sequence('')
+        assert output == []
+
+    def test_no_input_symbols(self):
+        output = symbols_to_sequence('')
+        assert output == []
+
+    def test_arpabet_word(self):
+        # NB: Models are trained on these integer encodings. 
+        # If these tests break, the order of symbols may have changed. 
+        output = arpabet_to_sequence('T EH1 S T')
+        assert output == [133, 94, 131, 133]
+
+    def test_grapheme_word(self):
+        # NB: Models are trained on these integer encodings. 
+        # If these tests break, the order of symbols may have changed. 
+        output = symbols_to_sequence('test')
+        assert output == [57, 42, 56, 57]
+
+    def test_grapheme_sentence(self):
+        # NB: Models are trained on these integer encodings. 
+        # If these tests break, the order of symbols may have changed. 
+        output = symbols_to_sequence('this is a test.')
+        assert output == [57, 45, 46, 56, 11, 46, 56, 11, 38, 11, 57, 42, 56, 57, 7]

tts/vocodes_common.py

@@ -1,4 +1,5 @@
 import sys
+
 sys.path.append('waveglow/')
 import shutil
 import json
@@ -29,6 +30,11 @@ import subprocess
 import magic
 import os
 
+from text.pipeline.english_pipeline_v1 import EnglishPipelineV1
+from text.pipeline.legacy_fakeyou_pipeline import LegacyFakeYouPipeline
+from text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
+from text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
+
 def print_gpu_info():
     print('========================================')
     print('Python interpreter', sys.executable)
@@ -82,7 +88,27 @@ def load_hifigan_model(checkpoint_path, super_resolution = False):
     denoiser = HifiDenoiser(hifigan, mode="normal")
     return hifigan, h, denoiser
 
-def preprocess_text(text, raw_input=True):
+ENGLISH_PIPELINE_V1 = EnglishPipelineV1(EnglishPhonetization(CMUDict(DEFAULT_CMUDICT_PATH)))
+LEGACY_FAKEYOU_PIPELINE = LegacyFakeYouPipeline()
+
+def preprocess_text(raw_user_text: str):
+    integer_sequence = []
+
+    # TODO/NB: This word-based flag is for testing the rollout
+    if "this" in raw_user_text:
+        print("\n\n>>> ENGLISH V1\n\n")
+        integer_sequence = ENGLISH_PIPELINE_V1.user_input_to_sequence(raw_user_text)
+    else: 
+        print("\n\n>>> LEGACY\n\n")
+        integer_sequence = LEGACY_FAKEYOU_PIPELINE.user_input_to_sequence(raw_user_text)
+
+    with torch.no_grad():
+        # save VRAM by not including gradients
+        sequence = np.array(integer_sequence)[None, :]
+        sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long()
+    return sequence
+
+def preprocess_text_old(text, raw_input=True):
     # TODO: Handle ARPAbet
     pre_sequence = ""
     for line in text.split("\n"):
@@ -96,6 +122,7 @@ def preprocess_text(text, raw_input=True):
             pass
         pre_sequence += line
 
+    print(pre_sequence)
     with torch.no_grad():
         # save VRAM by not including gradients
         sequence = np.array(text_to_sequence(pre_sequence, ['english_cleaners']))[None, :]