Work on text parsing, testing

tts/README.md

@@ -13,6 +13,13 @@ The following files are entrypoints and an essential part of FakeYou.com functio
    files).
 
 
+To Run Tests
+------------
+
+```
+pytest text/
+```
+
 Vocodes Server Parameters
 -------------------------
 

tts/phonetics/base.py

@@ -13,4 +13,3 @@ class ArpabetTransformer:
 
 
 
-

tts/text/normalization/BaseNormalization.py

@@ -0,0 +1,5 @@
+
+class BaseNormalization:
+
+    def normalize(self, raw_input_text):
+        return raw_input_text

tts/text/normalization/EnglishNormalization.py

@@ -0,0 +1,6 @@
+from text.normalization.BaseNormalization import BaseNormalization
+
+class EnglishNormalization(BaseNormalization):
+
+    def normalize(self, raw_input_text):
+        return raw_input_text

tts/text/normalization/SpanishNormalization.py

@@ -0,0 +1,6 @@
+from text.normalization.BaseNormalization import BaseNormalization
+
+class SpanishNormalization(BaseNormalization):
+
+    def normalize(self, raw_input_text):
+        return raw_input_text

tts/text/phonetization/parser.py

@@ -0,0 +1,46 @@
+"""
+Parse raw text sequences into Arpabet and Grapheme sequences so that they can be 
+treated independently.
+
+Inspired by and including code from original Tacotron2 by Keith Ito.
+"""
+
+import re
+
+# Regular expression matching text enclosed in curly braces:
+_curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)')
+
+class ArpabetSequence:
+    def __init__(self, sequence):
+        self.sequence = sequence
+
+class GraphemeSquence:
+    def __init__(self, sequence):
+        self.sequence = sequence
+
+def parse_grapheme_and_arpabet_sequence(input_text):
+    """
+    Parse a string into an array of ArpabetSequence and GraphemeSequence so that each can be treated independently.
+    """
+    sequence = []
+
+    # Check for curly braces and treat their contents as ARPAbet:
+    while len(input_text):
+        input_text = input_text.strip()
+        m = _curly_re.match(input_text)
+        if not m:
+            sequence.append(GraphemeSquence(input_text))
+            break
+
+        before = m.group(1).strip()
+        bracketed = m.group(2).strip()
+        after = m.group(3).strip()
+
+        if before:
+            sequence.append(GraphemeSquence(before))
+        if bracketed:
+            sequence.append(ArpabetSequence(bracketed))
+
+        input_text = after
+
+    return sequence

tts/text/phonetization/test_parser.py

@@ -0,0 +1,45 @@
+from tts.text.phonetization.parser import ArpabetSequence, GraphemeSquence, parse_grapheme_and_arpabet_sequence
+
+class TestParser:
+
+    def test_only_text(self):
+        parsed = parse_grapheme_and_arpabet_sequence("This is a test")
+        assert len(parsed) == 1
+        assert isinstance(parsed[0], GraphemeSquence)
+
+    def test_only_arpabet(self):
+        parsed = parse_grapheme_and_arpabet_sequence("{T EH1 S T IH0 NG}")
+        assert len(parsed) == 1
+        assert isinstance(parsed[0], ArpabetSequence)
+
+    def test_text_and_arpabet(self):
+        parsed = parse_grapheme_and_arpabet_sequence("This is a {T EH1 S T}")
+        assert len(parsed) == 2
+        assert isinstance(parsed[0], GraphemeSquence)
+        assert isinstance(parsed[1], ArpabetSequence)
+
+    def test_text_and_arpabet_interleved_1(self):
+        parsed = parse_grapheme_and_arpabet_sequence("{DH IH1 S} is a {T EH1 S T}")
+        assert len(parsed) == 3
+        assert isinstance(parsed[0], ArpabetSequence)
+        assert isinstance(parsed[1], GraphemeSquence)
+        assert isinstance(parsed[2], ArpabetSequence)
+
+    def test_text_and_arpabet_interleved_2(self):
+        parsed = parse_grapheme_and_arpabet_sequence("Check out {DH IH1 S} {T EH1 S T} of the {S IH1 S T AH0 M}")
+        assert len(parsed) == 5
+        assert isinstance(parsed[0], GraphemeSquence)
+        assert isinstance(parsed[1], ArpabetSequence)
+        assert isinstance(parsed[2], ArpabetSequence)
+        assert isinstance(parsed[3], GraphemeSquence)
+        assert isinstance(parsed[4], ArpabetSequence)
+
+    def test_text_and_arpabet_interleved_2(self):
+        parsed = parse_grapheme_and_arpabet_sequence("Check out {DH IH1 S} {T EH1 S T} of the {S IH1 S T AH0 M}.")
+        assert len(parsed) == 6
+        assert isinstance(parsed[0], GraphemeSquence)
+        assert isinstance(parsed[1], ArpabetSequence)
+        assert isinstance(parsed[2], ArpabetSequence)
+        assert isinstance(parsed[3], GraphemeSquence)
+        assert isinstance(parsed[4], ArpabetSequence)
+        assert isinstance(parsed[5], GraphemeSquence)

tts/vocodes_common.py

@@ -96,6 +96,7 @@ def preprocess_text(text, raw_input=True):
             pass
         pre_sequence += line
 
+    print(pre_sequence)
     with torch.no_grad():
         # save VRAM by not including gradients
         sequence = np.array(text_to_sequence(pre_sequence, ['english_cleaners']))[None, :]