tests for english normalization

tts/text/normalization/BaseNormalization.py

@@ -1,4 +1,3 @@
-
 class BaseNormalization:
 
     def normalize(self, raw_input_text: str) -> str:

tts/text/normalization/EnglishNormalization.py

@@ -1,6 +0,0 @@
-from text.normalization.BaseNormalization import BaseNormalization
-
-class EnglishNormalization(BaseNormalization):
-
-    def normalize(self, raw_input_text: str) -> str:
-        return raw_input_text

tts/text/normalization/common_cleaners.py

@@ -0,0 +1,32 @@
+"""
+Copied or adapted from Keith Ito's Tacotron2 code.
+https://github.com/keithito/tacotron
+"""
+
+import re
+from unidecode import unidecode
+
+# Regular expression matching whitespace:
+_whitespace_re = re.compile(r'\s+')
+
+def lowercase(text: str) -> str:
+  return text.lower()
+
+def collapse_whitespace(text) -> str:
+  return re.sub(_whitespace_re, ' ', text)
+
+def convert_to_ascii(text: str) -> str:
+  return unidecode(text)
+
+def basic_cleaners(text: str) -> str:
+  '''Basic pipeline that lowercases and collapses whitespace without transliteration.'''
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text
+
+def transliteration_cleaners(text: str) -> str:
+  '''Pipeline for non-English text that transliterates to ASCII.'''
+  text = convert_to_ascii(text)
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text

tts/text/normalization/english_normalization.py

@@ -0,0 +1,17 @@
+from tts.text.normalization.BaseNormalization import BaseNormalization
+from tts.text.normalization.common_cleaners import collapse_whitespace
+from tts.text.normalization.common_cleaners import convert_to_ascii
+from tts.text.normalization.common_cleaners import lowercase
+
+class EnglishNormalization(BaseNormalization):
+
+    def normalize(self, raw_input_text: str) -> str:
+        text = convert_to_ascii(raw_input_text)
+        text = lowercase(text)
+
+        # TODO: Expand abbreviations
+        # TODO: Expand numbers
+
+        text = collapse_whitespace(text)
+
+        return text

tts/text/normalization/test_english_normalization.py

@@ -0,0 +1,13 @@
+from tts.text.normalization.english_normalization import EnglishNormalization
+
+class TestEnglishNormalization:
+
+    def test_whitespace_collapse(self):
+        normalizer = EnglishNormalization()
+        output = normalizer.normalize(" \t \n   normalized   \n \t ")
+        assert output == " normalized "
+
+    def test_lowercase(self):
+        normalizer = EnglishNormalization()
+        output = normalizer.normalize("This is a test of Normalization.")
+        assert output == "this is a test of normalization."