From 1779d988cbc302bf956ad5cb037b8f75e53e0c9d Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Thu, 30 Jun 2022 18:56:18 -0400
Subject: [PATCH] tests for english normalization

---
 tts/text/normalization/BaseNormalization.py   |  1 -
 .../normalization/EnglishNormalization.py     |  6 ----
 tts/text/normalization/common_cleaners.py     | 32 +++++++++++++++++++
 .../normalization/english_normalization.py    | 17 ++++++++++
 .../test_english_normalization.py             | 13 ++++++++
 5 files changed, 62 insertions(+), 7 deletions(-)
 delete mode 100644 tts/text/normalization/EnglishNormalization.py
 create mode 100644 tts/text/normalization/common_cleaners.py
 create mode 100644 tts/text/normalization/english_normalization.py
 create mode 100644 tts/text/normalization/test_english_normalization.py

diff --git a/tts/text/normalization/BaseNormalization.py b/tts/text/normalization/BaseNormalization.py
index 81e3133..fbce993 100644
--- a/tts/text/normalization/BaseNormalization.py
+++ b/tts/text/normalization/BaseNormalization.py
@@ -1,4 +1,3 @@
-
 class BaseNormalization:
 
     def normalize(self, raw_input_text: str) -> str:
diff --git a/tts/text/normalization/EnglishNormalization.py b/tts/text/normalization/EnglishNormalization.py
deleted file mode 100644
index c5d93b6..0000000
--- a/tts/text/normalization/EnglishNormalization.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from text.normalization.BaseNormalization import BaseNormalization
-
-class EnglishNormalization(BaseNormalization):
-
-    def normalize(self, raw_input_text: str) -> str:
-        return raw_input_text
diff --git a/tts/text/normalization/common_cleaners.py b/tts/text/normalization/common_cleaners.py
new file mode 100644
index 0000000..2fefc93
--- /dev/null
+++ b/tts/text/normalization/common_cleaners.py
@@ -0,0 +1,32 @@
+"""
+Copied or adapted from Keith Ito's Tacotron2 code.
+https://github.com/keithito/tacotron
+"""
+
+import re
+from unidecode import unidecode
+
+# Regular expression matching whitespace:
+_whitespace_re = re.compile(r'\s+')
+
+def lowercase(text: str) -> str:
+  return text.lower()
+
+def collapse_whitespace(text) -> str:
+  return re.sub(_whitespace_re, ' ', text)
+
+def convert_to_ascii(text: str) -> str:
+  return unidecode(text)
+
+def basic_cleaners(text: str) -> str:
+  '''Basic pipeline that lowercases and collapses whitespace without transliteration.'''
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text
+
+def transliteration_cleaners(text: str) -> str:
+  '''Pipeline for non-English text that transliterates to ASCII.'''
+  text = convert_to_ascii(text)
+  text = lowercase(text)
+  text = collapse_whitespace(text)
+  return text
diff --git a/tts/text/normalization/english_normalization.py b/tts/text/normalization/english_normalization.py
new file mode 100644
index 0000000..cc37609
--- /dev/null
+++ b/tts/text/normalization/english_normalization.py
@@ -0,0 +1,17 @@
+from tts.text.normalization.BaseNormalization import BaseNormalization
+from tts.text.normalization.common_cleaners import collapse_whitespace
+from tts.text.normalization.common_cleaners import convert_to_ascii
+from tts.text.normalization.common_cleaners import lowercase
+
+class EnglishNormalization(BaseNormalization):
+
+    def normalize(self, raw_input_text: str) -> str:
+        text = convert_to_ascii(raw_input_text)
+        text = lowercase(text)
+
+        # TODO: Expand abbreviations
+        # TODO: Expand numbers
+
+        text = collapse_whitespace(text)
+
+        return text
diff --git a/tts/text/normalization/test_english_normalization.py b/tts/text/normalization/test_english_normalization.py
new file mode 100644
index 0000000..4e19092
--- /dev/null
+++ b/tts/text/normalization/test_english_normalization.py
@@ -0,0 +1,13 @@
+from tts.text.normalization.english_normalization import EnglishNormalization
+
+class TestEnglishNormalization:
+
+    def test_whitespace_collapse(self):
+        normalizer = EnglishNormalization()
+        output = normalizer.normalize(" \t \n   normalized   \n \t ")
+        assert output == " normalized "
+
+    def test_lowercase(self):
+        normalizer = EnglishNormalization()
+        output = normalizer.normalize("This is a test of Normalization.")
+        assert output == "this is a test of normalization."