mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-22 08:16:08 +00:00
refactor
This commit is contained in:
@@ -1,13 +1,11 @@
|
|||||||
from typing import List
|
from typing import List
|
||||||
from tts.text.pipeline.base_pipeline import BasePipeline
|
from tts.text.pipeline.base_pipeline import BasePipeline
|
||||||
from tts.text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
|
from tts.text.pipeline_components.input_parsing.parser import ArpabetSequence, GraphemeSequence, ParsedSequence, parse_grapheme_and_arpabet_sequence
|
||||||
from tts.text.pipeline_components.normalization.english_normalization import EnglishNormalization
|
from tts.text.pipeline_components.normalization.english_normalization import normalize_english
|
||||||
from tts.text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
|
from tts.text.pipeline_components.phonetization.cmudict import DEFAULT_CMUDICT_PATH, CMUDict
|
||||||
from tts.text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
|
from tts.text.pipeline_components.phonetization.english_phonetization import EnglishPhonetization
|
||||||
from tts.text.pipeline_components.sequencing.english_symbols_v1 import SPACE_INTEGER_ENCODING, arpabet_to_sequence, symbols_to_sequence
|
from tts.text.pipeline_components.sequencing.english_symbols_v1 import SPACE_INTEGER_ENCODING, arpabet_to_sequence, symbols_to_sequence
|
||||||
|
|
||||||
_ENGLISH_NORMALIZER = EnglishNormalization()
|
|
||||||
|
|
||||||
_ENGLISH_DICTIONARY = CMUDict(DEFAULT_CMUDICT_PATH)
|
_ENGLISH_DICTIONARY = CMUDict(DEFAULT_CMUDICT_PATH)
|
||||||
_ENGLISH_PHONETIZER = EnglishPhonetization(_ENGLISH_DICTIONARY)
|
_ENGLISH_PHONETIZER = EnglishPhonetization(_ENGLISH_DICTIONARY)
|
||||||
|
|
||||||
@@ -27,7 +25,7 @@ class EnglishPipelineV1(BasePipeline):
|
|||||||
for i in range(len(parsed_sequence)):
|
for i in range(len(parsed_sequence)):
|
||||||
subsequence = parsed_sequence[i]
|
subsequence = parsed_sequence[i]
|
||||||
if isinstance(subsequence, GraphemeSequence):
|
if isinstance(subsequence, GraphemeSequence):
|
||||||
normalized = _ENGLISH_NORMALIZER.normalize(subsequence.sequence)
|
normalized = normalize_english(subsequence.sequence)
|
||||||
parsed_sequence[i] = GraphemeSequence(normalized)
|
parsed_sequence[i] = GraphemeSequence(normalized)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +0,0 @@
|
|||||||
class BaseNormalization:
|
|
||||||
|
|
||||||
def normalize(self, raw_input_text: str) -> str:
|
|
||||||
return raw_input_text
|
|
||||||
@@ -1,17 +1,14 @@
|
|||||||
from tts.text.pipeline_components.normalization.base_normalization import BaseNormalization
|
|
||||||
from tts.text.pipeline_components.normalization.common_cleaners import collapse_whitespace
|
from tts.text.pipeline_components.normalization.common_cleaners import collapse_whitespace
|
||||||
from tts.text.pipeline_components.normalization.common_cleaners import convert_to_ascii
|
from tts.text.pipeline_components.normalization.common_cleaners import convert_to_ascii
|
||||||
from tts.text.pipeline_components.normalization.common_cleaners import lowercase
|
from tts.text.pipeline_components.normalization.common_cleaners import lowercase
|
||||||
|
|
||||||
class EnglishNormalization(BaseNormalization):
|
def normalize_english(raw_input_text: str) -> str:
|
||||||
|
text = convert_to_ascii(raw_input_text)
|
||||||
|
text = lowercase(text)
|
||||||
|
|
||||||
def normalize(self, raw_input_text: str) -> str:
|
# TODO: Expand abbreviations
|
||||||
text = convert_to_ascii(raw_input_text)
|
# TODO: Expand numbers
|
||||||
text = lowercase(text)
|
|
||||||
|
|
||||||
# TODO: Expand abbreviations
|
text = collapse_whitespace(text)
|
||||||
# TODO: Expand numbers
|
|
||||||
|
|
||||||
text = collapse_whitespace(text)
|
return text
|
||||||
|
|
||||||
return text
|
|
||||||
|
|||||||
@@ -1,6 +1,14 @@
|
|||||||
from tts.text.pipeline_components.normalization.base_normalization import BaseNormalization
|
from tts.text.pipeline_components.normalization.common_cleaners import collapse_whitespace
|
||||||
|
from tts.text.pipeline_components.normalization.common_cleaners import convert_to_ascii
|
||||||
|
from tts.text.pipeline_components.normalization.common_cleaners import lowercase
|
||||||
|
|
||||||
class SpanishNormalization(BaseNormalization):
|
def normalize_spanish(raw_input_text: str) -> str:
|
||||||
|
text = convert_to_ascii(raw_input_text)
|
||||||
|
text = lowercase(text)
|
||||||
|
|
||||||
def normalize(self, raw_input_text: str) -> str:
|
# TODO: Expand abbreviations (Spanish)
|
||||||
return raw_input_text
|
# TODO: Expand numbers (Spanish)
|
||||||
|
|
||||||
|
text = collapse_whitespace(text)
|
||||||
|
|
||||||
|
return text
|
||||||
|
|||||||
@@ -1,13 +1,11 @@
|
|||||||
from tts.text.pipeline_components.normalization.english_normalization import EnglishNormalization
|
from tts.text.pipeline_components.normalization.english_normalization import normalize_english
|
||||||
|
|
||||||
class TestEnglishNormalization:
|
class TestEnglishNormalization:
|
||||||
|
|
||||||
def test_whitespace_collapse(self):
|
def test_whitespace_collapse(self):
|
||||||
normalizer = EnglishNormalization()
|
output = normalize_english(" \t \n normalized \n \t ")
|
||||||
output = normalizer.normalize(" \t \n normalized \n \t ")
|
|
||||||
assert output == " normalized "
|
assert output == " normalized "
|
||||||
|
|
||||||
def test_lowercase(self):
|
def test_lowercase(self):
|
||||||
normalizer = EnglishNormalization()
|
output = normalize_english("This is a test of Normalization.")
|
||||||
output = normalizer.normalize("This is a test of Normalization.")
|
|
||||||
assert output == "this is a test of normalization."
|
assert output == "this is a test of normalization."
|
||||||
|
|||||||
Reference in New Issue
Block a user