Entirely reworked text handling pipeline

This breaks the text pipeline into the following pieces:

- input parsing (split known bracketed phoneme sections out first)

- normalization (fix spacing, numbers, abbreviations, etc.)

- phonetization (do CMUdict lookup or g2p prediction)

- sequencing (convert the symbolic representation to integer
  representation)

Each "pipeline" can have a unique combination of operations for each of
these pipeline steps.

This will grow over time.
This commit is contained in:
Brandon Thomas
2022-06-28 03:25:02 -04:00
parent 6a43c8d7ca
commit 76c6dbf12b
33 changed files with 135390 additions and 1 deletions
@@ -0,0 +1,72 @@
"""
Parse raw text sequences into Arpabet and Grapheme sequences so that they can be
treated independently.
Inspired by and including code from original Tacotron2 by Keith Ito.
"""
# pyright: strict
import re
from typing import List, Union, Any
# Regular expression matching text enclosed in curly braces:
_curly_re = re.compile(r'(.*?)\{(.+?)\}(.*)')
class ArpabetSequence:
"""
Denotes an Arpabet phoneme sequence corresponding to a single word.
"""
def __init__(self, sequence: str):
self.sequence = sequence
def __eq__(self, other: Any) -> bool:
return isinstance(other, ArpabetSequence) and self.sequence == other.sequence
def __repr__(self):
return f"<ArpabetSequence: {self.sequence}>"
class GraphemeSequence:
"""
Denotes any string of non-Arpabet text, which may include multiple words,
spacing, and punctuation characters.
"""
def __init__(self, sequence: str):
self.sequence = sequence
def __eq__(self, other: Any) -> bool:
return isinstance(other, GraphemeSequence) and self.sequence == other.sequence
def __repr__(self):
return f"<GraphemeSequence: {self.sequence}>"
ParsedSequence = List[Union[ArpabetSequence, GraphemeSequence]]
def parse_grapheme_and_arpabet_sequence(input_text: str) -> ParsedSequence:
"""
Parse a string into a list of ArpabetSequence and GraphemeSequence so that each
subsequence can be treated independently.
"""
sequence : ParsedSequence = []
# Check for curly braces and treat their contents as ARPAbet:
while len(input_text):
input_text = input_text.strip()
m = _curly_re.match(input_text)
if not m:
sequence.append(GraphemeSequence(input_text))
break
before = m.group(1).strip()
bracketed = m.group(2).strip()
after = m.group(3).strip()
if before:
sequence.append(GraphemeSequence(before))
if bracketed:
sequence.append(ArpabetSequence(bracketed))
input_text = after
return sequence