mirror of
https://github.com/storytold/storyteller-ml.git
synced 2026-10-16 00:13:07 +00:00
Michael Chung
62 lines
2.3 KiB
Python
62 lines
2.3 KiB
Python
import re
|
|
from utils.g2p.japanese import japanese_to_romaji_with_accent, japanese_to_ipa, japanese_to_ipa2, japanese_to_ipa3
|
|
from utils.g2p.mandarin import number_to_chinese, chinese_to_bopomofo, latin_to_bopomofo, chinese_to_romaji, chinese_to_lazy_ipa, chinese_to_ipa, chinese_to_ipa2
|
|
from utils.g2p.english import english_to_lazy_ipa, english_to_ipa2, english_to_lazy_ipa2
|
|
patterns = [r'\[EN\](.*?)\[EN\]', r'\[ZH\](.*?)\[ZH\]', r'\[JA\](.*?)\[JA\]']
|
|
def japanese_cleaners(text):
|
|
text = japanese_to_romaji_with_accent(text)
|
|
text = re.sub(r'([A-Za-z])$', r'\1.', text)
|
|
return text
|
|
|
|
def japanese_cleaners2(text):
|
|
return japanese_cleaners(text).replace('ts', 'ʦ').replace('...', '…')
|
|
|
|
def chinese_cleaners(text):
|
|
'''Pipeline for Chinese text'''
|
|
text = number_to_chinese(text)
|
|
text = chinese_to_bopomofo(text)
|
|
text = latin_to_bopomofo(text)
|
|
text = re.sub(r'([ˉˊˇˋ˙])$', r'\1。', text)
|
|
return text
|
|
|
|
def cje_cleaners(text):
|
|
matches = []
|
|
for pattern in patterns:
|
|
matches.extend(re.finditer(pattern, text))
|
|
|
|
matches.sort(key=lambda x: x.start()) # Sort matches by their start positions
|
|
|
|
outputs = ""
|
|
output_langs = []
|
|
|
|
for match in matches:
|
|
text_segment = text[match.start():match.end()]
|
|
phon = clean_one(text_segment)
|
|
if "[EN]" in text_segment:
|
|
lang = 'en'
|
|
elif "[ZH]" in text_segment:
|
|
lang = 'zh'
|
|
elif "[JA]" in text_segment:
|
|
lang = 'ja'
|
|
else:
|
|
raise ValueError("If you see this error, please report this bug to issues.")
|
|
outputs += phon
|
|
output_langs += [lang] * len(phon)
|
|
assert len(outputs) == len(output_langs)
|
|
return outputs, output_langs
|
|
|
|
|
|
def clean_one(text):
|
|
if text.find('[ZH]') != -1:
|
|
text = re.sub(r'\[ZH\](.*?)\[ZH\]',
|
|
lambda x: chinese_to_ipa(x.group(1))+' ', text)
|
|
if text.find('[JA]') != -1:
|
|
text = re.sub(r'\[JA\](.*?)\[JA\]',
|
|
lambda x: japanese_to_ipa2(x.group(1))+' ', text)
|
|
if text.find('[EN]') != -1:
|
|
text = re.sub(r'\[EN\](.*?)\[EN\]',
|
|
lambda x: english_to_ipa2(x.group(1))+' ', text)
|
|
text = re.sub(r'\s+$', '', text)
|
|
text = re.sub(r'([^\.,!\?\-…~])$', r'\1.', text)
|
|
return text
|