Introduce code from Rust monorepo. This is a mess

tts/nemo/collections/common/tokenizers/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer
+from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer
+from nemo.collections.common.tokenizers.sentencepiece_tokenizer import SentencePieceTokenizer
+from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
+from nemo.collections.common.tokenizers.word_tokenizer import WordTokenizer

tts/nemo/collections/common/tokenizers/char_tokenizer.py

@@ -0,0 +1,142 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
+
+__all__ = ['CharTokenizer']
+
+
+class CharTokenizer(TokenizerSpec):
+    "Tokenizes each character"
+
+    def __init__(
+        self,
+        vocab_file: str,
+        mask_token: Optional[str] = None,
+        bos_token: Optional[str] = None,
+        eos_token: Optional[str] = None,
+        pad_token: Optional[str] = None,
+        sep_token: Optional[str] = None,
+        cls_token: Optional[str] = None,
+        unk_token: Optional[str] = None,
+    ):
+        """
+        Args:
+            vocab_file: path to file with vocabulary which consists
+                of characters separated by \n
+            mask_token: mask token 
+            bos_token: the beginning of sequence token
+            eos_token: the end of sequence token. Usually equal to sep_token
+            pad_token: token to use for padding
+            sep_token: token used for separating sequences
+            cls_token: class token. Usually equal to bos_token
+            unk_token: token to use for unknown tokens
+        """
+
+        vocab_list = open(vocab_file, "r").readlines()
+        self.vocab = {vocab_list[i].strip(): i for i in range(len(vocab_list))}
+
+        special_tokens_dict = {}
+        if unk_token:
+            special_tokens_dict["unk_token"] = unk_token
+        if sep_token:
+            special_tokens_dict["sep_token"] = sep_token
+        if mask_token:
+            special_tokens_dict["mask_token"] = mask_token
+        if bos_token:
+            special_tokens_dict["bos_token"] = bos_token
+        if eos_token:
+            special_tokens_dict["eos_token"] = eos_token
+        if pad_token:
+            special_tokens_dict["pad_token"] = pad_token
+        if cls_token:
+            special_tokens_dict["cls_token"] = cls_token
+
+        self.add_special_tokens(special_tokens_dict)
+        self.inv_vocab = {v: k for k, v in self.vocab.items()}
+        self.vocab_size = len(self.vocab)
+        self.special_tokens = self.tokens_to_ids(special_tokens_dict.values())
+
+    def add_special_tokens(self, special_tokens_dict: dict) -> int:
+        """
+        Adds a dictionary of special tokens (eos, pad, cls...).
+        If special tokens are NOT in the vocabulary, they are added
+        to it (indexed starting from the last index of the current vocabulary).
+        Args:
+            special_tokens_dict: dict of special tokens
+        """
+        for token in special_tokens_dict:
+            token_str = special_tokens_dict[token]
+            if token_str not in self.vocab:
+                self.vocab[token_str] = len(self.vocab)
+            setattr(self, token, token_str)
+
+    def text_to_tokens(self, text):
+        token_candidates = [char for char in text]
+        tokens = []
+        for token in token_candidates:
+            if token in self.vocab:
+                tokens.append(token)
+            else:
+                tokens.append(self.unk_token)
+        return tokens
+
+    def tokens_to_text(self, tokens):
+        return self.ids_to_text(self.tokens_to_ids(tokens))
+
+    def text_to_ids(self, text):
+        return [self.vocab[token] for token in self.text_to_tokens(text)]
+
+    def ids_to_text(self, ids):
+        ids_ = [id_ for id_ in ids if id_ not in self.special_tokens]
+        return "".join(self.ids_to_tokens(ids_))
+
+    def tokens_to_ids(self, tokens):
+        return [self.vocab[token] for token in tokens]
+
+    def token_to_id(self, token):
+        return self.vocab[token]
+
+    def ids_to_tokens(self, ids):
+        return [self.inv_vocab[id] for id in ids]
+
+    @property
+    def pad_id(self):
+        return self.vocab[self.pad_token]
+
+    @property
+    def bos_id(self):
+        return self.vocab[self.bos_token]
+
+    @property
+    def eos_id(self):
+        return self.vocab[self.eos_token]
+
+    @property
+    def unk_id(self):
+        return self.vocab[self.unk_token]
+
+    @property
+    def mask_id(self):
+        return self.vocab[self.mask_token]
+
+    @property
+    def sep_id(self):
+        return self.vocab[self.sep_token]
+
+    @property
+    def cls_id(self):
+        return self.vocab[self.cls_token]

tts/nemo/collections/common/tokenizers/chinese_tokenizers.py

@@ -0,0 +1,63 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# The MIT License (MIT)
+# Copyright (c) 2016 The-Orizon
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+
+# The detokenize function is based on : https://github.com/The-Orizon/nlputils/blob/master/detokenizer.py
+
+import re
+from typing import List
+
+import jieba
+import opencc
+from pangu import spacing
+
+
+class ChineseProcessor:
+    """
+    Tokenizer, Detokenizer and Normalizer utilities for Chinese.
+    """
+
+    def __init__(self):
+        self.normalizer = opencc.OpenCC('t2s.json')
+
+    def normalize(self, text: str) -> str:
+        return self.normalizer.convert(text)
+
+    def detokenize(self, text: List[str]) -> str:
+        RE_WS_IN_FW = re.compile(
+            r'([\u2018\u2019\u201c\u201d\u2e80-\u312f\u3200-\u32ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef])\s+(?=[\u2018\u2019\u201c\u201d\u2e80-\u312f\u3200-\u32ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff00-\uffef])'
+        )
+
+        detokenize = lambda s: spacing(RE_WS_IN_FW.sub(r'\1', s)).strip()
+        return detokenize(' '.join(text))
+
+    def tokenize(self, text: str) -> str:
+        text = jieba.cut(text)
+        return ' '.join(text)

tts/nemo/collections/common/tokenizers/en_ja_tokenizers.py

@@ -0,0 +1,57 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import List
+
+from sacremoses import MosesDetokenizer, MosesPunctNormalizer, MosesTokenizer
+
+
+class EnJaProcessor:
+    """
+    Tokenizer, Detokenizer and Normalizer utilities for Japanese & English
+    Args:
+        lang_id: One of ['en', 'ja'].
+    """
+
+    def __init__(self, lang_id: str):
+        self.lang_id = lang_id
+        self.moses_tokenizer = MosesTokenizer(lang=lang_id)
+        self.moses_detokenizer = MosesDetokenizer(lang=lang_id)
+        self.normalizer = MosesPunctNormalizer(
+            lang=lang_id, pre_replace_unicode_punct=True, post_remove_control_chars=True
+        )
+
+    def detokenize(self, tokens: List[str]) -> str:
+        """
+        Detokenizes a list of tokens
+        Args:
+            tokens: list of strings as tokens
+        Returns:
+            detokenized Japanese or English string
+        """
+        return self.moses_detokenizer.detokenize(tokens)
+
+    def tokenize(self, text) -> str:
+        """
+        Tokenizes text using Moses. Returns a string of tokens.
+        """
+        tokens = self.moses_tokenizer.tokenize(text)
+        return ' '.join(tokens)
+
+    def normalize(self, text) -> str:
+        # Normalization doesn't handle Japanese periods correctly;
+        # '。'becomes '.'.
+        if self.lang_id == 'en':
+            return self.normalizer.normalize(text)
+        else:
+            return text

tts/nemo/collections/common/tokenizers/fairseq_tokenizer.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the LICENSE file in
+# the root directory of this source tree. An additional grant of patent rights
+# can be found in the PATENTS file in the same directory.
+
+""" Code from
+https://github.com/NVIDIA/DeepLearningExamples/blob/
+master/PyTorch/Translation/Transformer/fairseq/tokenizer.py
+"""
+
+import re
+import sys
+import unicodedata
+from collections import defaultdict
+
+__all__ = ['get_unicode_categories', 'tokenize_en']
+
+
+def get_unicode_categories():
+    cats = defaultdict(list)
+    for c in map(chr, range(sys.maxunicode + 1)):
+        cats[unicodedata.category(c)].append(c)
+    return cats
+
+
+NUMERICS = ''.join(get_unicode_categories()['No'])
+
+
+def tokenize_en(line):
+    line = line.strip()
+    line = ' ' + line + ' '
+    # remove ASCII junk
+    line = re.sub(r'\s+', ' ', line)
+    line = re.sub(r'[\x00-\x1F]', '', line)
+    # fix whitespaces
+    line = re.sub(r'\ +', ' ', line)
+    line = re.sub('^ ', '', line)
+    line = re.sub(' $', '', line)
+    # separate other special characters
+    line = re.sub(r'([^\s\.\'\`\,\-\w]|[_' + NUMERICS + '])', r' \g<1> ', line)
+    line = re.sub(r'(\w)\-(?=\w)', r'\g<1> @-@ ', line)
+
+    # multidots stay together
+    line = re.sub(r'\.([\.]+)', r' DOTMULTI\g<1>', line)
+    while re.search(r'DOTMULTI\.', line):
+        line = re.sub(r'DOTMULTI\.([^\.])', r'DOTDOTMULTI \g<1>', line)
+        line = re.sub(r'DOTMULTI\.', r'DOTDOTMULTI', line)
+
+    # separate out "," except if within numbers (5,300)
+    line = re.sub(r'([\D])[,]', r'\g<1> , ', line)
+    line = re.sub(r'[,]([\D])', r' , \g<1>', line)
+
+    # separate "," after a number if it's the end of sentence
+    line = re.sub(r'(\d)[,]$', r'\g<1> ,', line)
+
+    # split contractions right
+    line = re.sub(r'([\W\d])[\']([\W\d])', r'\g<1> \' \g<2>', line)
+    line = re.sub(r'(\W)[\']([\w\D])', r'\g<1> \' \g<2>', line)
+    line = re.sub(r'([\w\D])[\']([\W\d])', r'\g<1> \' \g<2>', line)
+    line = re.sub(r'([\w\D])[\']([\w\D])', r'\g<1> \'\g<2>', line)
+    # special case for "1990's"
+    line = re.sub(r'([\W\d])[\']([s])', r'\g<1> \'\g<2>', line)
+
+    # apply nonbreaking prefixes
+    words = line.split()
+    line = ''
+    for i in range(len(words)):
+        word = words[i]
+        match = re.search(r'^(\S+)\.$', word)
+        if match:
+            pre = match.group(1)
+            if i == len(words) - 1:
+                """split last words independently as they are unlikely
+                to be non-breaking prefixes"""
+                word = pre + ' .'
+            else:
+                word = pre + ' .'
+
+        word += ' '
+        line += word
+
+    # clean up extraneous spaces
+    line = re.sub(' +', ' ', line)
+    line = re.sub('^ ', '', line)
+    line = re.sub(' $', '', line)
+
+    # .' at end of sentence is missed
+    line = re.sub(r'\.\' ?$', ' . \' ', line)
+
+    # restore multi-dots
+    while re.search('DOTDOTMULTI', line):
+        line = re.sub('DOTDOTMULTI', 'DOTMULTI.', line)
+
+    line = re.sub('DOTMULTI', '.', line)
+
+    # escape special characters
+    line = re.sub(r'\&', r'&amp;', line)
+    line = re.sub(r'\|', r'&#124;', line)
+    line = re.sub(r'\<', r'&lt;', line)
+    line = re.sub(r'\>', r'&gt;', line)
+    line = re.sub(r'\'', r'&apos;', line)
+    line = re.sub(r'\"', r'&quot;', line)
+    line = re.sub(r'\[', r'&#91;', line)
+    line = re.sub(r'\]', r'&#93;', line)
+
+    # ensure final line breaks
+    # if line[-1] is not '\n':
+    #     line += '\n'
+
+    return line

tts/nemo/collections/common/tokenizers/huggingface/__init__.py

@@ -0,0 +1,15 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from nemo.collections.common.tokenizers.huggingface.auto_tokenizer import AutoTokenizer

tts/nemo/collections/common/tokenizers/huggingface/auto_tokenizer.py

@@ -0,0 +1,260 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+from typing import Optional
+
+from transformers import AutoTokenizer as AUTOTOKENIZER
+
+from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
+from nemo.utils import logging
+
+__all__ = [
+    'AutoTokenizer',
+]
+
+
+def handle_quotes(text):
+    text_ = ""
+    quote = 0
+    i = 0
+    while i < len(text):
+        if text[i] == "\"":
+            if quote % 2:
+                text_ = text_[:-1] + "\""
+            else:
+                text_ += "\""
+                i += 1
+            quote += 1
+        else:
+            text_ += text[i]
+        i += 1
+    return text_
+
+
+def remove_spaces(text):
+    text = text.replace("( ", "(")
+    text = text.replace(" )", ")")
+    text = text.replace("[ ", "[")
+    text = text.replace(" ]", "]")
+    text = text.replace(" / ", "/")
+    text = text.replace("„ ", "„")
+    text = text.replace(" - ", "-")
+    text = text.replace(" ' ", "'")
+    text = re.sub(r'([0-9])( )([\.,])', '\\1\\3', text)
+    text = re.sub(r'([\.,])( )([0-9])', '\\1\\3', text)
+    text = re.sub(r'([0-9])(:)( )([0-9])', '\\1\\2\\4', text)
+    text = text.replace(" %", "%")
+    text = text.replace("$ ", "$")
+    text = re.sub(r'([^0-9])(,)([0-9])', '\\1\\2 \\3', text)
+    return text
+
+
+class AutoTokenizer(TokenizerSpec):
+    '''
+        Wrapper of HuggingFace AutoTokenizer https://huggingface.co/transformers/model_doc/auto.html#autotokenizer.
+    '''
+
+    def __init__(
+        self,
+        pretrained_model_name: str,
+        vocab_file: Optional[str] = None,
+        mask_token: Optional[str] = None,
+        bos_token: Optional[str] = None,
+        eos_token: Optional[str] = None,
+        pad_token: Optional[str] = None,
+        sep_token: Optional[str] = None,
+        cls_token: Optional[str] = None,
+        unk_token: Optional[str] = None,
+        use_fast: Optional[bool] = False,
+    ):
+
+        """
+        Args:
+            pretrained_model_name: corresponds to HuggingFace-AutoTokenizer's 'pretrained_model_name_or_path' input argument. 
+                For more details please refer to https://huggingface.co/transformers/_modules/transformers/tokenization_auto.html#AutoTokenizer.from_pretrained. 
+                The list of all supported models can be found here: ALL_PRETRAINED_CONFIG_ARCHIVE_MAP
+            vocab_file: path to file with vocabulary which consists
+                of characters separated by '\n'.
+            mask_token: mask token 
+            bos_token: the beginning of sequence token
+            eos_token: the end of sequence token. Usually equal to sep_token
+            pad_token: token to use for padding
+            sep_token: token used for separating sequences
+            cls_token: class token. Usually equal to bos_token
+            unk_token: token to use for unknown tokens
+            use_fast: whether to use fast HuggingFace tokenizer
+        """
+        try:
+            if vocab_file is not None:
+                message = 'Using "slow" HuggingFace tokenizer'
+                if use_fast:
+                    message += f'{vocab_file} is ignored in "fast" tokenizers, using a "slow" version'
+                self.tokenizer = AUTOTOKENIZER.from_pretrained(
+                    pretrained_model_name_or_path=pretrained_model_name, vocab_file=vocab_file, use_fast=False
+                )
+            else:
+                self.tokenizer = AUTOTOKENIZER.from_pretrained(
+                    pretrained_model_name_or_path=pretrained_model_name, use_fast=use_fast
+                )
+        except Exception as e:
+            raise ValueError(f'{pretrained_model_name} is not supported by HuggingFace. {e}')
+
+        special_tokens_dict = {}
+
+        # # setting special tokens, by default the default model's special tokens will be preserved
+        # # unless passes new values to the special tokens
+        if unk_token is not None:
+            special_tokens_dict["unk_token"] = unk_token
+        if mask_token is not None:
+            special_tokens_dict["mask_token"] = mask_token
+        if pad_token is not None:
+            special_tokens_dict["pad_token"] = pad_token
+
+        if sep_token is not None:
+            special_tokens_dict["sep_token"] = sep_token
+        elif self.tokenizer.sep_token is None and self.tokenizer.eos_token:
+            special_tokens_dict["sep_token"] = self.tokenizer.eos_token
+        if bos_token is not None:
+            special_tokens_dict["bos_token"] = bos_token
+        elif self.tokenizer.bos_token is None and self.tokenizer.cls_token:
+            special_tokens_dict["bos_token"] = self.tokenizer.cls_token
+        if eos_token is not None:
+            special_tokens_dict["eos_token"] = eos_token
+        elif self.tokenizer.eos_token is None and self.tokenizer.sep_token:
+            special_tokens_dict["eos_token"] = self.tokenizer.sep_token
+        if cls_token is not None:
+            special_tokens_dict["cls_token"] = cls_token
+        elif self.tokenizer.cls_token is None and self.tokenizer.bos_token:
+            special_tokens_dict["cls_token"] = self.tokenizer.bos_token
+
+        new_tokens_in_vocab = []
+        for token in [mask_token, bos_token, eos_token, pad_token, sep_token, cls_token, unk_token]:
+            if token is not None and token not in self.tokenizer.get_vocab():
+                new_tokens_in_vocab.append(token)
+
+        if len(new_tokens_in_vocab) > 0:
+            """
+            Special tokens that were not previously included in the tokenizer's vocabulary file will be added to 
+            the vocabulary and, as a result, the model should be resized, for example:
+            
+            # define your model
+            pretrained_model_name = 'roberta-base'
+            model = nemo_nlp.modules.get_lm_model(pretrained_model_name=pretrained_model_name)
+            
+            # define pretrained tokenizer
+            tokenizer_default = nemo_nlp.modules.get_tokenizer(tokenizer_name=pretrained_model_name)
+            
+            special_tokens = {'bos_token': '<BOS>',
+                              'cls_token': '<CSL>',
+                              'additional_special_tokens': ['<MY_NER_TOKEN>', '<ANOTHER_TOKEN>']}
+            tokenizer_default.add_special_tokens(special_tokens_dict=special_tokens)
+            
+            # resize your model so that the embeddings for newly added tokens are updated during training/finetuning
+            model.resize_token_embeddings(tokenizer_default.vocab_size)
+            
+            See NLP_Tokenizers.ipynb for more details.
+            """
+            logging.warning(
+                f'{new_tokens_in_vocab} \n will be added to the vocabulary.\n'
+                f'Please resize your model accordingly, '
+                f'see NLP_Tokenizers.ipynb for more details.'
+            )
+        self.add_special_tokens(special_tokens_dict)
+
+    @property
+    def vocab_size(self):
+        return len(self.tokenizer)
+
+    def add_special_tokens(self, special_tokens_dict: dict) -> int:
+        """
+        Adds a dictionary of special tokens (eos, pad, cls...). If special tokens are NOT in the vocabulary, they are added
+        to it (indexed starting from the last index of the current vocabulary).
+        Args:
+            special_tokens_dict: dict of string. Keys should be in the list of predefined special attributes:
+                [``bos_token``, ``eos_token``, ``unk_token``, ``sep_token``, ``pad_token``, ``cls_token``, ``mask_token``,
+                ``additional_special_tokens``].
+            Tokens are only added if they are not already in the vocabulary.
+        Returns:
+            Number of tokens added to the vocabulary.
+        """
+        num_tokens_added = self.tokenizer.add_special_tokens(special_tokens_dict)
+
+        if num_tokens_added > 0:
+            logging.info(f'{num_tokens_added} special tokens added, resize your model accordingly.')
+        for k in self.tokenizer.SPECIAL_TOKENS_ATTRIBUTES:
+            setattr(self, k, getattr(self.tokenizer, k, None))
+        return num_tokens_added
+
+    def text_to_tokens(self, text):
+        tokens = self.tokenizer.tokenize(text)
+        return tokens
+
+    def tokens_to_text(self, tokens):
+        text = self.tokenizer.convert_tokens_to_string(tokens)
+        return remove_spaces(handle_quotes(text.strip()))
+
+    def token_to_id(self, token):
+        return self.tokens_to_ids([token])[0]
+
+    def tokens_to_ids(self, tokens):
+        ids = self.tokenizer.convert_tokens_to_ids(tokens)
+        return ids
+
+    def ids_to_tokens(self, ids):
+        tokens = self.tokenizer.convert_ids_to_tokens(ids)
+        return tokens
+
+    def text_to_ids(self, text):
+        tokens = self.text_to_tokens(text)
+        ids = self.tokens_to_ids(tokens)
+        return ids
+
+    def ids_to_text(self, ids):
+        tokens = self.ids_to_tokens(ids)
+        tokens_clean = [t for t in tokens if t not in self.tokenizer.all_special_tokens]
+        text = self.tokens_to_text(tokens_clean)
+        return text
+
+    @property
+    def pad_id(self):
+        return self.tokens_to_ids([getattr(self, 'pad_token')])[0]
+
+    @property
+    def bos_id(self):
+        return self.tokens_to_ids([getattr(self, 'bos_token')])[0]
+
+    @property
+    def eos_id(self):
+        return self.tokens_to_ids([getattr(self, 'eos_token')])[0]
+
+    @property
+    def sep_id(self):
+        return self.tokens_to_ids([getattr(self, 'sep_token')])[0]
+
+    @property
+    def cls_id(self):
+        return self.tokens_to_ids([getattr(self, 'cls_token')])[0]
+
+    @property
+    def unk_id(self):
+        return self.tokens_to_ids([getattr(self, 'unk_token')])[0]
+
+    @property
+    def mask_id(self):
+        return self.tokens_to_ids([getattr(self, 'mask_token')])[0]
+
+    @property
+    def name(self):
+        return type(self.tokenizer).__name__

tts/nemo/collections/common/tokenizers/moses_tokenizers.py

@@ -0,0 +1,47 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import List
+
+from sacremoses import MosesDetokenizer, MosesPunctNormalizer, MosesTokenizer
+
+
+class MosesProcessor:
+    """
+    Tokenizer, Detokenizer and Normalizer utilities in Moses
+    """
+
+    def __init__(self, lang_id: str):
+        self.moses_tokenizer = MosesTokenizer(lang=lang_id)
+        self.moses_detokenizer = MosesDetokenizer(lang=lang_id)
+        self.normalizer = MosesPunctNormalizer(lang=lang_id)
+
+    def detokenize(self, tokens: List[str]) -> str:
+        """
+        Detokenizes a list of tokens
+        Args:
+            tokens: list of strings as tokens
+        Returns:
+            detokenized string
+        """
+        return self.moses_detokenizer.detokenize(tokens)
+
+    def tokenize(self, text: str):
+        """
+        Tokenizes text using Moses -> Sentencepiece.
+        """
+        return self.moses_tokenizer.tokenize(text, escape=False, return_str=True)
+
+    def normalize(self, text: str):
+        return self.normalizer.normalize(text)

tts/nemo/collections/common/tokenizers/sentencepiece_tokenizer.py

@@ -0,0 +1,353 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+from typing import Dict, List, Optional, Union
+
+import numpy as np
+import sentencepiece
+
+from nemo.collections.common.parts.utils import if_exist
+from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
+from nemo.utils import logging
+
+__all__ = ['SentencePieceTokenizer', 'create_spt_model']
+
+
+class SentencePieceTokenizer(TokenizerSpec):
+    '''
+    Sentencepiecetokenizer https://github.com/google/sentencepiece.
+        Args:
+        model_path: path to sentence piece tokenizer model. To create the model use create_spt_model()
+        special_tokens: either list of special tokens or dictionary of token name to token value
+        legacy: when set to True, the previous behavior of the SentecePiece wrapper will be restored, 
+            including the possibility to add special tokens inside wrapper.
+    '''
+
+    def __init__(
+        self, model_path: str, special_tokens: Optional[Union[Dict[str, str], List[str]]] = None, legacy: bool = False
+    ):
+        if not model_path or not os.path.exists(model_path):
+            raise ValueError(f"model_path: {model_path} is invalid")
+        self.tokenizer = sentencepiece.SentencePieceProcessor()
+        self.tokenizer.Load(model_path)
+        self.original_vocab_size = self.tokenizer.get_piece_size()
+        self.vocab_size = self.tokenizer.get_piece_size()
+        self.legacy = legacy
+        self.special_token_to_id = {}
+        self.id_to_special_token = {}
+        if special_tokens:
+            if not self.legacy:
+                raise ValueError(
+                    "Special tokens must be None when legacy is set to False. Provide special tokens at train time."
+                )
+            self.add_special_tokens(special_tokens)
+
+    def text_to_tokens(self, text):
+        if self.legacy:
+            tokens = []
+            idx = 0
+            last_idx = 0
+
+            while 1:
+                indices = {}
+
+                for token in self.special_token_to_id:
+                    try:
+                        indices[token] = text[idx:].index(token)
+                    except ValueError:
+                        continue
+
+                if len(indices) == 0:
+                    break
+
+                next_token = min(indices, key=indices.get)
+                next_idx = idx + indices[next_token]
+
+                tokens.extend(self.tokenizer.encode_as_pieces(text[idx:next_idx]))
+                tokens.append(next_token)
+                idx = next_idx + len(next_token)
+
+            tokens.extend(self.tokenizer.encode_as_pieces(text[idx:]))
+            return tokens
+
+        return self.tokenizer.encode_as_pieces(text)
+
+    def text_to_ids(self, text):
+        if self.legacy:
+            ids = []
+            idx = 0
+            last_idx = 0
+
+            while 1:
+                indices = {}
+
+                for token in self.special_token_to_id:
+                    try:
+                        indices[token] = text[idx:].index(token)
+                    except ValueError:
+                        continue
+
+                if len(indices) == 0:
+                    break
+
+                next_token = min(indices, key=indices.get)
+                next_idx = idx + indices[next_token]
+
+                ids.extend(self.tokenizer.encode_as_ids(text[idx:next_idx]))
+                ids.append(self.special_token_to_id[next_token])
+                idx = next_idx + len(next_token)
+
+            ids.extend(self.tokenizer.encode_as_ids(text[idx:]))
+            return ids
+
+        return self.tokenizer.encode_as_ids(text)
+
+    def tokens_to_text(self, tokens):
+        if isinstance(tokens, np.ndarray):
+            tokens = tokens.tolist()
+
+        return self.tokenizer.decode_pieces(tokens)
+
+    def ids_to_text(self, ids):
+        if isinstance(ids, np.ndarray):
+            ids = ids.tolist()
+
+        if self.legacy:
+            text = ""
+            last_i = 0
+
+            for i, id in enumerate(ids):
+                if id in self.id_to_special_token:
+                    text += self.tokenizer.decode_ids(ids[last_i:i]) + " "
+                    text += self.id_to_special_token[id] + " "
+                    last_i = i + 1
+
+            text += self.tokenizer.decode_ids(ids[last_i:])
+            return text.strip()
+
+        return self.tokenizer.decode_ids(ids)
+
+    def token_to_id(self, token):
+        if self.legacy and token in self.special_token_to_id:
+            return self.special_token_to_id[token]
+
+        return self.tokenizer.piece_to_id(token)
+
+    def ids_to_tokens(self, ids):
+        tokens = []
+        for id in ids:
+            if id >= self.original_vocab_size:
+                tokens.append(self.id_to_special_token[id])
+            else:
+                tokens.append(self.tokenizer.id_to_piece(id))
+        return tokens
+
+    def tokens_to_ids(self, tokens: Union[str, List[str]]) -> Union[int, List[int]]:
+        if isinstance(tokens, str):
+            tokens = [tokens]
+        ids = []
+        for token in tokens:
+            ids.append(self.token_to_id(token))
+        return ids
+
+    def add_special_tokens(self, special_tokens):
+        if not self.legacy:
+            raise AttributeError("Special Token addition does not work when legacy is set to False.")
+
+        if isinstance(special_tokens, list):
+            for token in special_tokens:
+                if (
+                    self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id()
+                    and token not in self.special_token_to_id
+                ):
+                    self.special_token_to_id[token] = self.vocab_size
+                    self.id_to_special_token[self.vocab_size] = token
+                    self.vocab_size += 1
+        elif isinstance(special_tokens, dict):
+            for token_name, token in special_tokens.items():
+                setattr(self, token_name, token)
+                if (
+                    self.tokenizer.piece_to_id(token) == self.tokenizer.unk_id()
+                    and token not in self.special_token_to_id
+                ):
+                    self.special_token_to_id[token] = self.vocab_size
+                    self.id_to_special_token[self.vocab_size] = token
+                    self.vocab_size += 1
+
+    @property
+    def pad_id(self):
+        if self.legacy:
+            pad_id = self.tokens_to_ids([self.pad_token])[0]
+        else:
+            pad_id = self.tokenizer.pad_id()
+        return pad_id
+
+    @property
+    def bos_id(self):
+        if self.legacy:
+            bos_id = self.tokens_to_ids([self.bos_token])[0]
+        else:
+            bos_id = self.tokenizer.bos_id()
+        return bos_id
+
+    @property
+    def eos_id(self):
+        if self.legacy:
+            eos_id = self.tokens_to_ids([self.eos_token])[0]
+        else:
+            eos_id = self.tokenizer.eos_id()
+        return eos_id
+
+    @property
+    def sep_id(self):
+        if self.legacy:
+            return self.tokens_to_ids([self.sep_token])[0]
+        else:
+            raise NameError("Use function token_to_id to retrieve special tokens other than unk, pad, bos, and eos.")
+
+    @property
+    def cls_id(self):
+        if self.legacy:
+            return self.tokens_to_ids([self.cls_token])[0]
+        else:
+            raise NameError("Use function token_to_id to retrieve special tokens other than unk, pad, bos, and eos.")
+
+    @property
+    def unk_id(self):
+        return self.tokenizer.unk_id()
+
+
+def create_spt_model(
+    data_file: str,
+    vocab_size: int,
+    sample_size: int,
+    do_lower_case: bool,
+    tokenizer_type: str = 'unigram',
+    output_dir: Optional[str] = None,
+    character_coverage: float = 1.0,
+    train_extremely_large_corpus: bool = False,
+    max_sentencepiece_length: int = -1,
+    bos: bool = False,
+    eos: bool = False,
+    pad: bool = False,
+    control_symbols: List[str] = None,
+    user_defined_symbols: List[str] = None,
+):
+    """
+    Creates sentence piece tokenizer model from data file.
+    Args:
+        data_file: data file
+        vocab_size: vocabulary size
+        sample_size: maximum size of sentences the trainer loads
+        do_lower_case: if text should be lower cased before tokenizer model is created
+        character_coverage: float value between 0 and 1 (as a percentage). For languages with a vast charset,
+            can be < 1.0, but for all other languages, it should be set as 1.0
+        output_dir: folder to save created tokenizer model. If not specified will store model at data_file/../spt folder
+        train_extremely_large_corpus: If training on huge datasets, pass this flag to allow SentencePiece
+            to build the tokenizer.
+        max_sentencepiece_length: Limits the maximum length of the SentencePiece subword that can be constructed.
+            By default, no limit is placed.
+        bos: when True, bos token "<s>" is added to the vocabulary.
+        eos: when True, eos token "</s>" is added to the vocabulary.
+        pad: when True, pad token "<pad>" is added to the vocabulary.
+        control_symbols: control symbols to add to tokenizer, as defined by sentencepiece.
+            These tokens get removed at decode time and are not encoded from the text - can only be added to the input programatically.
+        user_defined_symbols: user symbols to add to tokenizer, as defined by sentencepiece.
+            These tokens remain in the decoded text and are encoded automatically when present in the input text.
+    """
+
+    if not data_file or not os.path.exists(data_file):
+        raise ValueError(f"data_file must be valid file path, but got {data_file}")
+    data_dir = os.path.dirname(data_file)
+    vocab = []
+    special_tokens = ["<s>", "</s>", "<pad>", "<unk>"]
+    if not output_dir:
+        output_dir = f'{data_dir}/spt'
+    if if_exist(output_dir, ['tokenizer.model']):
+        logging.info(f"tokenizer model {output_dir}/tokenizer.model already exists")
+        return f'{output_dir}/tokenizer.model', f'{output_dir}/vocab.txt'
+    logging.info(f'Processing {data_file} and store at {output_dir}')
+    os.makedirs(output_dir, exist_ok=True)
+
+    cmd = (
+        f"--input={data_file} --model_prefix={output_dir}/tokenizer "
+        f"--vocab_size={vocab_size} "
+        f"--shuffle_input_sentence=true --hard_vocab_limit=false "
+        f"--model_type={tokenizer_type} "
+        f"--character_coverage={character_coverage}"
+    )
+
+    pad_id = 3
+    if not bos:
+        pad_id -= 1
+        cmd += " --bos_id=-1"
+
+    if not eos:
+        pad_id -= 1
+        cmd += " --eos_id=-1"
+
+    if pad:
+        cmd += f" --pad_id={pad_id}"
+
+    if control_symbols:
+        control_string = (",").join(control_symbols)
+        cmd += f" --control_symbols={control_string}"
+        special_tokens += control_symbols
+
+    if user_defined_symbols:
+        user_string = (",").join(user_defined_symbols)
+        cmd += f" --user_defined_symbols={user_string}"
+        special_tokens += user_defined_symbols
+
+    if do_lower_case:
+        cmd += " --normalization_rule_name=nmt_nfkc_cf"
+
+    if sample_size > 0:
+        cmd += f" --input_sentence_size={sample_size}"
+
+    if train_extremely_large_corpus:
+        cmd += " --train_extremely_large_corpus=true"
+
+    if max_sentencepiece_length >= 0:
+        cmd += f" --max_sentencepiece_length={max_sentencepiece_length}"
+
+    sentencepiece.SentencePieceTrainer.Train(cmd)
+
+    # Add BERT control symbols
+    tokens = []
+
+    with open(f"{output_dir}/tokenizer.vocab", "r") as f:
+        # Read tokens from each line and parse for vocab
+        for line in f:
+            piece = line.split("\t")[0]
+            if piece in special_tokens:
+                # skip special tokens
+                continue
+            token = piece[1:] if piece.startswith("▁") else f"##{piece}"
+
+            if len(token) > 0:
+                tokens.append(token)
+            else:
+                tokens.append(piece[0])
+
+    vocab.extend(tokens)
+
+    # Save vocabulary to output file
+    vocab_file = f'{output_dir}/vocab.txt'
+    with open(vocab_file, "w") as f:
+        for token in vocab:
+            f.write(f"{token}\n")
+    return f'{output_dir}/tokenizer.model', vocab_file

tts/nemo/collections/common/tokenizers/tokenizer_spec.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import List
+
+__all__ = ['TokenizerSpec']
+
+
+class TokenizerSpec(ABC):
+    """
+    Inherit this class to implement a new tokenizer.
+    """
+
+    @abstractmethod
+    def text_to_tokens(self, text):
+        pass
+
+    @abstractmethod
+    def tokens_to_text(self, tokens):
+        pass
+
+    @abstractmethod
+    def tokens_to_ids(self, tokens):
+        pass
+
+    @abstractmethod
+    def ids_to_tokens(self, ids):
+        pass
+
+    @abstractmethod
+    def text_to_ids(self, text):
+        pass
+
+    @abstractmethod
+    def ids_to_text(self, ids):
+        pass
+
+    def add_special_tokens(self, special_tokens: List[str]):
+        raise NotImplementedError("To be implemented")
+
+    @property
+    def name(self):
+        return type(self).__name__

tts/nemo/collections/common/tokenizers/word_tokenizer.py

@@ -0,0 +1,72 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Optional
+
+from nemo.collections.common.tokenizers.char_tokenizer import CharTokenizer
+
+__all__ = ['WordTokenizer']
+
+
+class WordTokenizer(CharTokenizer):
+    "Tokenizes at word boundary"
+
+    def __init__(
+        self,
+        vocab_file: str,
+        mask_token: Optional[str] = None,
+        bos_token: Optional[str] = None,
+        eos_token: Optional[str] = None,
+        pad_token: Optional[str] = None,
+        sep_token: Optional[str] = None,
+        cls_token: Optional[str] = None,
+        unk_token: Optional[str] = None,
+    ):
+        """
+        Args:
+            vocab_file: path to file with vocabulary which consists
+                of characters separated by \n
+            mask_token: mask token 
+            bos_token: the beginning of sequence token
+            eos_token: the end of sequence token. Usually equal to sep_token
+            pad_token: token to use for padding
+            sep_token: token used for separating sequences
+            cls_token: class token. Usually equal to bos_token
+            unk_token: token to use for unknown tokens
+        """
+
+        super().__init__(
+            vocab_file=vocab_file,
+            mask_token=mask_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            unk_token=unk_token,
+            sep_token=sep_token,
+            cls_token=cls_token,
+        )
+
+    def text_to_tokens(self, text):
+        token_candidates = text.strip().split()
+        tokens = []
+        for token in token_candidates:
+            if token in self.vocab:
+                tokens.append(token)
+            else:
+                tokens.append(self.unk_token)
+        return tokens
+
+    def ids_to_text(self, ids):
+        ids_ = [id_ for id_ in ids if id_ not in self.special_tokens]
+        return " ".join(self.ids_to_tokens(ids_))

tts/nemo/collections/common/tokenizers/youtokentome_tokenizer.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from pathlib import Path
+
+import youtokentome as yttm
+
+from nemo.collections.common.tokenizers import TokenizerSpec
+
+__all__ = ['YouTokenToMeTokenizer']
+
+
+class YouTokenToMeTokenizer(TokenizerSpec):
+    def __init__(self, model_path, bpe_dropout=0.0, legacy=False):
+        model_path = Path(model_path).expanduser()
+        self.tokenizer = yttm.BPE(model=str(model_path))
+        self.vocab_size = len(self.tokenizer.vocab())
+        self.special_tokens = self.tokens_to_ids(["<PAD>", "<UNK>", "<BOS>", "<EOS>"])
+        self.bpe_dropout = bpe_dropout
+        self.legacy = legacy
+
+    def text_to_tokens(self, text):
+        return self.tokenizer.encode(text, output_type=yttm.OutputType.SUBWORD, dropout_prob=self.bpe_dropout)
+
+    def tokens_to_text(self, tokens):
+        return self.ids_to_text(self.tokens_to_ids(tokens))
+
+    def text_to_ids(self, text):
+        return self.tokenizer.encode(text, output_type=yttm.OutputType.ID, dropout_prob=self.bpe_dropout)
+
+    def ids_to_text(self, ids):
+        ids_ = [id_ for id_ in ids if id_ not in self.special_tokens]
+        return self.tokenizer.decode([ids_])[0]
+
+    def tokens_to_ids(self, tokens):
+        return [self.tokenizer.subword_to_id(token) for token in tokens]
+
+    def ids_to_tokens(self, ids):
+        if self.legacy:
+            ids_ = [id_ for id_ in ids if id_ not in self.special_tokens]
+        else:
+            ids_ = ids
+        return [self.tokenizer.id_to_subword(id_) for id_ in ids_]
+
+    @property
+    def pad_id(self):
+        return self.tokenizer.subword_to_id("<PAD>")
+
+    @property
+    def bos_id(self):
+        return self.tokenizer.subword_to_id("<BOS>")
+
+    @property
+    def eos_id(self):
+        return self.tokenizer.subword_to_id("<EOS>")
+
+    @property
+    def unk_id(self):
+        return self.tokenizer.subword_to_id("<UNK>")