Introduce code from Rust monorepo. This is a mess

tts/nemo/collections/common/tokenizers/tokenizer_spec.py

@@ -0,0 +1,55 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import ABC, abstractmethod
+from typing import List
+
+__all__ = ['TokenizerSpec']
+
+
+class TokenizerSpec(ABC):
+    """
+    Inherit this class to implement a new tokenizer.
+    """
+
+    @abstractmethod
+    def text_to_tokens(self, text):
+        pass
+
+    @abstractmethod
+    def tokens_to_text(self, tokens):
+        pass
+
+    @abstractmethod
+    def tokens_to_ids(self, tokens):
+        pass
+
+    @abstractmethod
+    def ids_to_tokens(self, ids):
+        pass
+
+    @abstractmethod
+    def text_to_ids(self, text):
+        pass
+
+    @abstractmethod
+    def ids_to_text(self, ids):
+        pass
+
+    def add_special_tokens(self, special_tokens: List[str]):
+        raise NotImplementedError("To be implemented")
+
+    @property
+    def name(self):
+        return type(self).__name__