Introduce code from Rust monorepo. This is a mess

tts/nemo/collections/common/parts/preprocessing/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tts/nemo/collections/common/parts/preprocessing/cleaners.py

@@ -0,0 +1,259 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+import inflect
+from unidecode import unidecode
+
+from nemo.utils import logging
+
+NUM_CHECK = re.compile(r'([$]?)(^|\s)(\S*[0-9]\S*)(?=(\s|$)((\S*)(\s|$))?)')
+
+TIME_CHECK = re.compile(r'([0-9]{1,2}):([0-9]{2})(am|pm)?')
+CURRENCY_CHECK = re.compile(r'\$')
+ORD_CHECK = re.compile(r'([0-9]+)(st|nd|rd|th)')
+THREE_CHECK = re.compile(r'([0-9]{3})([.,][0-9]{1,2})?([!.?])?$')
+DECIMAL_CHECK = re.compile(r'([.,][0-9]{1,2})$')
+
+ABBREVIATIONS_COMMON = [
+    (re.compile('\\b%s\\.' % x[0]), x[1])
+    for x in [
+        ("ms", "miss"),
+        ("mrs", "misess"),
+        ("mr", "mister"),
+        ("messrs", "messeurs"),
+        ("dr", "doctor"),
+        ("drs", "doctors"),
+        ("st", "saint"),
+        ("co", "company"),
+        ("jr", "junior"),
+        ("sr", "senior"),
+        ("rev", "reverend"),
+        ("hon", "honorable"),
+        ("sgt", "sergeant"),
+        ("capt", "captain"),
+        ("maj", "major"),
+        ("col", "colonel"),
+        ("lt", "lieutenant"),
+        ("gen", "general"),
+        ("prof", "professor"),
+        ("lb", "pounds"),
+        ("rep", "representative"),
+        ("st", "street"),
+        ("ave", "avenue"),
+        ("etc", "et cetera"),
+        ("jan", "january"),
+        ("feb", "february"),
+        ("mar", "march"),
+        ("apr", "april"),
+        ("jun", "june"),
+        ("jul", "july"),
+        ("aug", "august"),
+        ("sep", "september"),
+        ("oct", "october"),
+        ("nov", "november"),
+        ("dec", "december"),
+    ]
+]
+
+ABBREVIATIONS_EXPANDED = [
+    (re.compile('\\b%s\\.' % x[0]), x[1])
+    for x in [
+        ("ltd", "limited"),
+        ("fig", "figure"),
+        ("figs", "figures"),
+        ("gent", "gentlemen"),
+        ("ft", "fort"),
+        ("esq", "esquire"),
+        ("prep", "preperation"),
+        ("bros", "brothers"),
+        ("ind", "independent"),
+        ("mme", "madame"),
+        ("pro", "professional"),
+        ("vs", "versus"),
+        ("inc", "include"),
+    ]
+]
+
+ABBREVIATIONS_TTS_FASTPITCH = [
+    (re.compile('\\b%s\\.' % x[0]), x[1])
+    for x in [
+        ("ms", "miss"),
+        ("mrs", "misess"),
+        ("mr", "mister"),
+        ("dr", "doctor"),
+        ("drs", "doctors"),
+        ("st", "saint"),
+        ("co", "company"),
+        ("jr", "junior"),
+        ("sr", "senior"),
+        ("rev", "reverend"),
+        ("hon", "honorable"),
+        ("sgt", "sergeant"),
+        ("capt", "captain"),
+        ("maj", "major"),
+        ("col", "colonel"),
+        ("lt", "lieutenant"),
+        ("gen", "general"),
+        ("prof", "professor"),
+        ("lb", "pounds"),
+        ("rep", "representative"),
+        ("st", "street"),
+        ("ave", "avenue"),
+        ("jan", "january"),
+        ("feb", "february"),
+        ("mar", "march"),
+        ("apr", "april"),
+        ("jun", "june"),
+        ("jul", "july"),
+        ("aug", "august"),
+        ("sep", "september"),
+        ("oct", "october"),
+        ("nov", "november"),
+        ("dec", "december"),
+        ("ltd", "limited"),
+        ("fig", "figure"),
+        ("figs", "figures"),
+        ("gent", "gentlemen"),
+        ("ft", "fort"),
+        ("esq", "esquire"),
+        ("prep", "preperation"),
+        ("bros", "brothers"),
+        ("ind", "independent"),
+        ("mme", "madame"),
+        ("pro", "professional"),
+        ("vs", "versus"),
+    ]
+]
+
+
+inflect = inflect.engine()
+
+
+def clean_text(string, table, punctuation_to_replace, abbreviation_version=None):
+    warn_common_chars(string)
+    string = unidecode(string)
+    string = string.lower()
+    string = re.sub(r'\s+', " ", string)
+    string = clean_numbers(string)
+    string = clean_abbreviations(string, version=abbreviation_version)
+    string = clean_punctuations(string, table, punctuation_to_replace)
+    string = re.sub(r'\s+', " ", string).strip()
+    return string
+
+
+def warn_common_chars(string):
+    if re.search(r'[£€]', string):
+        logging.warning("Your transcript contains one of '£' or '€' which we do not currently handle")
+
+
+def clean_numbers(string):
+    cleaner = NumberCleaner()
+    string = NUM_CHECK.sub(cleaner.clean, string)
+    return string
+
+
+def clean_abbreviations(string, version=None):
+    abbbreviations = ABBREVIATIONS_COMMON
+    if version == "fastpitch":
+        abbbreviations = ABBREVIATIONS_TTS_FASTPITCH
+    elif version == "expanded":
+        abbbreviations.extend = ABBREVIATIONS_EXPANDED
+    for regex, replacement in abbbreviations:
+        string = re.sub(regex, replacement, string)
+    return string
+
+
+def clean_punctuations(string, table, punctuation_to_replace):
+    for punc, replacement in punctuation_to_replace.items():
+        string = re.sub('\\{}'.format(punc), " {} ".format(replacement), string)
+    if table:
+        string = string.translate(table)
+    return string
+
+
+class NumberCleaner:
+    def __init__(self):
+        super().__init__()
+        self.reset()
+
+    def reset(self):
+        self.curr_num = []
+        self.currency = None
+
+    def format_final_number(self, whole_num, decimal):
+        if self.currency:
+            return_string = inflect.number_to_words(whole_num)
+            return_string += " dollar" if whole_num == 1 else " dollars"
+            if decimal:
+                return_string += " and " + inflect.number_to_words(decimal)
+                return_string += " cent" if whole_num == decimal else " cents"
+            self.reset()
+            return return_string
+
+        self.reset()
+        if decimal:
+            whole_num += "." + decimal
+            return inflect.number_to_words(whole_num)
+        else:
+            # Check if there are non-numbers
+            def convert_to_word(match):
+                return " " + inflect.number_to_words(match.group(0)) + " "
+
+            return re.sub(r'[0-9,]+', convert_to_word, whole_num)
+
+    def clean(self, match):
+        ws = match.group(2)
+        number = match.group(3)
+        _proceeding_symbol = match.group(7)
+
+        time_match = TIME_CHECK.match(number)
+        if time_match:
+            string = ws + inflect.number_to_words(time_match.group(1)) + "{}{}"
+            mins = int(time_match.group(2))
+            min_string = ""
+            if mins != 0:
+                min_string = " " + inflect.number_to_words(time_match.group(2))
+            ampm_string = ""
+            if time_match.group(3):
+                ampm_string = " " + time_match.group(3)
+            return string.format(min_string, ampm_string)
+
+        ord_match = ORD_CHECK.match(number)
+        if ORD_CHECK.match(number):
+            return ws + inflect.number_to_words(ord_match.group(0))
+
+        if self.currency is None:
+            # Check if it is a currency
+            self.currency = match.group(1) or CURRENCY_CHECK.match(number)
+
+        # Check to see if next symbol is a number
+        # If it is a number and it has 3 digits, then it is probably a
+        # continuation
+        three_match = THREE_CHECK.match(match.group(6))
+        if three_match:
+            self.curr_num.append(number)
+            return " "
+        # Else we can output
+        else:
+            # Check for decimals
+            whole_num = "".join(self.curr_num) + number
+            decimal = None
+            decimal_match = DECIMAL_CHECK.search(whole_num)
+            if decimal_match:
+                decimal = decimal_match.group(1)[1:]
+                whole_num = whole_num[: -len(decimal) - 1]
+            whole_num = re.sub(r'\.', '', whole_num)
+            return ws + self.format_final_number(whole_num, decimal)

tts/nemo/collections/common/parts/preprocessing/collections.py

@@ -0,0 +1,468 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+import json
+import os
+from typing import Any, Dict, List, Optional, Union
+
+import pandas as pd
+
+from nemo.collections.common.parts.preprocessing import manifest, parsers
+from nemo.utils import logging
+
+
+class _Collection(collections.UserList):
+    """List of parsed and preprocessed data."""
+
+    OUTPUT_TYPE = None  # Single element output type.
+
+
+class Text(_Collection):
+    """Simple list of preprocessed text entries, result in list of tokens."""
+
+    OUTPUT_TYPE = collections.namedtuple('TextEntity', 'tokens')
+
+    def __init__(self, texts: List[str], parser: parsers.CharParser):
+        """Instantiates text manifest and do the preprocessing step.
+
+        Args:
+            texts: List of raw texts strings.
+            parser: Instance of `CharParser` to convert string to tokens.
+        """
+
+        data, output_type = [], self.OUTPUT_TYPE
+        for text in texts:
+            tokens = parser(text)
+
+            if tokens is None:
+                logging.warning("Fail to parse '%s' text line.", text)
+                continue
+
+            data.append(output_type(tokens))
+
+        super().__init__(data)
+
+
+class FromFileText(Text):
+    """Another form of texts manifest with reading from file."""
+
+    def __init__(self, file: str, parser: parsers.CharParser):
+        """Instantiates text manifest and do the preprocessing step.
+
+        Args:
+            file: File path to read from.
+            parser: Instance of `CharParser` to convert string to tokens.
+        """
+
+        texts = self.__parse_texts(file)
+
+        super().__init__(texts, parser)
+
+    @staticmethod
+    def __parse_texts(file: str) -> List[str]:
+        if not os.path.exists(file):
+            raise ValueError('Provided texts file does not exists!')
+
+        _, ext = os.path.splitext(file)
+        if ext == '.csv':
+            texts = pd.read_csv(file)['transcript'].tolist()
+        elif ext == '.json':  # Not really a correct json.
+            texts = list(item['text'] for item in manifest.item_iter(file))
+        else:
+            with open(file, 'r') as f:
+                texts = f.readlines()
+
+        return texts
+
+
+class AudioText(_Collection):
+    """List of audio-transcript text correspondence with preprocessing."""
+
+    OUTPUT_TYPE = collections.namedtuple(
+        typename='AudioTextEntity', field_names='id audio_file duration text_tokens offset text_raw speaker orig_sr',
+    )
+
+    def __init__(
+        self,
+        ids: List[int],
+        audio_files: List[str],
+        durations: List[float],
+        texts: List[str],
+        offsets: List[str],
+        speakers: List[Optional[int]],
+        orig_sampling_rates: List[Optional[int]],
+        parser: parsers.CharParser,
+        min_duration: Optional[float] = None,
+        max_duration: Optional[float] = None,
+        max_number: Optional[int] = None,
+        do_sort_by_duration: bool = False,
+        index_by_file_id: bool = False,
+    ):
+        """Instantiates audio-text manifest with filters and preprocessing.
+
+        Args:
+            ids: List of examples positions.
+            audio_files: List of audio files.
+            durations: List of float durations.
+            texts: List of raw text transcripts.
+            offsets: List of duration offsets or None.
+            speakers: List of optional speakers ids.
+            orig_sampling_rates: List of original sampling rates of audio files.
+            parser: Instance of `CharParser` to convert string to tokens.
+            min_duration: Minimum duration to keep entry with (default: None).
+            max_duration: Maximum duration to keep entry with (default: None).
+            max_number: Maximum number of samples to collect.
+            do_sort_by_duration: True if sort samples list by duration. Not compatible with index_by_file_id.
+            index_by_file_id: If True, saves a mapping from filename base (ID) to index in data.
+        """
+
+        output_type = self.OUTPUT_TYPE
+        data, duration_filtered, num_filtered, total_duration = [], 0.0, 0, 0.0
+        if index_by_file_id:
+            self.mapping = {}
+
+        for id_, audio_file, duration, offset, text, speaker, orig_sr in zip(
+            ids, audio_files, durations, offsets, texts, speakers, orig_sampling_rates
+        ):
+            # Duration filters.
+            if min_duration is not None and duration < min_duration:
+                duration_filtered += duration
+                num_filtered += 1
+                continue
+
+            if max_duration is not None and duration > max_duration:
+                duration_filtered += duration
+                num_filtered += 1
+                continue
+
+            text_tokens = parser(text)
+            if text_tokens is None:
+                duration_filtered += duration
+                num_filtered += 1
+                continue
+
+            total_duration += duration
+
+            data.append(output_type(id_, audio_file, duration, text_tokens, offset, text, speaker, orig_sr))
+            if index_by_file_id:
+                file_id, _ = os.path.splitext(os.path.basename(audio_file))
+                self.mapping[file_id] = len(data) - 1
+
+            # Max number of entities filter.
+            if len(data) == max_number:
+                break
+
+        if do_sort_by_duration:
+            if index_by_file_id:
+                logging.warning("Tried to sort dataset by duration, but cannot since index_by_file_id is set.")
+            else:
+                data.sort(key=lambda entity: entity.duration)
+
+        logging.info("Dataset loaded with %d files totalling %.2f hours", len(data), total_duration / 3600)
+        logging.info("%d files were filtered totalling %.2f hours", num_filtered, duration_filtered / 3600)
+
+        super().__init__(data)
+
+
+class ASRAudioText(AudioText):
+    """`AudioText` collector from asr structured json files."""
+
+    def __init__(self, manifests_files: Union[str, List[str]], *args, **kwargs):
+        """Parse lists of audio files, durations and transcripts texts.
+
+        Args:
+            manifests_files: Either single string file or list of such -
+                manifests to yield items from.
+            *args: Args to pass to `AudioText` constructor.
+            **kwargs: Kwargs to pass to `AudioText` constructor.
+        """
+
+        ids, audio_files, durations, texts, offsets, speakers, orig_srs = [], [], [], [], [], [], []
+        for item in manifest.item_iter(manifests_files):
+            ids.append(item['id'])
+            audio_files.append(item['audio_file'])
+            durations.append(item['duration'])
+            texts.append(item['text'])
+            offsets.append(item['offset'])
+            speakers.append(item['speaker'])
+            orig_srs.append(item['orig_sr'])
+
+        super().__init__(ids, audio_files, durations, texts, offsets, speakers, orig_srs, *args, **kwargs)
+
+
+class SpeechLabel(_Collection):
+    """List of audio-label correspondence with preprocessing."""
+
+    OUTPUT_TYPE = collections.namedtuple(typename='SpeechLabelEntity', field_names='audio_file duration label offset',)
+
+    def __init__(
+        self,
+        audio_files: List[str],
+        durations: List[float],
+        labels: List[Union[int, str]],
+        offsets: List[Optional[float]],
+        min_duration: Optional[float] = None,
+        max_duration: Optional[float] = None,
+        max_number: Optional[int] = None,
+        do_sort_by_duration: bool = False,
+        index_by_file_id: bool = False,
+    ):
+        """Instantiates audio-label manifest with filters and preprocessing.
+
+        Args:
+            audio_files: List of audio files.
+            durations: List of float durations.
+            labels: List of labels.
+            offsets: List of offsets or None.
+            min_duration: Minimum duration to keep entry with (default: None).
+            max_duration: Maximum duration to keep entry with (default: None).
+            max_number: Maximum number of samples to collect.
+            do_sort_by_duration: True if sort samples list by duration.
+            index_by_file_id: If True, saves a mapping from filename base (ID) to index in data.
+        """
+
+        if index_by_file_id:
+            self.mapping = {}
+        output_type = self.OUTPUT_TYPE
+        data, duration_filtered = [], 0.0
+        for audio_file, duration, command, offset in zip(audio_files, durations, labels, offsets):
+            # Duration filters.
+            if min_duration is not None and duration < min_duration:
+                duration_filtered += duration
+                continue
+
+            if max_duration is not None and duration > max_duration:
+                duration_filtered += duration
+                continue
+
+            data.append(output_type(audio_file, duration, command, offset))
+
+            if index_by_file_id:
+                file_id, _ = os.path.splitext(os.path.basename(audio_file))
+                self.mapping[file_id] = len(data) - 1
+
+            # Max number of entities filter.
+            if len(data) == max_number:
+                break
+
+        if do_sort_by_duration:
+            if index_by_file_id:
+                logging.warning("Tried to sort dataset by duration, but cannot since index_by_file_id is set.")
+            else:
+                data.sort(key=lambda entity: entity.duration)
+
+        logging.info(
+            "Filtered duration for loading collection is %f.", duration_filtered,
+        )
+        self.uniq_labels = sorted(set(map(lambda x: x.label, data)))
+        logging.info("# {} files loaded accounting to # {} labels".format(len(data), len(self.uniq_labels)))
+
+        super().__init__(data)
+
+
+class ASRSpeechLabel(SpeechLabel):
+    """`SpeechLabel` collector from structured json files."""
+
+    def __init__(self, manifests_files: Union[str, List[str]], is_regression_task=False, *args, **kwargs):
+        """Parse lists of audio files, durations and transcripts texts.
+
+        Args:
+            manifests_files: Either single string file or list of such -
+                manifests to yield items from.
+            is_regression_task: It's a regression task
+            *args: Args to pass to `SpeechLabel` constructor.
+            **kwargs: Kwargs to pass to `SpeechLabel` constructor.
+        """
+        audio_files, durations, labels, offsets = [], [], [], []
+
+        for item in manifest.item_iter(manifests_files, parse_func=self.__parse_item):
+            audio_files.append(item['audio_file'])
+            durations.append(item['duration'])
+            if not is_regression_task:
+                labels.append(item['label'])
+            else:
+                labels.append(float(item['label']))
+
+            offsets.append(item['offset'])
+
+        super().__init__(audio_files, durations, labels, offsets, *args, **kwargs)
+
+    def __parse_item(self, line: str, manifest_file: str) -> Dict[str, Any]:
+        item = json.loads(line)
+
+        # Audio file
+        if 'audio_filename' in item:
+            item['audio_file'] = item.pop('audio_filename')
+        elif 'audio_filepath' in item:
+            item['audio_file'] = item.pop('audio_filepath')
+        else:
+            raise ValueError(
+                f"Manifest file has invalid json line " f"structure: {line} without proper audio file key."
+            )
+        item['audio_file'] = os.path.expanduser(item['audio_file'])
+
+        # Duration.
+        if 'duration' not in item:
+            raise ValueError(f"Manifest file has invalid json line " f"structure: {line} without proper duration key.")
+
+        # Label.
+        if 'command' in item:
+            item['label'] = item.pop('command')
+        elif 'target' in item:
+            item['label'] = item.pop('target')
+        elif 'label' in item:
+            pass
+        else:
+            raise ValueError(f"Manifest file has invalid json line " f"structure: {line} without proper label key.")
+
+        item = dict(
+            audio_file=item['audio_file'],
+            duration=item['duration'],
+            label=item['label'],
+            offset=item.get('offset', None),
+        )
+
+        return item
+
+
+class FeatureSequenceLabel(_Collection):
+    """List of feature sequence of label correspondence with preprocessing."""
+
+    OUTPUT_TYPE = collections.namedtuple(typename='FeatureSequenceLabelEntity', field_names='feature_file seq_label',)
+
+    def __init__(
+        self,
+        feature_files: List[str],
+        seq_labels: List[str],
+        max_number: Optional[int] = None,
+        index_by_file_id: bool = False,
+    ):
+        """Instantiates feature-SequenceLabel manifest with filters and preprocessing.
+
+        Args:
+            feature_files: List of feature files.
+            seq_labels: List of sequences of abels.
+            max_number: Maximum number of samples to collect.
+            index_by_file_id: If True, saves a mapping from filename base (ID) to index in data.
+        """
+
+        output_type = self.OUTPUT_TYPE
+        data, num_filtered = (
+            [],
+            0.0,
+        )
+        self.uniq_labels = set()
+
+        if index_by_file_id:
+            self.mapping = {}
+
+        for feature_file, seq_label in zip(feature_files, seq_labels):
+
+            label_tokens, uniq_labels_in_seq = self.relative_speaker_parser(seq_label)
+
+            data.append(output_type(feature_file, label_tokens))
+            self.uniq_labels |= uniq_labels_in_seq
+
+            if label_tokens is None:
+                num_filtered += 1
+                continue
+
+            if index_by_file_id:
+                file_id, _ = os.path.splitext(os.path.basename(feature_file))
+                self.mapping[feature_file] = len(data) - 1
+
+            # Max number of entities filter.
+            if len(data) == max_number:
+                break
+
+        logging.info("# {} files loaded including # {} unique labels".format(len(data), len(self.uniq_labels)))
+        super().__init__(data)
+
+    def relative_speaker_parser(self, seq_label):
+        """ Convert sequence of speaker labels to relative labels.
+        Convert sequence of absolute speaker to sequence of relative speaker [E A C A E E C] -> [0 1 2 1 0 0 2]
+        In this seq of label , if label do not appear before, assign new relative labels len(pos); else reuse previous assigned relative labels.
+        Args:
+            seq_label (str): A string of a sequence of labels.
+
+        Return:
+            relative_seq_label (List) : A list of relative sequence of labels
+            unique_labels_in_seq (Set): A set of unique labels in the sequence
+        """
+        seq = seq_label.split()
+        conversion_dict = dict()
+        relative_seq_label = []
+
+        for seg in seq:
+            if seg in conversion_dict:
+                converted = conversion_dict[seg]
+            else:
+                converted = len(conversion_dict)
+                conversion_dict[seg] = converted
+
+            relative_seq_label.append(converted)
+
+        unique_labels_in_seq = set(conversion_dict.keys())
+        return relative_seq_label, unique_labels_in_seq
+
+
+class ASRFeatureSequenceLabel(FeatureSequenceLabel):
+    """`FeatureSequenceLabel` collector from asr structured json files."""
+
+    def __init__(
+        self, manifests_files: Union[str, List[str]], max_number: Optional[int] = None, index_by_file_id: bool = False,
+    ):
+
+        """Parse lists of feature files and sequences of labels.
+
+        Args:
+            manifests_files: Either single string file or list of such -
+                manifests to yield items from.
+            max_number:  Maximum number of samples to collect; pass to `FeatureSequenceLabel` constructor.
+            index_by_file_id: If True, saves a mapping from filename base (ID) to index in data; pass to `FeatureSequenceLabel` constructor.
+        """
+
+        feature_files, seq_labels = [], []
+        for item in manifest.item_iter(manifests_files, parse_func=self._parse_item):
+            feature_files.append(item['feature_file'])
+            seq_labels.append(item['seq_label'])
+
+        super().__init__(feature_files, seq_labels, max_number, index_by_file_id)
+
+    def _parse_item(self, line: str, manifest_file: str) -> Dict[str, Any]:
+        item = json.loads(line)
+
+        # Feature file
+        if 'feature_filename' in item:
+            item['feature_file'] = item.pop('feature_filename')
+        elif 'feature_filepath' in item:
+            item['feature_file'] = item.pop('feature_filepath')
+        else:
+            raise ValueError(
+                f"Manifest file has invalid json line " f"structure: {line} without proper feature file key."
+            )
+        item['feature_file'] = os.path.expanduser(item['feature_file'])
+
+        # Seq of Label.
+        if 'seq_label' in item:
+            item['seq_label'] = item.pop('seq_label')
+        else:
+            raise ValueError(
+                f"Manifest file has invalid json line " f"structure: {line} without proper seq_label key."
+            )
+
+        item = dict(feature_file=item['feature_file'], seq_label=item['seq_label'],)
+
+        return item

tts/nemo/collections/common/parts/preprocessing/manifest.py

@@ -0,0 +1,120 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from os.path import expanduser
+from typing import Any, Callable, Dict, Iterator, List, Optional, Union
+
+
+class ManifestBase:
+    def __init__(self, *args, **kwargs):
+        raise ValueError(
+            "This class is deprecated, look at https://github.com/NVIDIA/NeMo/pull/284 for correct behaviour."
+        )
+
+
+class ManifestEN:
+    def __init__(self, *args, **kwargs):
+        raise ValueError(
+            "This class is deprecated, look at https://github.com/NVIDIA/NeMo/pull/284 for correct behaviour."
+        )
+
+
+def item_iter(
+    manifests_files: Union[str, List[str]], parse_func: Callable[[str, Optional[str]], Dict[str, Any]] = None
+) -> Iterator[Dict[str, Any]]:
+    """Iterate through json lines of provided manifests.
+
+    NeMo ASR pipelines often assume certain manifest files structure. In
+    particular, each manifest file should consist of line-per-sample files with
+    each line being correct json dict. Each such json dict should have a field
+    for audio file string, a field for duration float and a field for text
+    string. Offset also could be additional field and is set to None by
+    default.
+
+    Args:
+        manifests_files: Either single string file or list of such -
+            manifests to yield items from.
+
+        parse_func: A callable function which accepts as input a single line
+            of a manifest and optionally the manifest file itself,
+            and parses it, returning a dictionary mapping from str -> Any.
+
+    Yields:
+        Parsed key to value item dicts.
+
+    Raises:
+        ValueError: If met invalid json line structure.
+    """
+
+    if isinstance(manifests_files, str):
+        manifests_files = [manifests_files]
+
+    if parse_func is None:
+        parse_func = __parse_item
+
+    k = -1
+    for manifest_file in manifests_files:
+        with open(expanduser(manifest_file), 'r') as f:
+            for line in f:
+                k += 1
+                item = parse_func(line, manifest_file)
+                item['id'] = k
+
+                yield item
+
+
+def __parse_item(line: str, manifest_file: str) -> Dict[str, Any]:
+    item = json.loads(line)
+
+    # Audio file
+    if 'audio_filename' in item:
+        item['audio_file'] = item.pop('audio_filename')
+    elif 'audio_filepath' in item:
+        item['audio_file'] = item.pop('audio_filepath')
+    else:
+        raise ValueError(
+            f"Manifest file {manifest_file} has invalid json line structure: {line} without proper audio file key."
+        )
+    item['audio_file'] = expanduser(item['audio_file'])
+
+    # Duration.
+    if 'duration' not in item:
+        raise ValueError(
+            f"Manifest file {manifest_file} has invalid json line structure: {line} without proper duration key."
+        )
+
+    # Text.
+    if 'text' in item:
+        pass
+    elif 'text_filepath' in item:
+        with open(item.pop('text_filepath'), 'r') as f:
+            item['text'] = f.read().replace('\n', '')
+    elif 'normalized_text' in item:
+        item['text'] = item['normalized_text']
+    else:
+        raise ValueError(
+            f"Manifest file {manifest_file} has invalid json line structure: {line} without proper text key."
+        )
+
+    item = dict(
+        audio_file=item['audio_file'],
+        duration=item['duration'],
+        text=item['text'],
+        offset=item.get('offset', None),
+        speaker=item.get('speaker', None),
+        orig_sr=item.get('orig_sample_rate', None),
+    )
+
+    return item

tts/nemo/collections/common/parts/preprocessing/parsers.py

@@ -0,0 +1,185 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import string
+from typing import List, Optional
+
+import frozendict
+
+from nemo.collections.common.parts.preprocessing import cleaners
+
+
+class CharParser:
+    """Functor for parsing raw strings into list of int tokens.
+
+    Examples:
+        >>> parser = CharParser(['a', 'b', 'c'])
+        >>> parser('abc')
+        [0, 1, 2]
+    """
+
+    def __init__(
+        self,
+        labels: List[str],
+        *,
+        unk_id: int = -1,
+        blank_id: int = -1,
+        do_normalize: bool = True,
+        do_lowercase: bool = True,
+    ):
+        """Creates simple mapping char parser.
+
+        Args:
+            labels: List of labels to allocate indexes for. Essentially,
+                this is a id to str mapping.
+            unk_id: Index to choose for OOV words (default: -1).
+            blank_id: Index to filter out from final list of tokens
+                (default: -1).
+            do_normalize: True if apply normalization step before tokenizing
+                (default: True).
+            do_lowercase: True if apply lowercasing at normalizing step
+                (default: True).
+        """
+
+        self._labels = labels
+        self._unk_id = unk_id
+        self._blank_id = blank_id
+        self._do_normalize = do_normalize
+        self._do_lowercase = do_lowercase
+
+        self._labels_map = {label: index for index, label in enumerate(labels)}
+        self._special_labels = set([label for label in labels if len(label) > 1])
+
+    def __call__(self, text: str) -> Optional[List[int]]:
+        if self._do_normalize:
+            text = self._normalize(text)
+            if text is None:
+                return None
+
+        text_tokens = self._tokenize(text)
+
+        return text_tokens
+
+    def _normalize(self, text: str) -> Optional[str]:
+        text = text.strip()
+
+        if self._do_lowercase:
+            text = text.lower()
+
+        return text
+
+    def _tokenize(self, text: str) -> List[int]:
+        tokens = []
+        # Split by word for find special labels.
+        for word_id, word in enumerate(text.split(' ')):
+            if word_id != 0:  # Not first word - so we insert space before.
+                tokens.append(self._labels_map.get(' ', self._unk_id))
+
+            if word in self._special_labels:
+                tokens.append(self._labels_map[word])
+                continue
+
+            for char in word:
+                tokens.append(self._labels_map.get(char, self._unk_id))
+
+        # If unk_id == blank_id, OOV tokens are removed.
+        tokens = [token for token in tokens if token != self._blank_id]
+
+        return tokens
+
+
+class ENCharParser(CharParser):
+    """Incorporates english-specific parsing logic."""
+
+    PUNCTUATION_TO_REPLACE = frozendict.frozendict({'+': 'plus', '&': 'and', '%': 'percent'})
+
+    def __init__(self, abbreviation_version=None, make_table=True, *args, **kwargs):
+        """Creates english-specific mapping char parser.
+
+        This class overrides normalizing implementation.
+
+        Args:
+            *args: Positional args to pass to `CharParser` constructor.
+            **kwargs: Key-value args to pass to `CharParser` constructor.
+        """
+
+        super().__init__(*args, **kwargs)
+
+        self._table = None
+        if make_table:
+            self._table = self.__make_trans_table()
+        self.abbreviation_version = abbreviation_version
+
+    def __make_trans_table(self):
+        punctuation = string.punctuation
+
+        for char in self.PUNCTUATION_TO_REPLACE:
+            punctuation = punctuation.replace(char, '')
+
+        for label in self._labels:
+            punctuation = punctuation.replace(label, '')
+
+        table = str.maketrans(punctuation, ' ' * len(punctuation))
+
+        return table
+
+    def _normalize(self, text: str) -> Optional[str]:
+        # noinspection PyBroadException
+        try:
+            text = cleaners.clean_text(
+                string=text,
+                table=self._table,
+                punctuation_to_replace=self.PUNCTUATION_TO_REPLACE,
+                abbreviation_version=self.abbreviation_version,
+            )
+        except Exception:
+            return None
+
+        return text
+
+
+NAME_TO_PARSER = frozendict.frozendict({'base': CharParser, 'en': ENCharParser})
+
+
+def make_parser(labels: Optional[List[str]] = None, name: str = 'base', **kwargs,) -> CharParser:
+    """Creates parser from labels, set of arguments and concise parser name.
+
+    Args:
+        labels: List of labels to allocate indexes for. If set to
+            None then labels would be ascii table list. Essentially, this is a
+            id to str mapping (default: None).
+        name: Concise name of parser to create (default: 'base').
+            (default: -1).
+        **kwargs: Other set of kwargs to pass to parser constructor.
+
+    Returns:
+        Instance of `CharParser`.
+
+    Raises:
+        ValueError: For invalid parser name.
+
+    Examples:
+        >>> type(make_parser(['a', 'b', 'c'], 'en'))
+        ENCharParser
+    """
+
+    if name not in NAME_TO_PARSER:
+        raise ValueError('Invalid parser name.')
+
+    if labels is None:
+        labels = list(string.printable)
+
+    parser_type = NAME_TO_PARSER[name]
+    parser = parser_type(labels=labels, **kwargs)
+
+    return parser