Introduce code from Rust monorepo. This is a mess

2026-10-13 23:53:02 +00:00 · 2022-02-28 05:43:54 -05:00
parent 921fb49f9a
commit ba326bfcfd
1058 changed files with 192192 additions and 0 deletions
@@ -0,0 +1,178 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Copyright (c) 2020, SeanNaren.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# To convert mp3 files to wav using sox, you must have installed sox with mp3 support
+# For example sudo apt-get install libsox-fmt-mp3
+import argparse
+import csv
+import json
+import logging
+import multiprocessing
+import os
+import subprocess
+import sys
+import tarfile
+from multiprocessing.pool import ThreadPool
+from pathlib import Path
+from typing import List
+
+import sox
+from sox import Transformer
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description='Downloads and processes Mozilla Common Voice dataset.')
+parser.add_argument("--data_root", default='CommonVoice_dataset/', type=str, help="Directory to store the dataset.")
+parser.add_argument('--manifest_dir', default='./', type=str, help='Output directory for manifests')
+parser.add_argument("--num_workers", default=multiprocessing.cpu_count(), type=int, help="Workers to process dataset.")
+parser.add_argument('--sample_rate', default=16000, type=int, help='Sample rate')
+parser.add_argument('--n_channels', default=1, type=int, help='Number of channels for output wav files')
+parser.add_argument(
+    '--files_to_process',
+    nargs='+',
+    default=['test.tsv', 'dev.tsv', 'train.tsv'],
+    type=str,
+    help='list of *.csv file names to process',
+)
+parser.add_argument(
+    '--version',
+    default='cv-corpus-5.1-2020-06-22',
+    type=str,
+    help='Version of the dataset (obtainable via https://commonvoice.mozilla.org/en/datasets',
+)
+parser.add_argument(
+    '--language',
+    default='en',
+    type=str,
+    help='Which language to download.(default english,'
+    'check https://commonvoice.mozilla.org/en/datasets for more language codes',
+)
+args = parser.parse_args()
+COMMON_VOICE_URL = (
+    f"https://voice-prod-bundler-ee1969a6ce8178826482b88e843c335139bd3fb4.s3.amazonaws.com/"
+    "{}/{}.tar.gz".format(args.version, args.language)
+)
+
+
+def create_manifest(data: List[tuple], output_name: str, manifest_path: str):
+    output_file = Path(manifest_path) / output_name
+    output_file.parent.mkdir(exist_ok=True, parents=True)
+
+    with output_file.open(mode='w') as f:
+        for wav_path, duration, text in tqdm(data, total=len(data)):
+            f.write(
+                json.dumps({'audio_filepath': os.path.abspath(wav_path), "duration": duration, 'text': text}) + '\n'
+            )
+
+
+def process_files(csv_file, data_root, num_workers):
+    """ Read *.csv file description, convert mp3 to wav, process text.
+        Save results to data_root.
+
+    Args:
+        csv_file: str, path to *.csv file with data description, usually start from 'cv-'
+        data_root: str, path to dir to save results; wav/ dir will be created
+    """
+    wav_dir = os.path.join(data_root, 'wav/')
+    os.makedirs(wav_dir, exist_ok=True)
+    audio_clips_path = os.path.dirname(csv_file) + '/clips/'
+
+    def process(x):
+        file_path, text = x
+        file_name = os.path.splitext(os.path.basename(file_path))[0]
+        text = text.lower().strip()
+        audio_path = os.path.join(audio_clips_path, file_path)
+        output_wav_path = os.path.join(wav_dir, file_name + '.wav')
+
+        tfm = Transformer()
+        tfm.rate(samplerate=args.sample_rate)
+        tfm.channels(n_channels=args.n_channels)
+        tfm.build(input_filepath=audio_path, output_filepath=output_wav_path)
+        duration = sox.file_info.duration(output_wav_path)
+        return output_wav_path, duration, text
+
+    logging.info('Converting mp3 to wav for {}.'.format(csv_file))
+    with open(csv_file) as csvfile:
+        reader = csv.DictReader(csvfile, delimiter='\t')
+        next(reader, None)  # skip the headers
+        data = [(row['path'], row['sentence']) for row in reader]
+        with ThreadPool(num_workers) as pool:
+            data = list(tqdm(pool.imap(process, data), total=len(data)))
+    return data
+
+
+def main():
+    data_root = args.data_root
+    os.makedirs(data_root, exist_ok=True)
+
+    target_unpacked_dir = os.path.join(data_root, "CV_unpacked")
+
+    if os.path.exists(target_unpacked_dir):
+        logging.info('Find existing folder {}'.format(target_unpacked_dir))
+    else:
+        logging.info("Could not find Common Voice, Downloading corpus...")
+
+        commands = [
+            'wget',
+            '--user-agent',
+            '"Mozilla/5.0 (Windows NT 10.0; WOW64) '
+            'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36"',
+            '-P',
+            data_root,
+            f'{COMMON_VOICE_URL}',
+        ]
+        commands = " ".join(commands)
+        subprocess.run(commands, shell=True, stderr=sys.stderr, stdout=sys.stdout, capture_output=False)
+        filename = f"{args.language}.tar.gz"
+        target_file = os.path.join(data_root, os.path.basename(filename))
+
+        os.makedirs(target_unpacked_dir, exist_ok=True)
+        logging.info("Unpacking corpus to {} ...".format(target_unpacked_dir))
+        tar = tarfile.open(target_file)
+        tar.extractall(target_unpacked_dir)
+        tar.close()
+
+    folder_path = os.path.join(target_unpacked_dir, args.version + f'/{args.language}/')
+
+    for csv_file in args.files_to_process:
+        data = process_files(
+            csv_file=os.path.join(folder_path, csv_file),
+            data_root=os.path.join(data_root, os.path.splitext(csv_file)[0]),
+            num_workers=args.num_workers,
+        )
+        logging.info('Creating manifests...')
+        create_manifest(
+            data=data,
+            output_name=f'commonvoice_{os.path.splitext(csv_file)[0]}_manifest.json',
+            manifest_path=args.manifest_dir,
+        )
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,185 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_aishell_data.py --data_root=<where to put data>
+
+import argparse
+import json
+import logging as _logging
+import os
+import tarfile
+import urllib.request
+from glob import glob
+
+import librosa as l
+from sklearn.model_selection import StratifiedShuffleSplit
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description='HI-MIA Data download')
+parser.add_argument("--data_root", required=True, default=None, type=str)
+parser.add_argument("--log_level", default=20, type=int)
+args = parser.parse_args()
+logging = _logging.getLogger(__name__)
+logging.addHandler(_logging.StreamHandler())
+logging.setLevel(args.log_level)
+
+URL = {
+    'dev': "http://www.openslr.org/resources/85/dev.tar.gz",
+    'test': "http://www.openslr.org/resources/85/test.tar.gz",
+    'train': "http://www.openslr.org/resources/85/train.tar.gz",
+}
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    source = URL[source]
+    if not os.path.exists(destination) and not os.path.exists(os.path.splitext(destination)[0]):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info("Downloaded {0}.".format(destination))
+    elif os.path.exists(destination):
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    elif os.path.exists(os.path.splitext(destination)[0]):
+        logging.warning(
+            "Assuming extracted folder %s contains the extracted files from %s. Will not download.",
+            os.path.basename(destination),
+            destination,
+        )
+    return destination
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_root)
+        audio_dir = os.path.join(data_dir, 'wav')
+        for subfolder, _, filelist in os.walk(audio_dir):
+            for ftar in filelist:
+                extract_file(os.path.join(subfolder, ftar), subfolder)
+    else:
+        logging.info('Skipping extracting. Data already there %s' % data_dir)
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        tar = tarfile.open(filepath)
+        tar.extractall(data_dir)
+        tar.close()
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __remove_tarred_files(filepath: str, data_dir: str):
+    if os.path.exists(data_dir) and os.path.isfile(filepath):
+        logging.info('Deleting %s' % filepath)
+        os.remove(filepath)
+
+
+def write_file(name, lines, idx):
+    with open(name, 'w') as fout:
+        for i in idx:
+            dic = lines[i]
+            json.dump(dic, fout)
+            fout.write('\n')
+    logging.info("wrote %s", name)
+
+
+def __process_data(data_folder: str, data_set: str):
+    """
+    To generate manifest
+    Args:
+        data_folder: source with wav files
+    Returns:
+
+    """
+    fullpath = os.path.abspath(data_folder)
+    scp = glob(fullpath + '/**/*.wav', recursive=True)
+    out = os.path.join(fullpath, data_set + '_all.json')
+    utt2spk = os.path.join(fullpath, 'utt2spk')
+    utt2spk_file = open(utt2spk, 'w')
+    id = -2  # speaker id
+
+    if os.path.exists(out):
+        logging.warning(
+            "%s already exists and is assumed to be processed. If not, please delete %s and rerun this script",
+            out,
+            out,
+        )
+        return
+
+    speakers = []
+    lines = []
+    with open(out, 'w') as outfile:
+        for line in tqdm(scp):
+            line = line.strip()
+            y, sr = l.load(line, sr=None)
+            if sr != 16000:
+                y, sr = l.load(line, sr=16000)
+                l.output.write_wav(line, y, sr)
+            dur = l.get_duration(y=y, sr=sr)
+            if data_set == 'test':
+                speaker = line.split('/')[-1].split('.')[0].split('_')[0]
+            else:
+                speaker = line.split('/')[id]
+            speaker = list(speaker)
+            speaker = ''.join(speaker)
+            speakers.append(speaker)
+            meta = {"audio_filepath": line, "duration": float(dur), "label": speaker}
+            lines.append(meta)
+            json.dump(meta, outfile)
+            outfile.write("\n")
+            utt2spk_file.write(line.split('/')[-1] + "\t" + speaker + "\n")
+
+    utt2spk_file.close()
+
+    if data_set != 'test':
+        sss = StratifiedShuffleSplit(n_splits=1, test_size=0.1, random_state=42)
+        for train_idx, test_idx in sss.split(speakers, speakers):
+            print(len(train_idx))
+
+        out = os.path.join(fullpath, 'train.json')
+        write_file(out, lines, train_idx)
+        out = os.path.join(fullpath, 'dev.json')
+        write_file(out, lines, test_idx)
+
+
+def main():
+    data_root = args.data_root
+    for data_set in URL.keys():
+
+        # data_set = 'data_aishell'
+        logging.info("\n\nWorking on: {0}".format(data_set))
+        file_path = os.path.join(data_root, data_set + ".tgz")
+        logging.info("Getting {0}".format(data_set))
+        __maybe_download_file(file_path, data_set)
+        logging.info("Extracting {0}".format(data_set))
+        data_folder = os.path.join(data_root, data_set)
+        __extract_all_files(file_path, data_root, data_folder)
+        __remove_tarred_files(file_path, data_folder)
+        logging.info("Processing {0}".format(data_set))
+        __process_data(data_folder, data_set)
+        logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,171 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# USAGE: python get_librispeech_data.py --data_root=<where to put data>
+#        --data_set=<datasets_to_download> --num_workers=<number of parallel workers>
+# where <datasets_to_download> can be: dev_clean, dev_other, test_clean,
+# test_other, train_clean_100, train_clean_360, train_other_500 or ALL
+# You can also put more than one data_set comma-separated:
+# --data_set=dev_clean,train_clean_100
+import argparse
+import fnmatch
+import functools
+import json
+import logging
+import multiprocessing
+import os
+import subprocess
+import tarfile
+import urllib.request
+
+from sox import Transformer
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description='LibriSpeech Data download')
+parser.add_argument("--data_root", required=True, default=None, type=str)
+parser.add_argument("--data_sets", default="dev_clean", type=str)
+parser.add_argument("--num_workers", default=4, type=int)
+args = parser.parse_args()
+
+URLS = {
+    'TRAIN_CLEAN_100': ("http://www.openslr.org/resources/12/train-clean-100.tar.gz"),
+    'TRAIN_CLEAN_360': ("http://www.openslr.org/resources/12/train-clean-360.tar.gz"),
+    'TRAIN_OTHER_500': ("http://www.openslr.org/resources/12/train-other-500.tar.gz"),
+    'DEV_CLEAN': "http://www.openslr.org/resources/12/dev-clean.tar.gz",
+    'DEV_OTHER': "http://www.openslr.org/resources/12/dev-other.tar.gz",
+    'TEST_CLEAN': "http://www.openslr.org/resources/12/test-clean.tar.gz",
+    'TEST_OTHER': "http://www.openslr.org/resources/12/test-other.tar.gz",
+}
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+    Returns:
+    """
+    source = URLS[source]
+    if not os.path.exists(destination):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info("Downloaded {0}.".format(destination))
+    else:
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    return destination
+
+
+def __extract_file(filepath: str, data_dir: str):
+    try:
+        tar = tarfile.open(filepath)
+        tar.extractall(data_dir)
+        tar.close()
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __process_transcript(file_path: str, dst_folder: str):
+    """
+    Converts flac files to wav from a given transcript, capturing the metadata.
+    Args:
+        file_path: path to a source transcript  with flac sources
+        dst_folder: path where wav files will be stored
+    Returns:
+        a list of metadata entries for processed files.
+    """
+    entries = []
+    root = os.path.dirname(file_path)
+    with open(file_path, encoding="utf-8") as fin:
+        for line in fin:
+            id, text = line[: line.index(" ")], line[line.index(" ") + 1 :]
+            transcript_text = text.lower().strip()
+
+            # Convert FLAC file to WAV
+            flac_file = os.path.join(root, id + ".flac")
+            wav_file = os.path.join(dst_folder, id + ".wav")
+            if not os.path.exists(wav_file):
+                Transformer().build(flac_file, wav_file)
+            # check duration
+            duration = subprocess.check_output("soxi -D {0}".format(wav_file), shell=True)
+
+            entry = {}
+            entry['audio_filepath'] = os.path.abspath(wav_file)
+            entry['duration'] = float(duration)
+            entry['text'] = transcript_text
+            entries.append(entry)
+    return entries
+
+
+def __process_data(data_folder: str, dst_folder: str, manifest_file: str, num_workers: int):
+    """
+    Converts flac to wav and build manifests's json
+    Args:
+        data_folder: source with flac files
+        dst_folder: where wav files will be stored
+        manifest_file: where to store manifest
+        num_workers: number of parallel workers processing files
+    Returns:
+    """
+
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+
+    files = []
+    entries = []
+
+    for root, dirnames, filenames in os.walk(data_folder):
+        for filename in fnmatch.filter(filenames, '*.trans.txt'):
+            files.append(os.path.join(root, filename))
+
+    with multiprocessing.Pool(num_workers) as p:
+        processing_func = functools.partial(__process_transcript, dst_folder=dst_folder)
+        results = p.imap(processing_func, files)
+        for result in tqdm(results, total=len(files)):
+            entries.extend(result)
+
+    with open(manifest_file, 'w') as fout:
+        for m in entries:
+            fout.write(json.dumps(m) + '\n')
+
+
+def main():
+    data_root = args.data_root
+    data_sets = args.data_sets
+    num_workers = args.num_workers
+
+    if data_sets == "ALL":
+        data_sets = "dev_clean,dev_other,train_clean_100,train_clean_360,train_other_500,test_clean,test_other"
+
+    for data_set in data_sets.split(','):
+        logging.info("\n\nWorking on: {0}".format(data_set))
+        filepath = os.path.join(data_root, data_set + ".tar.gz")
+        logging.info("Getting {0}".format(data_set))
+        __maybe_download_file(filepath, data_set.upper())
+        logging.info("Extracting {0}".format(data_set))
+        __extract_file(filepath, data_root)
+        logging.info("Processing {0}".format(data_set))
+        __process_data(
+            os.path.join(os.path.join(data_root, "LibriSpeech"), data_set.replace("_", "-"),),
+            os.path.join(os.path.join(data_root, "LibriSpeech"), data_set.replace("_", "-"),) + "-processed",
+            os.path.join(data_root, data_set + ".json"),
+            num_workers=num_workers,
+        )
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,140 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# USAGE: python get_openslr_rir_data.py --data_root=<where to put data>
+# Data is downloaded from OpenSLR's "Room Impulse Response and Noise Database"
+# RIRs in multichannel files are separated into single channel files and
+# a json file that can be used as in input to NeMo is created
+
+import argparse
+import glob
+import json
+import logging
+import os
+import subprocess
+import urllib.request
+from shutil import copy, move
+from zipfile import ZipFile
+
+parser = argparse.ArgumentParser(description='OpenSLR RIR Data download and process')
+parser.add_argument("--data_root", required=True, default=None, type=str)
+args = parser.parse_args()
+
+URLS = {
+    'SLR28': ("http://www.openslr.org/resources/28/rirs_noises.zip"),
+}
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+    Returns:
+    """
+    source = URLS[source]
+    if not os.path.exists(destination):
+        logging.info("{0} does not exist. Downloading ...".format(destination))
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info("Downloaded {0}.".format(destination))
+    else:
+        logging.info("Destination {0} exists. Skipping.".format(destination))
+    return destination
+
+
+def __extract_file(filepath: str, data_dir: str):
+    try:
+        with ZipFile(filepath, 'r') as zipObj:
+            zipObj.extractall(data_dir)
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __process_data(data_folder: str, dst_folder: str, manifest_file: str):
+    """
+    Converts flac to wav and build manifests's json
+    Args:
+        data_folder: source with flac files
+        dst_folder: where wav files will be stored
+        manifest_file: where to store manifest
+    Returns:
+    """
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+
+    real_rir_list = os.path.join(data_folder, "RIRS_NOISES", "real_rirs_isotropic_noises", "rir_list")
+    rirfiles = []
+    with open(real_rir_list, "r") as rir_f:
+        for line in rir_f:
+            rirfiles.append(os.path.join(data_folder, line.rstrip().split(" ")[4]))
+
+    real_rir_folder = os.path.join(dst_folder, "real_rirs")
+    if not os.path.exists(real_rir_folder):
+        os.makedirs(real_rir_folder)
+    # split multi-channel rir files to single channel
+    for rir_f in rirfiles:
+        n_chans = int(subprocess.check_output("soxi -c {0}".format(rir_f), shell=True))
+        if n_chans == 1:
+            copy(rir_f, real_rir_folder)
+        else:
+            for chan in range(1, n_chans + 1):
+                chan_file_name = os.path.join(
+                    real_rir_folder, os.path.splitext(os.path.basename(rir_f))[0] + "-" + str(chan) + ".wav"
+                )
+                _ = subprocess.check_output(f"sox {rir_f} {chan_file_name} remix {chan}", shell=True)
+
+    # move simulated rirs to processed
+    if not os.path.exists(os.path.join(dst_folder, "simulated_rirs")):
+        move(os.path.join(data_folder, "RIRS_NOISES", "simulated_rirs"), dst_folder)
+
+    os.chdir(dst_folder)
+    all_rirs = glob.glob("**/*.wav", recursive=True)
+    with open(manifest_file, 'w') as man_f:
+        entry = {}
+        for rir in all_rirs:
+            rir_file = os.path.join(dst_folder, rir)
+            duration = subprocess.check_output("soxi -D {0}".format(rir_file), shell=True)
+            entry['audio_filepath'] = rir_file
+            entry['duration'] = float(duration)
+            entry['offset'] = 0
+            entry['text'] = '_'
+            man_f.write(json.dumps(entry) + "\n")
+
+    print('Done!')
+
+
+def main():
+    data_root = os.path.abspath(args.data_root)
+    data_set = "slr28"
+    logging.getLogger().setLevel(logging.INFO)
+    logging.info("\n\nWorking on: {0}".format(data_set))
+    filepath = os.path.join(data_root, data_set + ".zip")
+    logging.info("Getting {0}".format(data_set))
+    __maybe_download_file(filepath, data_set.upper())
+    logging.info("Extracting {0}".format(data_set))
+    __extract_file(filepath, data_root)
+    logging.info("Processing {0}".format(data_set))
+    __process_data(
+        data_root,
+        os.path.join(os.path.join(data_root, "processed")),
+        os.path.join(os.path.join(data_root, "processed", "rir.json")),
+    )
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,14 @@
+These scripts are used for extracting features from LJSpeech.
+
+You will need to run the following scripts (note that these will call the other
+scripts as necessary):
+
+- `create_manifests_and_textfiles.py`: Creates the manifest and text files that
+  are used by the Montreal Forced Aligner (MFA) library.
+- `extract_ljspeech_phonemes_and_durs.sh`: Extracts phonemes and alignments via
+  MFA and calculates phoneme durations.
+  Needs to be run with Conda active since it creates a Conda environment.
+  Also note that you may need to install OpenBlas manually (e.g. sudo apt-get
+  install libopenblas-dev), sometimes MFA can't find it if installed via Conda.
+  (This script calls `create_token2idx_dict.py` and `calculate_durs.py`.)
+- `extract_ljspeech_energy_pitch.py`
@@ -0,0 +1,126 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Calculates durations for LJSpeech based on MFA TextGrid alignments.
+"""
+import argparse
+import glob
+import json
+import os
+import pickle
+from math import ceil
+
+import numpy as np
+import tgt
+import torch
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Calculates phoneme durations for LJSpeech from TextGrids.")
+parser.add_argument('--ljspeech_dir', required=True, default=None, type=str)
+parser.add_argument(
+    '--mappings',
+    required=False,
+    default=None,
+    type=str,
+    help='JSON file of mappings created with create_token2idx_dict.py',
+)
+parser.add_argument('--sr', required=False, default=22050, type=int)
+parser.add_argument('--hop_length', required=False, default=256, type=int)
+args = parser.parse_args()
+
+
+def calculate_durations(textgrid, phone2idx):
+    tokens = []
+    durs = []
+
+    frames_per_second = args.sr / args.hop_length
+    tg = tgt.read_textgrid(textgrid, include_empty_intervals=True)
+    data_tier = tg.get_tier_by_name("phones")
+
+    # Get total frames
+    total_frames = ceil((data_tier.end_time - data_tier.start_time) * frames_per_second)
+
+    # Find start and end frames of each token
+    se_in_frames = np.array([(frames_per_second * d.start_time, frames_per_second * d.end_time) for d in data_tier])
+    se_in_frames = np.round(se_in_frames)
+    durs = (se_in_frames[:, 1] - se_in_frames[:, 0]).astype(int)
+    blank_set = ('sil', 'sp', 'spn', '', '<unk>')
+    blank_token = " "
+
+    # merge repeated blank tokens
+    tokens, durations = [], []
+    for i in range(len(data_tier)):
+        x = data_tier[i].text
+        if x == 'spn':
+            return None, None, None
+        x = blank_token if x in blank_set else x
+
+        if len(tokens) and tokens[-1] == blank_token and x == blank_token:
+            durations[-1] += durs[i]
+        else:
+            tokens.append(x)
+            durations.append(durs[i])
+
+    tokens_enc = [phone2idx[token] for token in tokens]
+    tokens_enc, durations = torch.LongTensor(tokens_enc), torch.LongTensor(durations)
+
+    # Add rounding error to final token
+    durations[-1] += total_frames - durations.sum()
+
+    return tokens, tokens_enc, durations
+
+
+def main():
+    textgrid_list = glob.glob(os.path.join(args.ljspeech_dir, 'alignments/wavs/*.TextGrid'))
+
+    # Create target_dir if necessary
+    target_dir = os.path.join(args.ljspeech_dir, 'phoneme_durations/')
+    print(f"Calculating phoneme durations, files will be in: {target_dir}")
+
+    if not os.path.exists(target_dir):
+        print(f"Creating {target_dir}")
+        os.mkdir(target_dir)
+
+    # Read phoneme to idx mappings
+    phone2idx = None
+    if args.mappings:
+        with open(args.mappings, 'r') as f:
+            mappings = json.load(f)
+            phone2idx = mappings['phone2idx']
+
+    oov_samples = []
+
+    # Iterate through all TextGrid files
+    for textgrid in tqdm(textgrid_list):
+        basename = os.path.splitext(os.path.basename(textgrid))[0][5:]  # Chop off 'wavs_' prefix
+
+        phones_mfa, tokens_mfa, durs = calculate_durations(textgrid, phone2idx)
+
+        if phones_mfa is None:
+            oov_samples.append(basename)
+            continue
+
+        # Save to file
+        target_path = os.path.join(target_dir, f'{basename}.pt')
+        torch.save({'text_encoded': tokens_mfa, 'token_duration': durs}, target_path)
+
+    print(f"Getting rid of {len(oov_samples)} samples with OOV words.")
+    oov_target = os.path.join(args.ljspeech_dir, 'wavs_to_ignore.pkl')
+    with open(oov_target, 'wb') as f:
+        pickle.dump(oov_samples, f)
+    print(f"List of OOV samples written to: {oov_target}")
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,71 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Creates the manifest and .txt transcript files for an LJSpeech split.
+The manifest will be used for training, and the .txt files are for the MFA library to find.
+"""
+
+import argparse
+import json
+import os
+
+import sox
+import wget
+
+from nemo.collections.common.parts.preprocessing import parsers
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--ljspeech_base', required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def main():
+    filelist_base = 'https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists/'
+    filelists = ['train', 'val', 'test']
+
+    # NeMo parser for text normalization
+    text_parser = parsers.make_parser(name='en')
+
+    for split in filelists:
+        # Download file list if necessary
+        filelist_path = os.path.join(args.ljspeech_base, f"ljs_audio_text_{split}_filelist.txt")
+        if not os.path.exists(filelist_path):
+            wget.download(f"{filelist_base}/ljs_audio_text_{split}_filelist.txt", out=args.ljspeech_base)
+
+        manifest_target = os.path.join(args.ljspeech_base, f"ljspeech_{split}.json")
+        with open(manifest_target, 'w') as f_out:
+            with open(filelist_path, 'r') as filelist:
+                print(f"\nCreating {manifest_target}...")
+                for line in filelist:
+                    basename = line[6:16]
+                    text = text_parser._normalize(line[21:].strip())
+
+                    # Make sure corresponding wavfile exists and write .txt transcript
+                    wav_path = os.path.join(args.ljspeech_base, 'wavs/', basename + '.wav')
+                    assert os.path.exists(wav_path)
+                    txt_path = os.path.join(args.ljspeech_base, 'wavs/', basename + '.txt')
+                    with open(txt_path, 'w') as f_txt:
+                        f_txt.write(text)
+
+                    # Write manifest entry
+                    entry = {
+                        'audio_filepath': wav_path,
+                        'duration': sox.file_info.duration(wav_path),
+                        'text': text,
+                    }
+                    f_out.write(json.dumps(entry) + '\n')
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,68 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Creates a dictionary from token to index based on dictionary .txt given.
+"""
+import argparse
+import json
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--dictionary', required=True, default=None, type=str)
+parser.add_argument('--dict_out', required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def main():
+    if not os.path.exists(args.dictionary):
+        raise FileNotFoundError(f"Could not find dictionary file {args.dictionary}")
+
+    phonemes = set()
+    word2phones = {}
+    with open(args.dictionary, 'r') as f:
+        for line in f:
+            line = line.split()
+            word = line[0]
+            tokens = line[1:]
+
+            word2phones[word] = tokens
+            phonemes.update(tokens)
+
+    # Small list of additional punctuation
+    word2phones[','] = [' ']
+    word2phones[';'] = [' ']
+    word2phones['.'] = [' ']
+    word2phones['!'] = [' ']
+    word2phones['?'] = [' ']
+    word2phones['"'] = [' ']
+    word2phones['-'] = [' ']
+
+    phone2idx = {k: i for i, k in enumerate(phonemes)}
+    phone2idx[' '] = len(phone2idx)
+    phone2idx['sil'] = phone2idx[' ']  # Silence
+    phone2idx['sp'] = phone2idx[' ']  # Space
+    phone2idx['spn'] = phone2idx[' ']  # OOV/unk
+
+    dicts = {
+        'phone2idx': phone2idx,
+        'word2phones': word2phones,
+    }
+    with open(args.dict_out, 'w') as f:
+        json.dump(dicts, f)
+
+    print(f"Total number of phone indices: {len(phone2idx)}")
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,88 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Extracts energy and pitch data from LJSpeech wav files and writes them to a directory as LJxxx-xxxx.npy files.
+Assuming that the wavs are located at `<LJSpeech_base_path>/wavs`, this script will write the
+energy files to `<LJSpeech_base_path>/energies/` and the pitch files to `<LJSpeech_base_path>/pitches`,
+creating the relevant directories if necessary.
+
+USAGE: python extract_ljspeech_energy_pitch.py --ljspeech_dir=<LJSpeech_base_path>
+"""
+import argparse
+from pathlib import Path
+
+import librosa
+import numpy as np
+
+tqdm = None
+
+try:
+    from tqdm import tqdm
+except ModuleNotFoundError:
+    pass
+
+parser = argparse.ArgumentParser(
+    description="Extracts energies (L2-norm of STFT frame amplitudes) and pitches (F0) from LJSpeech data."
+)
+parser.add_argument("--ljspeech_dir", required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def main():
+    wavfile_dir = Path(args.ljspeech_dir) / "wavs"
+    wavfile_list = list(wavfile_dir.glob('*.wav'))
+
+    target_dir = Path(args.ljspeech_dir)
+    # Create target dir <LJSpeech_base_dir>/energies and <LJSpeech_base_dir>/pitches if necessary
+    if not Path(target_dir / "energies").exists():
+        print(f"Creating target directory: {target_dir/'energies'}")
+        Path(target_dir / "energies").mkdir()
+    if not Path(target_dir / "pitches").exists():
+        print(f"Creating target directory: {target_dir/'pitches'}")
+        Path(target_dir / "pitches").mkdir()
+
+    if tqdm is not None:
+        wavfile_list = tqdm(wavfile_list)
+    for count, file_ in enumerate(wavfile_list):
+        basename = Path(file_).stem
+        pitch_path = target_dir / "pitches" / f"{basename}.npy"
+        energy_path = target_dir / "energies" / f"{basename}.npy"
+        if pitch_path.exists() and energy_path.exists():
+            continue
+        audio, sr = librosa.load(file_, sr=22050)
+
+        # Calculate f0
+        # Please note that fmin and fmax are good approximates for the speaker in LJSpeech and may not generalize to
+        # other speakers
+        f0, _, _ = librosa.pyin(audio, fmin=80, fmax=800, frame_length=1024, sr=sr, fill_na=0.0)
+
+        # Save to new file
+        np.save(pitch_path, f0)
+
+        # Calculate energy
+        stft_amplitude = np.abs(librosa.stft(audio, n_fft=1024, hop_length=256, win_length=1024))
+        energy = np.linalg.norm(stft_amplitude, axis=0)  # axis=0 since librosa.stft -> (freq bins, frames)
+
+        # Save to new file
+        np.save(energy_path, energy)
+
+        assert energy.shape == f0.shape
+        if tqdm is None and count % 1000 == 0:
+            print(f"Finished processing {count} wav files...")
+
+    print(f"Finished energy extraction for a total of {len(wavfile_list)} wav files.")
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# Extracts the phonemes and alignments for the LJSpeech dataset via the
+# Montreal Forced Aligner (MFA) library, and computes durations per phoneme.
+# Assumes you have downloaded and expanded the LJSpeech dataset, and that they
+# are located at the directory specified.
+#
+# This script will create:
+# - <LJSPEECH_BASE>/mappings.json: Contains word->phone and phone->idx mappings
+# - <LJSPEECH_BASE>/wavs_to_ignore.pkl: A pickled list of wavs to ignore
+#     because of OOV words in their transcripts
+# - <LJSPEECH_BASE>/phoneme_durations/LJ*.npz: Numpy files for each wavfile
+#     with fields 'tokens' (token indices) and 'durs' (durations per token)
+#
+# It will also create intermediate files:
+# - <LJSPEECH_BASE>/alignments/wavs/LJ*.TextGrid: MFA output
+#
+# Note: This script will create a conda environment to set up MFA, so please
+#       make sure that conda is active before running this script.
+#
+# Note: You may need to install OpenBlas yourself if the MFA script can't find
+#       it. (e.g. sudo apt-get install libopenblas-dev)
+#
+# Example Usage:
+# ./extract_ljspeech_phonemes_and_durs.sh /data/LJSpeech-1.1
+
+ENV_NAME='aligner'
+
+SAMPLE_RATE=22050
+WINDOW_STRIDE=256
+
+CMUDICT_URL='https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict'
+SPLITS_BASE_URL='https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists/'
+
+# Usage info
+show_help() {
+cat << EOF
+Usage: $(basename "$0") [-h] \
+          [--skip_env_setup] \
+          [--g2p_dict=<G2P_DICT_PATH>] \
+          <LJSPEECH_BASE>
+Extracts phonemes and their respective durations for the LJSpeech dataset using
+the Montreal Forced Aligner (MFA).
+
+    -h                Help message
+    --skip_env_setup  (Optional) Skips setting up the MFA conda environment
+                      "aligner".  Use only if you already have this set up.
+    --g2p_dict        (Optional) Path to the grapheme to phoneme dictionary
+                      text file, if already generated. If set, skips the G2P
+                      step.
+EOF
+}
+
+SKIP_ENV_SETUP=false
+G2P_DICT=''
+
+while :; do
+  case $1 in
+    -h|-\?|--help)
+      show_help
+      exit
+      ;;
+    --g2p_dict=?*)
+      G2P_DICT=${1#*=}
+      ;;
+    --skip_env_setup)
+      SKIP_ENV_SETUP=true
+      ;;
+    *)
+      break
+  esac
+  shift
+done
+
+if [[ -z $1 ]]; then
+  echo "Must specify a LJSpeech base directory."
+  exit 1
+fi
+LJSPEECH_BASE=$1
+
+# Check for conda
+read -r -d '' CONDA_MESSAGE << EOM
+This script requires either Anaconda or Miniconda to be active, as it installs the Montreal Forced Aligner via a conda environment.
+See their documentation (https://montreal-forced-aligner.readthedocs.io/en/latest/installation.html) for more details.
+EOM
+if [ -z "$(which conda)" ]; then
+  echo $CONDA_MESSAGE
+  exit
+fi
+
+CONDA_PREFIX=$(conda info --base)
+source $CONDA_PREFIX/etc/profile.d/conda.sh
+
+# Set up env for MFA (env name "aligner") and install
+if $SKIP_ENV_SETUP; then
+  echo "Skipping environment setup. Assuming env name "aligner" exists."
+else
+  echo "Setting up conda environment for MFA (env name \"aligner\")..."
+  conda create -n $ENV_NAME -c conda-forge openblas python=3.8 openfst pynini ngram baumwelch
+  conda activate $ENV_NAME
+  pip install montreal-forced-aligner tgt torch
+  mfa thirdparty download
+  echo "Conda environment \"$ENV_NAME\" set up."
+fi
+
+if ! conda activate $ENV_NAME; then
+  echo "Could not activate environment, see Conda output above."
+  exit
+fi
+
+# Download CMU word-to-phoneme dict and clean out comments so they're not mistaken for tokens
+if [ -z $G2P_DICT ]; then
+  if [ ! -f $LJSPEECH_BASE/cmudict.dict ]; then
+    echo "Downloading CMU dict."
+    wget -P $LJSPEECH_BASE $CMUDICT_URL
+  fi
+  if [ ! -f $LJSPEECH_BASE/uncommented_cmudict.dict ]; then
+    echo "Creating uncommented version of CMUdict."
+    sed 's/\ \#.*//' $LJSPEECH_BASE/cmudict.dict > $LJSPEECH_BASE/uncommented_cmudict.dict
+  fi
+  G2P_DICT=$LJSPEECH_BASE/uncommented_cmudict.dict
+fi
+
+# Run alignment
+echo "Starting MFA with dictionary at: $G2P_DICT"
+
+read -r -d '' MFA_ERROR_MSG << EOM
+Could not run MFA. If it could not find OpenBlas, you may need to install it manually.
+(e.g. sudo apt-get install libopenblas-dev)
+EOM
+
+if ! mfa download acoustic english; then
+  echo $MFA_ERROR_MSG
+fi
+mfa align --clean $LJSPEECH_BASE $G2P_DICT english $LJSPEECH_BASE/alignments
+
+# Create JSON mappings from word to phonemes and phonemes to indices
+echo "Creating word->phone and phone->idx mappings at $LJSPEECH_BASE/mappings.json..."
+python create_token2idx_dict.py \
+  --dictionary=$G2P_DICT \
+  --dict_out=$LJSPEECH_BASE/mappings.json
+
+# Calculate phoneme durations
+echo "Calculating phoneme durations..."
+python calculate_durs.py \
+  --ljspeech_dir=$LJSPEECH_BASE \
+  --mappings=$LJSPEECH_BASE/mappings.json \
+  --sr=$SAMPLE_RATE \
+  --hop_length=$WINDOW_STRIDE
+echo "Phoneme durations and tokens written."
@@ -0,0 +1,88 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import argparse
+import glob
+import json
+import logging
+import os
+import subprocess
+import tarfile
+
+import librosa
+import wget
+
+parser = argparse.ArgumentParser(description="AN4 dataset download and processing")
+parser.add_argument("--data_root", required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def build_manifest(data_root, transcripts_path, manifest_path, wav_path):
+    with open(transcripts_path, 'r') as fin:
+        with open(manifest_path, 'w') as fout:
+            for line in fin:
+                # Lines look like this:
+                # <s> transcript </s> (fileID)
+                transcript = line[: line.find('(') - 1].lower()
+                transcript = transcript.replace('<s>', '').replace('</s>', '')
+                transcript = transcript.strip()
+
+                file_id = line[line.find('(') + 1 : -2]  # e.g. "cen4-fash-b"
+                audio_path = os.path.join(
+                    data_root, wav_path, file_id[file_id.find('-') + 1 : file_id.rfind('-')], file_id + '.wav',
+                )
+
+                duration = librosa.core.get_duration(filename=audio_path)
+
+                # Write the metadata to the manifest
+                metadata = {
+                    "audio_filepath": audio_path,
+                    "duration": duration,
+                    "text": transcript,
+                }
+                json.dump(metadata, fout)
+                fout.write('\n')
+
+
+def main():
+    data_root = os.path.abspath(args.data_root)
+
+    # Convert from .sph to .wav
+    logging.info("Converting audio files to .wav...")
+    sph_list = glob.glob(os.path.join(data_root, 'an4/**/*.sph'), recursive=True)
+    for sph_path in sph_list:
+        wav_path = sph_path[:-4] + '.wav'
+        cmd = ['sox', sph_path, wav_path]
+        subprocess.run(cmd)
+    logging.info("Finished conversion.")
+
+    # Build manifests
+    logging.info("Building training manifest...")
+    train_transcripts = os.path.join(data_root, 'an4/etc/an4_train.transcription')
+    train_manifest = os.path.join(data_root, 'an4/train_manifest.json')
+    train_wavs = os.path.join(data_root, 'an4/wav/an4_clstk')
+    build_manifest(data_root, train_transcripts, train_manifest, train_wavs)
+    logging.info("Training manifests created.")
+
+    logging.info("Building test manifest...")
+    test_transcripts = os.path.join(data_root, 'an4/etc/an4_test.transcription')
+    test_manifest = os.path.join(data_root, 'an4/test_manifest.json')
+    test_wavs = os.path.join(data_root, 'an4/wav/an4test_clstk')
+    build_manifest(data_root, test_transcripts, test_manifest, test_wavs)
+    logging.info("Test manifest created.")
+
+    logging.info("Done with AN4 processing!")
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,70 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import sys
+
+from nemo.collections.tts.data.datalayers import preprocess_linear_specs_dataset
+
+sys.path.insert(0, '../')
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Create dataset fitted for training and validating deep griffin iteration from wavefiles'
+    )
+    parser.add_argument(
+        "-v",
+        "--valid_filelist",
+        help="Filelist for validation set, with all validation audio files listed",
+        required=True,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-t",
+        "--train_filelist",
+        help="Filelist for train set, with all train audio files listed",
+        required=True,
+        default=None,
+        type=str,
+    )
+    parser.add_argument(
+        "-n",
+        "--n_fft",
+        help="Value for the n_fft parameter, and the filter length for the STFT",
+        default=512,
+        type=int,
+    )
+    parser.add_argument("--hop_length", help="STFT parameter", default=256, type=int)
+    parser.add_argument(
+        "-d", "--destination", help="Destination to save the preprocessed data set to", default="/tmp", type=str
+    )
+    parser.add_argument(
+        "-s",
+        "--num_snr",
+        help="Number of distinctive noisy samples to generate for each clear sample at the file list",
+        default=1,
+        type=int,
+    )
+
+    args = parser.parse_args()
+
+    preprocess_linear_specs_dataset(
+        args.valid_filelist, args.train_filelist, args.n_fft, args.hop_length, args.num_snr, args.destination
+    )
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,256 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is heavily derived from the Patter HUB5 processing script written
+# by Ryan Leary
+
+import argparse
+import glob
+import json
+import os
+import re
+import subprocess
+import sys
+from collections import namedtuple
+from math import ceil, floor
+from operator import attrgetter
+
+import numpy as np
+import scipy.io.wavfile as wavfile
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Prepare HUB5 data for training/eval")
+parser.add_argument(
+    "--data_root", default=None, type=str, required=True, help="The path to the root LDC HUB5 dataset directory.",
+)
+parser.add_argument(
+    "--dest_root",
+    default=None,
+    type=str,
+    required=True,
+    help="Path to the destination root directory for processed files.",
+)
+
+# Optional arguments
+parser.add_argument(
+    "--min_slice_duration", default=10.0, type=float, help="Minimum audio slice duration after processing.",
+)
+
+args = parser.parse_args()
+
+StmUtterance = namedtuple(
+    'StmUtterance', ['filename', 'channel', 'speaker_id', 'begin', 'end', 'label', 'transcript',],
+)
+STM_LINE_FMT = re.compile(r"^(\w+)\s+(\w+)\s+(\w+)\s+([0-9.]+)\s+([0-9.]+)\s+(<.*>)?\s+(.+)$")
+
+# Transcription errors and their fixes
+TRANSCRIPT_BUGS = {"en_4622-B-12079-12187": "KIND OF WEIRD BUT"}
+
+
+def get_utt_id(segment):
+    """
+    Gives utterance IDs in a form like: en_4156-a-36558-37113
+    """
+    return "{}-{}-{}-{}".format(segment.filename, segment.channel, int(segment.begin * 100), int(segment.end * 100),)
+
+
+def convert_utterances(sph_path, wav_path):
+    """
+    Converts a sphere audio file to wav.
+    """
+    cmd = ["sph2pipe", "-f", "wav", "-p", sph_path, wav_path]
+    subprocess.run(cmd)
+
+
+def create_wavs(data_root, dest_root):
+    """
+    Converts the English sph files to wav using sph2pipe.
+    """
+    sph_root = os.path.join(data_root, "hub5e_00", "english")
+    sph_list = glob.glob(os.path.join(sph_root, "*.sph"))
+
+    # Iterate over each sphere file and conver to wav
+    for sph_path in tqdm(sph_list, desc="Converting to wav", unit="file"):
+        sph_name, _ = os.path.splitext(os.path.basename(sph_path))
+        wav_path = os.path.join(dest_root, 'full_audio_wav', sph_name + ".wav")
+        cmd = ["sph2pipe", "-f", "wav", "-p", sph_path, wav_path]
+        subprocess.run(cmd)
+
+
+def process_transcripts(dataset_root):
+    """
+    Reads in transcripts for each audio segment and processes them.
+    """
+    stm_path = os.path.join(dataset_root, "2000_hub5_eng_eval_tr", "reference", "hub5e00.english.000405.stm",)
+    results = []
+    chars = set()
+
+    with open(stm_path, "r") as fh:
+        for line in fh:
+            # lines with ';;' are comments
+            if line.startswith(";;"):
+                continue
+
+            if "IGNORE_TIME_SEGMENT_" in line:
+                continue
+            line = line.replace("<B_ASIDE>", "").replace("<E_ASIDE>", "")
+            line = line.replace("(%HESITATION)", "UH")
+            line = line.replace("-", "")
+            line = line.replace("(%UH)", "UH")
+            line = line.replace("(%AH)", "UH")
+            line = line.replace("(", "").replace(")", "")
+
+            line = line.lower()
+
+            m = STM_LINE_FMT.search(line.strip())
+            utt = StmUtterance(*m.groups())
+
+            # Convert begin/end times to float
+            utt = utt._replace(begin=float(utt.begin))
+            utt = utt._replace(end=float(utt.end))
+
+            # Check for utterance in dict of transcript mistakes
+            transcript_update = TRANSCRIPT_BUGS.get(get_utt_id(utt))
+            if transcript_update is not None:
+                utt = utt._replace(transcript=transcript_update)
+
+            results.append(utt)
+            chars.update(list(utt.transcript))
+    return results, chars
+
+
+def write_one_segment(dest_root, speaker_id, count, audio, sr, duration, transcript):
+    """
+    Writes out one segment of audio, and writes its corresponding transcript
+    in the manifest.
+
+    Args:
+        dest_root: the path to the output directory root
+        speaker_id: ID of the speaker, used in file naming
+        count: number of segments from this speaker so far
+        audio: the segment's audio data
+        sr: sample rate of the audio
+        duration: duration of the audio
+        transcript: the corresponding transcript
+    """
+    audio_path = os.path.join(dest_root, "audio", f"{speaker_id}_{count:03}.wav")
+
+    manifest_path = os.path.join(dest_root, "manifest_hub5.json")
+
+    # Write audio
+    wavfile.write(audio_path, sr, audio)
+
+    # Write transcript
+    transcript = {
+        "audio_filepath": audio_path,
+        "duration": duration,
+        "text": transcript,
+    }
+    with open(manifest_path, 'a') as f:
+        json.dump(transcript, f)
+        f.write('\n')
+
+
+def segment_audio(info_list, dest_root, min_slice_duration):
+    """
+    Combines audio into >= min_slice_duration segments of the same speaker,
+    and writes the combined transcripts into a manifest.
+
+    Args:
+        info_list: list of StmUtterance objects with transcript information.
+        dest_root: path to output destination
+        min_slice_duration: min number of seconds per output audio slice
+    """
+    info_list = sorted(info_list, key=attrgetter('speaker_id', 'begin'))
+
+    prev_id = None  # For checking audio concatenation
+    id_count = 0
+
+    sample_rate, audio_data = None, None
+    transcript_buffer = ''
+    audio_buffer = []
+    buffer_duration = 0.0
+
+    # Iterate through utterances to build segments
+    for info in info_list:
+        if info.speaker_id != prev_id:
+            # Scrap the remainder in the buffers and start next segment
+            prev_id = info.speaker_id
+            id_count = 0
+
+            sample_rate, audio_data = wavfile.read(os.path.join(dest_root, 'full_audio_wav', info.filename + '.wav'))
+            transcript_buffer = ''
+            audio_buffer = []
+            buffer_duration = 0.0
+
+        # Append utterance info to buffers
+        transcript_buffer += info.transcript
+        channel = 0 if info.channel.lower() == 'a' else 1
+        audio_buffer.append(
+            audio_data[floor(info.begin * sample_rate) : ceil(info.end * sample_rate), channel,]
+        )
+        buffer_duration += info.end - info.begin
+
+        if buffer_duration < min_slice_duration:
+            transcript_buffer += ' '
+        else:
+            # Write out segment and transcript
+            id_count += 1
+            write_one_segment(
+                dest_root,
+                info.speaker_id,
+                id_count,
+                np.concatenate(audio_buffer, axis=0),
+                sample_rate,
+                buffer_duration,
+                transcript_buffer,
+            )
+
+            transcript_buffer = ''
+            audio_buffer = []
+            buffer_duration = 0.0
+
+
+def main():
+    data_root = args.data_root
+    dest_root = args.dest_root
+
+    min_slice_duration = args.min_slice_duration
+
+    if not os.path.exists(os.path.join(dest_root, 'full_audio_wav')):
+        os.makedirs(os.path.join(dest_root, 'full_audio_wav'))
+    if not os.path.exists(os.path.join(dest_root, 'audio')):
+        os.makedirs(os.path.join(dest_root, 'audio'))
+
+    # Create/wipe manifest contents
+    open(os.path.join(dest_root, "manifest_hub5.json"), 'w').close()
+
+    # Convert full audio files from .sph to .wav
+    create_wavs(data_root, dest_root)
+
+    # Get each audio transcript from transcript file
+    info_list, chars = process_transcripts(data_root)
+
+    print("Writing out vocab file", file=sys.stderr)
+    with open(os.path.join(dest_root, "vocab.txt"), 'w') as fh:
+        for x in sorted(list(chars)):
+            fh.write(x + "\n")
+
+    # Segment the audio data
+    print("Segmenting audio and writing manifest")
+    segment_audio(info_list, dest_root, min_slice_duration)
+
+
+if __name__ == '__main__':
+    main()
@@ -0,0 +1,402 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Usage:
+
+python process_speech_commands_data.py \
+    --data_root=<absolute path to where the data should be stored> \
+    --data_version=<either 1 or 2, indicating version of the dataset> \
+    --class_split=<either "all" or "sub", indicates whether all 30/35 classes should be used, or the 10+2 split should be used> \
+    --rebalance \
+    --log
+"""
+
+import argparse
+import glob
+import json
+import logging
+import os
+import re
+import tarfile
+import urllib.request
+
+import librosa
+import numpy as np
+
+URL_v1 = "http://download.tensorflow.org/data/speech_commands_v0.01.tar.gz"
+URL_v2 = "http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz"
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    if not os.path.exists(destination):
+        logging.info(f"{destination} does not exist. Downloading ...")
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info(f"Downloaded {destination}.")
+    else:
+        logging.info(f"Destination {destination} exists. Skipping.")
+    return destination
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_dir)
+    else:
+        logging.info(f'Skipping extracting. Data already there {data_dir}')
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        tar = tarfile.open(filepath)
+        tar.extractall(data_dir)
+        tar.close()
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __process_data(
+    data_folder: str, dst_folder: str, rebalance: bool = False, class_split: str = "all", skip_duration: bool = False
+):
+    """
+    To generate manifest
+
+    Args:
+        data_folder: source with wav files and validation / test lists
+        dst_folder: where manifest files will be stored
+        rebalance: rebalance the classes to have same number of samples.
+        class_split: whether to use all classes as distinct labels, or to use
+            10 classes subset and rest of the classes as noise or background.
+        skip_duration: Bool whether to skip duration computation. Use this only for
+            colab notebooks where knowing duration is not necessary for demonstration.
+
+    Returns:
+
+    """
+
+    if not os.path.exists(dst_folder):
+        os.makedirs(dst_folder)
+
+    # Used for 10 classes + silence + unknown class setup
+    class_subset = ["yes", "no", "up", "down", "left", "right", "on", "off", "stop", "go"]
+
+    pattern = re.compile(r"(.+\/)?(\w+)\/([^_]+)_.+wav")
+    all_files = glob.glob(os.path.join(data_folder, '*/*wav'))
+
+    with open(os.path.join(data_folder, 'validation_list.txt'), 'r') as fin:
+        validation_files = fin.readlines()
+
+    valset = set()
+    for entry in validation_files:
+        r = re.match(pattern, entry)
+        if r:
+            valset.add(r.group(3))
+
+    with open(os.path.join(data_folder, 'testing_list.txt'), 'r') as fin:
+        testing_files = fin.readlines()
+
+    testset = set()
+    for entry in testing_files:
+        r = re.match(pattern, entry)
+        if r:
+            testset.add(r.group(3))
+
+    logging.info("Validation and test set lists extracted")
+
+    label_count = {}
+    label_filepaths = {}
+    unknown_val_filepaths = []
+    unknown_test_filepaths = []
+
+    train, val, test = [], [], []
+    for entry in all_files:
+        r = re.match(pattern, entry)
+        if r:
+            label, uid = r.group(2), r.group(3)
+
+            if label == '_background_noise_' or label == 'silence':
+                continue
+
+            if class_split == "sub":
+                if label not in class_subset:
+                    label = "unknown"  # replace label
+
+                    if uid in valset:
+                        unknown_val_filepaths.append((label, entry))
+                    elif uid in testset:
+                        unknown_test_filepaths.append((label, entry))
+
+            sample = (label, entry)
+
+            if uid not in valset and uid not in testset:
+                if label in label_count:
+                    label_count[label] += 1
+                else:
+                    label_count[label] = 1
+
+                if label in label_filepaths:
+                    label_filepaths[label] += [sample]
+                else:
+                    label_filepaths[label] = [sample]
+
+            if label == 'unknown':
+                continue
+
+            if uid in valset:
+                val.append(sample)
+            elif uid in testset:
+                test.append(sample)
+            else:
+                train.append(sample)
+
+    logging.info("Prepared filepaths for dataset")
+
+    # Add silence and unknown class label samples
+    if class_split == "sub":
+        logging.info("Perforiming 10+2 class subsplit")
+
+        silence_path = os.path.join(data_folder, "silence")
+        if not os.path.exists(silence_path):
+            os.mkdir(silence_path)
+
+        silence_stride = 1000  # 0.0625 second stride
+        sampling_rate = 16000
+        folder = os.path.join(data_folder, "_background_noise_")
+
+        silence_files = []
+        rng = np.random.RandomState(0)
+
+        for file in os.listdir(folder):
+            if ".wav" in file:
+                load_path = os.path.join(folder, file)
+                y, sr = librosa.load(load_path, sr=sampling_rate)
+
+                for i in range(0, len(y) - sampling_rate, silence_stride):
+                    file_path = "silence/{}_{}.wav".format(file[:-4], i)
+                    y_slice = y[i : i + sampling_rate]
+                    magnitude = rng.uniform(0.0, 1.0)
+                    y_slice *= magnitude
+                    out_file_path = os.path.join(data_folder, file_path)
+                    librosa.output.write_wav(out_file_path, y_slice, sr)
+
+                    silence_files.append(('silence', out_file_path))
+
+        rng = np.random.RandomState(0)
+        rng.shuffle(silence_files)
+        logging.info(f"Constructed silence set of {len(silence_files)}")
+
+        # Create the splits
+        rng = np.random.RandomState(0)
+        silence_split = 0.1
+        unknown_split = 0.1
+
+        # train split
+        num_total_samples = sum([label_count[cls] for cls in class_subset])
+        num_silence_samples = int(np.ceil(silence_split * num_total_samples))
+
+        # initialize sample
+        label_count['silence'] = 0
+
+        for silence_id in range(num_silence_samples):
+            label_count['silence'] += 1
+
+            if 'silence' in label_filepaths:
+                label_filepaths['silence'] += [silence_files[silence_id]]
+            else:
+                label_filepaths['silence'] = [silence_files[silence_id]]
+
+        train.extend(label_filepaths['silence'])
+
+        # Update train unknown set
+        unknown_train_samples = label_filepaths['unknown']
+
+        rng.shuffle(unknown_train_samples)
+        unknown_size = int(np.ceil(unknown_split * num_total_samples))
+
+        label_count['unknown'] = unknown_size
+        label_filepaths['unknown'] = unknown_train_samples[:unknown_size]
+
+        train.extend(label_filepaths['unknown'])
+
+        logging.info("Train set prepared")
+
+        # val set silence
+        num_val_samples = len(val)
+        num_silence_samples = int(np.ceil(silence_split * num_val_samples))
+
+        val_idx = label_count['silence'] + 1
+        for silence_id in range(num_silence_samples):
+            val.append(silence_files[val_idx + silence_id])
+
+        # Update val unknown set
+        rng.shuffle(unknown_val_filepaths)
+        unknown_size = int(np.ceil(unknown_split * num_val_samples))
+
+        val.extend(unknown_val_filepaths[:unknown_size])
+
+        logging.info("Validation set prepared")
+
+        # test set silence
+        num_test_samples = len(test)
+        num_silence_samples = int(np.ceil(silence_split * num_test_samples))
+
+        test_idx = val_idx + num_silence_samples + 1
+        for silence_id in range(num_silence_samples):
+            test.append(silence_files[test_idx + silence_id])
+
+        # Update test unknown set
+        rng.shuffle(unknown_test_filepaths)
+        unknown_size = int(np.ceil(unknown_split * num_test_samples))
+
+        test.extend(unknown_test_filepaths[:unknown_size])
+
+        logging.info("Test set prepared")
+
+    max_command = None
+    max_count = -1
+    for command, count in label_count.items():
+        if command == 'unknown':
+            continue
+
+        if count > max_count:
+            max_count = count
+            max_command = command
+
+    if rebalance:
+        logging.info(f"Command with maximum number of samples = {max_command} with {max_count} samples")
+        logging.info(f"Rebalancing dataset by duplicating classes with less than {max_count} samples...")
+
+        for command, samples in label_filepaths.items():
+            filepaths = [sample[1] for sample in samples]
+
+            rng = np.random.RandomState(0)
+            filepaths = np.asarray(filepaths)
+            num_samples = len(filepaths)
+
+            if num_samples < max_count:
+                difference = max_count - num_samples
+                duplication_ids = rng.choice(num_samples, difference, replace=True)
+
+                filepaths = np.append(filepaths, filepaths[duplication_ids], axis=0)
+
+                logging.info(f"Extended class label {command} from {num_samples} samples to {len(filepaths)} samples")
+
+                label_filepaths[command] = [(command, filepath) for filepath in filepaths]
+
+        del train
+        train = []
+        for label, samples in label_filepaths.items():
+            train.extend(samples)
+
+    manifests = [
+        ('train_manifest.json', train),
+        ('validation_manifest.json', val),
+        ('test_manifest.json', test),
+    ]
+
+    for manifest_filename, dataset in manifests:
+        with open(os.path.join(dst_folder, manifest_filename), 'w') as fout:
+            num_files = len(dataset)
+            pct_file = num_files // 100
+            file_count = 0
+
+            logging.info(f"Preparing manifest : {manifest_filename} with #{num_files} files")
+
+            for label, audio_path in dataset:
+                if not skip_duration:
+                    duration = librosa.core.get_duration(filename=audio_path)
+                else:
+                    duration = 0.0
+
+                # Write the metadata to the manifest
+                metadata = {
+                    "audio_filepath": audio_path,
+                    "duration": duration,
+                    "command": label,
+                }
+                json.dump(metadata, fout)
+                fout.write('\n')
+                fout.flush()
+
+                file_count += 1
+                if file_count % pct_file == 0:
+                    if not skip_duration:
+                        logging.info(f"Finished serializing {file_count} / {num_files} into {manifest_filename}")
+
+        logging.info(f"Finished construction of manifest : {manifest_filename}")
+
+        if skip_duration:
+            logging.info(
+                f"\n<<NOTE>> Duration computation was skipped for demonstration purposes on Colaboratory.\n"
+                f"In order to replicate paper results and properly perform data augmentation, \n"
+                f"please recompute the manifest file without the `--skip_duration` flag !\n"
+            )
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Google Speech Command Data download')
+    parser.add_argument("--data_root", required=True, default=None, type=str)
+    parser.add_argument('--data_version', required=True, default=1, type=int, choices=[1, 2])
+    parser.add_argument('--class_split', required=False, default='all', type=str, choices=['all', 'sub'])
+    parser.add_argument('--rebalance', required=False, action='store_true')
+    parser.add_argument('--skip_duration', required=False, action='store_true')
+    parser.add_argument('--log', required=False, action='store_true')
+    parser.set_defaults(log=False, rebalance=False, skip_duration=False)
+    args = parser.parse_args()
+
+    if args.log:
+        logging.basicConfig(level=logging.DEBUG)
+
+    data_root = args.data_root
+    data_set = "google_speech_recognition_v{0}".format(args.data_version)
+    data_folder = os.path.join(data_root, data_set)
+
+    logging.info(f"Working on: {data_set}")
+
+    if args.data_version == 1:
+        URL = URL_v1
+    else:
+        URL = URL_v2
+
+    # Download and extract
+    if not os.path.exists(data_folder):
+        file_path = os.path.join(data_root, data_set + ".tar.bz2")
+        logging.info(f"Getting {data_set}")
+        __maybe_download_file(file_path, URL)
+        logging.info(f"Extracting {data_set}")
+        __extract_all_files(file_path, data_root, data_folder)
+
+    logging.info(f"Processing {data_set}")
+    __process_data(
+        data_folder,
+        data_folder,
+        rebalance=args.rebalance,
+        class_split=args.class_split,
+        skip_duration=args.skip_duration,
+    )
+    logging.info('Done!')
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,486 @@
+# Copyright (c) 2020, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Usage:
+
+python process_vad_data.py \
+    --out_dir=<output path to where the generated manifest should be stored> \
+    --speech_data_root=<path where the speech data are stored> \
+    --background_data_root=<path where the background data are stored> \
+    --rebalance_method=<'under' or 'over' or 'fixed'> \ 
+    --log
+    (Optional --demo (for demonstration in tutorial). If you want to use your own background noise data, make sure to delete --demo)
+"""
+import argparse
+import glob
+import json
+import logging
+import os
+import tarfile
+import urllib.request
+
+import librosa
+import numpy as np
+import soundfile as sf
+from sklearn.model_selection import train_test_split
+
+sr = 16000
+
+# google speech command v2
+URL = "http://download.tensorflow.org/data/speech_commands_v0.02.tar.gz"
+
+
+def __maybe_download_file(destination: str, source: str):
+    """
+    Downloads source to destination if it doesn't exist.
+    If exists, skips download
+    Args:
+        destination: local filepath
+        source: url of resource
+
+    Returns:
+
+    """
+    if not os.path.exists(destination):
+        logging.info(f"{destination} does not exist. Downloading ...")
+        urllib.request.urlretrieve(source, filename=destination + '.tmp')
+        os.rename(destination + '.tmp', destination)
+        logging.info(f"Downloaded {destination}.")
+    else:
+        logging.info(f"Destination {destination} exists. Skipping.")
+    return destination
+
+
+def extract_file(filepath: str, data_dir: str):
+    try:
+        tar = tarfile.open(filepath)
+        tar.extractall(data_dir)
+        tar.close()
+    except Exception:
+        logging.info('Not extracting. Maybe already there?')
+
+
+def __extract_all_files(filepath: str, data_root: str, data_dir: str):
+    if not os.path.exists(data_dir):
+        extract_file(filepath, data_dir)
+    else:
+        logging.info(f'Skipping extracting. Data already there {data_dir}')
+
+
+def split_train_val_test(data_dir, file_type, test_size=0.1, val_size=0.1, demo=False):
+    X = []
+    if file_type == "speech":
+        for o in os.listdir(data_dir):
+            if os.path.isdir(os.path.join(data_dir, o)) and o.split("/")[-1] != "_background_noise_":
+                X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav'))
+
+        if demo:
+            logging.info(
+                f"For Demonstration, we use {int(len(X)/100)}/{len(X)} speech data. Make sure to remove --demo flag when you actually train your model!"
+            )
+            X = np.random.choice(X, int(len(X) / 100), replace=False)
+
+    else:
+        for o in os.listdir(data_dir):
+            if os.path.isdir(os.path.join(data_dir, o)):
+                X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav'))
+            else:  # for using "_background_noise_" from google speech commands as background data
+                if o.endswith(".wav"):
+                    X.append(os.path.join(data_dir, o))
+
+    X_train, X_test = train_test_split(X, test_size=test_size, random_state=1)
+    val_size_tmp = val_size / (1 - test_size)
+    X_train, X_val = train_test_split(X_train, test_size=val_size_tmp, random_state=1)
+
+    with open(os.path.join(data_dir, file_type + "_training_list.txt"), "w") as outfile:
+        outfile.write("\n".join(X_train))
+    with open(os.path.join(data_dir, file_type + "_testing_list.txt"), "w") as outfile:
+        outfile.write("\n".join(X_test))
+    with open(os.path.join(data_dir, file_type + "_validation_list.txt"), "w") as outfile:
+        outfile.write("\n".join(X_val))
+
+    logging.info(f'Overall: {len(X)}, Train: {len(X_train)}, Validatoin: {len(X_val)}, Test: {len(X_test)}')
+    logging.info(f"Finish spliting train, val and test for {file_type}. Write to files!")
+
+
+def process_google_speech_train(data_dir):
+    X = []
+    for o in os.listdir(data_dir):
+        if os.path.isdir(os.path.join(data_dir, o)) and o.split("/")[-1] != "_background_noise_":
+            X.extend(glob.glob(os.path.join(data_dir, o) + '/*.wav'))
+
+    short_files = [i.split(data_dir)[1] for i in files]
+
+    with open(os.path.join(data_dir, 'testing_list.txt'), 'r') as allfile:
+        testing_list = allfile.read().splitlines()
+
+    with open(os.path.join(data_dir, 'validation_list.txt'), 'r') as allfile:
+        validation_list = allfile.read().splitlines()
+
+    exist_set = set(testing_list).copy()
+    exist_set.update(set(validation_list))
+
+    training_list = [i for i in short_files if i not in exist_set]
+
+    with open(os.path.join(data_dir, "training_list.txt"), "w") as outfile:
+        outfile.write("\n".join(training_list))
+
+    logging.info(
+        f'Overall: {len(files)}, Train: {len(training_list)}, Validatoin: {len(validation_list)}, Test: {len(testing_list)}'
+    )
+
+
+def write_manifest(
+    out_dir,
+    files,
+    prefix,
+    manifest_name,
+    start=0.0,
+    end=None,
+    duration_stride=1.0,
+    duration_max=None,
+    duration_limit=100.0,
+    filter_long=False,
+):
+    """
+    Given a list of files, segment each file and write them to manifest with restrictions.
+    Args:
+        out_dir: directory of generated manifest
+        files: list of files to be processed
+        prefix: label of samples
+        manifest_name: name of generated manifest
+        start: beginning of audio of generating segment
+        end: end of audio of generating segment
+        duration_stride: stride for segmenting audio samples
+        duration_max: duration for each segment
+        duration_limit: duration threshold for filtering out long audio samples
+        filter_long: boolean to determine whether to filter out long audio samples
+    Returns:
+    """
+    seg_num = 0
+    skip_num = 0
+    if duration_max is None:
+        duration_max = 1e9
+
+    if not os.path.exists(out_dir):
+        logging.info(f'Outdir {out_dir} does not exist. Creat directory.')
+        os.mkdir(out_dir)
+
+    output_path = os.path.join(out_dir, manifest_name + '.json')
+    with open(output_path, 'w') as fout:
+        for file in files:
+            label = prefix
+
+            try:
+                x, _sr = librosa.load(file, sr=sr)
+                duration = librosa.get_duration(x, sr=sr)
+
+            except Exception:
+                continue
+
+            if filter_long and duration > duration_limit:
+                skip_num += 1
+                continue
+
+            offsets = []
+            durations = []
+
+            if duration > duration_max:
+                current_offset = start
+
+                while current_offset < duration:
+                    if end is not None and current_offset > end:
+                        break
+
+                    difference = duration - current_offset
+
+                    if difference < duration_max:
+                        break
+
+                    offsets.append(current_offset)
+                    durations.append(duration_max)
+
+                    current_offset += duration_stride
+
+            else:
+                # Duration is not long enough! Skip
+                skip_num += 1
+
+            for duration, offset in zip(durations, offsets):
+                metadata = {
+                    'audio_filepath': file,
+                    'duration': duration,
+                    'label': label,
+                    'text': '_',  # for compatibility with ASRAudioText
+                    'offset': offset,
+                }
+                json.dump(metadata, fout)
+                fout.write('\n')
+                fout.flush()
+                seg_num += 1
+    return skip_num, seg_num, output_path
+
+
+def load_list_write_manifest(
+    data_dir,
+    out_dir,
+    filename,
+    prefix,
+    start,
+    end,
+    duration_stride=1.0,
+    duration_max=1.0,
+    duration_limit=100.0,
+    filter_long=True,
+):
+
+    filename = prefix + '_' + filename
+    file_path = os.path.join(data_dir, filename)
+
+    with open(file_path, 'r') as allfile:
+        files = allfile.read().splitlines()
+
+    manifest_name = filename.split('_list.txt')[0] + '_manifest'
+    skip_num, seg_num, output_path = write_manifest(
+        out_dir,
+        files,
+        prefix,
+        manifest_name,
+        start,
+        end,
+        duration_stride,
+        duration_max,
+        duration_limit,
+        filter_long=True,
+    )
+    return skip_num, seg_num, output_path
+
+
+def rebalance_json(data_dir, data_json, num, prefix):
+    data = []
+    seg = 0
+    for line in open(data_json, 'r'):
+        data.append(json.loads(line))
+    filename = data_json.split('/')[-1]
+    fout_path = os.path.join(data_dir, prefix + "_" + filename)
+
+    if len(data) >= num:
+        selected_sample = np.random.choice(data, num, replace=False)
+    else:
+        selected_sample = np.random.choice(data, num, replace=True)
+
+    with open(fout_path, 'a') as fout:
+        for i in selected_sample:
+            seg += 1
+            json.dump(i, fout)
+            fout.write('\n')
+            fout.flush()
+
+    logging.info(f'Get {seg}/{num} to  {fout_path} from {data_json}')
+    return fout_path
+
+
+def generate_variety_noise(data_dir, filename, prefix):
+
+    curr_dir = data_dir.split("_background_noise_")[0]
+    silence_path = os.path.join(curr_dir, "_background_noise_more")
+
+    if not os.path.exists(silence_path):
+        os.mkdir(silence_path)
+
+    silence_stride = 1000  # stride = 1/16 seconds
+    sampling_rate = 16000
+
+    silence_files = []
+    rng = np.random.RandomState(0)
+
+    filename = prefix + '_' + filename
+    file_path = os.path.join(data_dir, filename)
+
+    with open(file_path, 'r') as allfile:
+        files = allfile.read().splitlines()
+
+    for file in files:
+        y, sr = librosa.load(file, sr=sampling_rate)
+
+        for i in range(
+            0, len(y) - sampling_rate, silence_stride * 100
+        ):  # stride * 100 to generate less samples for demo
+            file_name = "{}_{}.wav".format(file.split("/")[-1], i)
+            y_slice = y[i : i + sampling_rate]
+            magnitude = rng.uniform(0.0, 1.0)
+            y_slice *= magnitude
+            out_file_path = os.path.join(silence_path, file_name)
+            sf.write(out_file_path, y_slice, sr)
+
+            silence_files.append(out_file_path)
+
+    new_list_file = os.path.join(silence_path, filename)
+    with open(new_list_file, "w") as outfile:
+        outfile.write("\n".join(silence_files))
+
+    logging.info(f"Generate {len(out_file_path)} background files for {file_path}. => {new_list_file} !")
+    return len(silence_files)
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Speech and backgound data download and preprocess')
+    parser.add_argument("--out_dir", required=False, default='./manifest/', type=str)
+    parser.add_argument("--speech_data_root", required=True, default=None, type=str)
+    parser.add_argument("--background_data_root", required=True, default=None, type=str)
+    parser.add_argument('--test_size', required=False, default=0.1, type=float)
+    parser.add_argument('--val_size', required=False, default=0.1, type=float)
+    parser.add_argument('--seg_len', required=False, default=0.63, type=float)
+    parser.add_argument('--log', required=False, action='store_true')
+    parser.add_argument('--rebalance_method', required=False, default=None, type=str)
+    parser.add_argument('--demo', required=False, action='store_true')
+    parser.set_defaults(log=False, generate=False)
+    args = parser.parse_args()
+
+    if not args.rebalance_method:
+        rebalance = False
+    else:
+        if args.rebalance_method != 'over' and args.rebalance_method != 'under' and args.rebalance_method != 'fixed':
+            raise NameError("Please select a valid sampling method: over/under/fixed.")
+        else:
+            rebalance = True
+
+    if args.log:
+        logging.basicConfig(level=logging.DEBUG)
+
+    # Download speech data
+    speech_data_root = args.speech_data_root
+    data_set = "google_speech_recognition_v2"
+    speech_data_folder = os.path.join(speech_data_root, data_set)
+
+    background_data_folder = args.background_data_root
+    logging.info(f"Working on: {data_set}")
+
+    # Download and extract speech data
+    if not os.path.exists(speech_data_folder):
+        file_path = os.path.join(speech_data_root, data_set + ".tar.bz2")
+        logging.info(f"Getting {data_set}")
+        __maybe_download_file(file_path, URL)
+        logging.info(f"Extracting {data_set}")
+        __extract_all_files(file_path, speech_data_root, speech_data_folder)
+
+    logging.info(f"Split speech data!")
+    # dataset provide testing.txt and validation.txt feel free to split data using that with process_google_speech_train
+    split_train_val_test(speech_data_folder, "speech", args.test_size, args.val_size, args.demo)
+
+    logging.info(f"Split background data!")
+    split_train_val_test(background_data_folder, "background", args.test_size, args.val_size)
+
+    out_dir = args.out_dir
+
+    # Process Speech manifest
+    logging.info(f"=== Write speech data to manifest!")
+    skip_num_val, speech_seg_num_val, speech_val = load_list_write_manifest(
+        speech_data_folder, out_dir, 'validation_list.txt', 'speech', 0.2, 0.8, args.seg_len, args.seg_len
+    )
+    skip_num_test, speech_seg_num_test, speech_test = load_list_write_manifest(
+        speech_data_folder, out_dir, 'testing_list.txt', 'speech', 0.2, 0.8, 0.01, args.seg_len
+    )
+    skip_num_train, speech_seg_num_train, speech_train = load_list_write_manifest(
+        speech_data_folder, out_dir, 'training_list.txt', 'speech', 0.2, 0.8, args.seg_len, args.seg_len
+    )
+
+    logging.info(f'Val: Skip {skip_num_val} samples. Get {speech_seg_num_val} segments! => {speech_val} ')
+    logging.info(f'Test: Skip {skip_num_test} samples. Get {speech_seg_num_test} segments! => {speech_test}')
+    logging.info(f'Train: Skip {skip_num_train} samples. Get {speech_seg_num_train} segments!=> {speech_train}')
+
+    # Process background manifest
+    # if we select to generate more background noise data
+    if args.demo:
+        logging.info("Start generating more background noise data")
+        generate_variety_noise(background_data_folder, 'validation_list.txt', 'background')
+        generate_variety_noise(background_data_folder, 'training_list.txt', 'background')
+        generate_variety_noise(background_data_folder, 'testing_list.txt', 'background')
+        background_data_folder = os.path.join(
+            background_data_folder.split("_background_noise_")[0], "_background_noise_more"
+        )
+
+    logging.info(f"=== Write background data to manifest!")
+    skip_num_val, background_seg_num_val, background_val = load_list_write_manifest(
+        background_data_folder, out_dir, 'validation_list.txt', 'background', 0, None, 0.15, args.seg_len
+    )
+    skip_num_test, background_seg_num_test, background_test = load_list_write_manifest(
+        background_data_folder, out_dir, 'testing_list.txt', 'background', 0, None, 0.01, args.seg_len
+    )
+    skip_num_train, background_seg_num_train, background_train = load_list_write_manifest(
+        background_data_folder, out_dir, 'training_list.txt', 'background', 0, None, 0.15, args.seg_len
+    )
+
+    logging.info(f'Val: Skip {skip_num_val} samples. Get {background_seg_num_val} segments! => {background_val}')
+    logging.info(f'Test: Skip {skip_num_test} samples. Get {background_seg_num_test} segments! => {background_test}')
+    logging.info(
+        f'Train: Skip {skip_num_train} samples. Get {background_seg_num_train} segments! => {background_train}'
+    )
+    min_val, max_val = min(speech_seg_num_val, background_seg_num_val), max(speech_seg_num_val, background_seg_num_val)
+    min_test, max_test = (
+        min(speech_seg_num_test, background_seg_num_test),
+        max(speech_seg_num_test, background_seg_num_test),
+    )
+    min_train, max_train = (
+        min(speech_seg_num_train, background_seg_num_train),
+        max(speech_seg_num_train, background_seg_num_train),
+    )
+
+    logging.info('Finish generating manifest!')
+
+    if rebalance:
+        # Random Oversampling: Randomly duplicate examples in the minority class.
+        # Random Undersampling: Randomly delete examples in the majority class.
+        if args.rebalance_method == 'under':
+            logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.")
+            logging.info(f'Val: {min_val} Test: {min_test} Train: {min_train}!')
+
+            rebalance_json(out_dir, background_val, min_val, 'balanced')
+            rebalance_json(out_dir, background_test, min_test, 'balanced')
+            rebalance_json(out_dir, background_train, min_train, 'balanced')
+
+            rebalance_json(out_dir, speech_val, min_val, 'balanced')
+            rebalance_json(out_dir, speech_test, min_test, 'balanced')
+            rebalance_json(out_dir, speech_train, min_train, 'balanced')
+
+        if args.rebalance_method == 'over':
+            logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.")
+            logging.info(f'Val: {max_val} Test: {max_test} Train: {max_train}!')
+
+            rebalance_json(out_dir, background_val, max_val, 'balanced')
+            rebalance_json(out_dir, background_test, max_test, 'balanced')
+            rebalance_json(out_dir, background_train, max_train, 'balanced')
+
+            rebalance_json(out_dir, speech_val, max_val, 'balanced')
+            rebalance_json(out_dir, speech_test, max_test, 'balanced')
+            rebalance_json(out_dir, speech_train, max_train, 'balanced')
+
+        if args.rebalance_method == 'fixed':
+            fixed_test, fixed_val, fixed_train = 200, 100, 500
+            logging.info(f"Rebalancing number of samples in classes using {args.rebalance_method} sampling.")
+            logging.info(f'Val: {fixed_val} Test: {fixed_test} Train: {fixed_train}!')
+
+            rebalance_json(out_dir, background_val, fixed_val, 'balanced')
+            rebalance_json(out_dir, background_test, fixed_test, 'balanced')
+            rebalance_json(out_dir, background_train, fixed_train, 'balanced')
+
+            rebalance_json(out_dir, speech_val, fixed_val, 'balanced')
+            rebalance_json(out_dir, speech_test, fixed_test, 'balanced')
+            rebalance_json(out_dir, speech_train, fixed_train, 'balanced')
+    else:
+        logging.info("Don't rebalance number of samples in classes.")
+
+
+if __name__ == '__main__':
+    main()