Introduce code from Rust monorepo. This is a mess

tts/NeMo/scripts/dataset_processing/ljspeech/README

@@ -0,0 +1,14 @@
+These scripts are used for extracting features from LJSpeech.
+
+You will need to run the following scripts (note that these will call the other
+scripts as necessary):
+
+- `create_manifests_and_textfiles.py`: Creates the manifest and text files that
+  are used by the Montreal Forced Aligner (MFA) library.
+- `extract_ljspeech_phonemes_and_durs.sh`: Extracts phonemes and alignments via
+  MFA and calculates phoneme durations.
+  Needs to be run with Conda active since it creates a Conda environment.
+  Also note that you may need to install OpenBlas manually (e.g. sudo apt-get
+  install libopenblas-dev), sometimes MFA can't find it if installed via Conda.
+  (This script calls `create_token2idx_dict.py` and `calculate_durs.py`.)
+- `extract_ljspeech_energy_pitch.py`

tts/NeMo/scripts/dataset_processing/ljspeech/calculate_durs.py

@@ -0,0 +1,126 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Calculates durations for LJSpeech based on MFA TextGrid alignments.
+"""
+import argparse
+import glob
+import json
+import os
+import pickle
+from math import ceil
+
+import numpy as np
+import tgt
+import torch
+from tqdm import tqdm
+
+parser = argparse.ArgumentParser(description="Calculates phoneme durations for LJSpeech from TextGrids.")
+parser.add_argument('--ljspeech_dir', required=True, default=None, type=str)
+parser.add_argument(
+    '--mappings',
+    required=False,
+    default=None,
+    type=str,
+    help='JSON file of mappings created with create_token2idx_dict.py',
+)
+parser.add_argument('--sr', required=False, default=22050, type=int)
+parser.add_argument('--hop_length', required=False, default=256, type=int)
+args = parser.parse_args()
+
+
+def calculate_durations(textgrid, phone2idx):
+    tokens = []
+    durs = []
+
+    frames_per_second = args.sr / args.hop_length
+    tg = tgt.read_textgrid(textgrid, include_empty_intervals=True)
+    data_tier = tg.get_tier_by_name("phones")
+
+    # Get total frames
+    total_frames = ceil((data_tier.end_time - data_tier.start_time) * frames_per_second)
+
+    # Find start and end frames of each token
+    se_in_frames = np.array([(frames_per_second * d.start_time, frames_per_second * d.end_time) for d in data_tier])
+    se_in_frames = np.round(se_in_frames)
+    durs = (se_in_frames[:, 1] - se_in_frames[:, 0]).astype(int)
+    blank_set = ('sil', 'sp', 'spn', '', '<unk>')
+    blank_token = " "
+
+    # merge repeated blank tokens
+    tokens, durations = [], []
+    for i in range(len(data_tier)):
+        x = data_tier[i].text
+        if x == 'spn':
+            return None, None, None
+        x = blank_token if x in blank_set else x
+
+        if len(tokens) and tokens[-1] == blank_token and x == blank_token:
+            durations[-1] += durs[i]
+        else:
+            tokens.append(x)
+            durations.append(durs[i])
+
+    tokens_enc = [phone2idx[token] for token in tokens]
+    tokens_enc, durations = torch.LongTensor(tokens_enc), torch.LongTensor(durations)
+
+    # Add rounding error to final token
+    durations[-1] += total_frames - durations.sum()
+
+    return tokens, tokens_enc, durations
+
+
+def main():
+    textgrid_list = glob.glob(os.path.join(args.ljspeech_dir, 'alignments/wavs/*.TextGrid'))
+
+    # Create target_dir if necessary
+    target_dir = os.path.join(args.ljspeech_dir, 'phoneme_durations/')
+    print(f"Calculating phoneme durations, files will be in: {target_dir}")
+
+    if not os.path.exists(target_dir):
+        print(f"Creating {target_dir}")
+        os.mkdir(target_dir)
+
+    # Read phoneme to idx mappings
+    phone2idx = None
+    if args.mappings:
+        with open(args.mappings, 'r') as f:
+            mappings = json.load(f)
+            phone2idx = mappings['phone2idx']
+
+    oov_samples = []
+
+    # Iterate through all TextGrid files
+    for textgrid in tqdm(textgrid_list):
+        basename = os.path.splitext(os.path.basename(textgrid))[0][5:]  # Chop off 'wavs_' prefix
+
+        phones_mfa, tokens_mfa, durs = calculate_durations(textgrid, phone2idx)
+
+        if phones_mfa is None:
+            oov_samples.append(basename)
+            continue
+
+        # Save to file
+        target_path = os.path.join(target_dir, f'{basename}.pt')
+        torch.save({'text_encoded': tokens_mfa, 'token_duration': durs}, target_path)
+
+    print(f"Getting rid of {len(oov_samples)} samples with OOV words.")
+    oov_target = os.path.join(args.ljspeech_dir, 'wavs_to_ignore.pkl')
+    with open(oov_target, 'wb') as f:
+        pickle.dump(oov_samples, f)
+    print(f"List of OOV samples written to: {oov_target}")
+
+
+if __name__ == '__main__':
+    main()

tts/NeMo/scripts/dataset_processing/ljspeech/create_manifests_and_textfiles.py

@@ -0,0 +1,71 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Creates the manifest and .txt transcript files for an LJSpeech split.
+The manifest will be used for training, and the .txt files are for the MFA library to find.
+"""
+
+import argparse
+import json
+import os
+
+import sox
+import wget
+
+from nemo.collections.common.parts.preprocessing import parsers
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--ljspeech_base', required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def main():
+    filelist_base = 'https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists/'
+    filelists = ['train', 'val', 'test']
+
+    # NeMo parser for text normalization
+    text_parser = parsers.make_parser(name='en')
+
+    for split in filelists:
+        # Download file list if necessary
+        filelist_path = os.path.join(args.ljspeech_base, f"ljs_audio_text_{split}_filelist.txt")
+        if not os.path.exists(filelist_path):
+            wget.download(f"{filelist_base}/ljs_audio_text_{split}_filelist.txt", out=args.ljspeech_base)
+
+        manifest_target = os.path.join(args.ljspeech_base, f"ljspeech_{split}.json")
+        with open(manifest_target, 'w') as f_out:
+            with open(filelist_path, 'r') as filelist:
+                print(f"\nCreating {manifest_target}...")
+                for line in filelist:
+                    basename = line[6:16]
+                    text = text_parser._normalize(line[21:].strip())
+
+                    # Make sure corresponding wavfile exists and write .txt transcript
+                    wav_path = os.path.join(args.ljspeech_base, 'wavs/', basename + '.wav')
+                    assert os.path.exists(wav_path)
+                    txt_path = os.path.join(args.ljspeech_base, 'wavs/', basename + '.txt')
+                    with open(txt_path, 'w') as f_txt:
+                        f_txt.write(text)
+
+                    # Write manifest entry
+                    entry = {
+                        'audio_filepath': wav_path,
+                        'duration': sox.file_info.duration(wav_path),
+                        'text': text,
+                    }
+                    f_out.write(json.dumps(entry) + '\n')
+
+
+if __name__ == '__main__':
+    main()

tts/NeMo/scripts/dataset_processing/ljspeech/create_token2idx_dict.py

@@ -0,0 +1,68 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Creates a dictionary from token to index based on dictionary .txt given.
+"""
+import argparse
+import json
+import os
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--dictionary', required=True, default=None, type=str)
+parser.add_argument('--dict_out', required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def main():
+    if not os.path.exists(args.dictionary):
+        raise FileNotFoundError(f"Could not find dictionary file {args.dictionary}")
+
+    phonemes = set()
+    word2phones = {}
+    with open(args.dictionary, 'r') as f:
+        for line in f:
+            line = line.split()
+            word = line[0]
+            tokens = line[1:]
+
+            word2phones[word] = tokens
+            phonemes.update(tokens)
+
+    # Small list of additional punctuation
+    word2phones[','] = [' ']
+    word2phones[';'] = [' ']
+    word2phones['.'] = [' ']
+    word2phones['!'] = [' ']
+    word2phones['?'] = [' ']
+    word2phones['"'] = [' ']
+    word2phones['-'] = [' ']
+
+    phone2idx = {k: i for i, k in enumerate(phonemes)}
+    phone2idx[' '] = len(phone2idx)
+    phone2idx['sil'] = phone2idx[' ']  # Silence
+    phone2idx['sp'] = phone2idx[' ']  # Space
+    phone2idx['spn'] = phone2idx[' ']  # OOV/unk
+
+    dicts = {
+        'phone2idx': phone2idx,
+        'word2phones': word2phones,
+    }
+    with open(args.dict_out, 'w') as f:
+        json.dump(dicts, f)
+
+    print(f"Total number of phone indices: {len(phone2idx)}")
+
+
+if __name__ == '__main__':
+    main()

tts/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_energy_pitch.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Extracts energy and pitch data from LJSpeech wav files and writes them to a directory as LJxxx-xxxx.npy files.
+Assuming that the wavs are located at `<LJSpeech_base_path>/wavs`, this script will write the
+energy files to `<LJSpeech_base_path>/energies/` and the pitch files to `<LJSpeech_base_path>/pitches`,
+creating the relevant directories if necessary.
+
+USAGE: python extract_ljspeech_energy_pitch.py --ljspeech_dir=<LJSpeech_base_path>
+"""
+import argparse
+from pathlib import Path
+
+import librosa
+import numpy as np
+
+tqdm = None
+
+try:
+    from tqdm import tqdm
+except ModuleNotFoundError:
+    pass
+
+parser = argparse.ArgumentParser(
+    description="Extracts energies (L2-norm of STFT frame amplitudes) and pitches (F0) from LJSpeech data."
+)
+parser.add_argument("--ljspeech_dir", required=True, default=None, type=str)
+args = parser.parse_args()
+
+
+def main():
+    wavfile_dir = Path(args.ljspeech_dir) / "wavs"
+    wavfile_list = list(wavfile_dir.glob('*.wav'))
+
+    target_dir = Path(args.ljspeech_dir)
+    # Create target dir <LJSpeech_base_dir>/energies and <LJSpeech_base_dir>/pitches if necessary
+    if not Path(target_dir / "energies").exists():
+        print(f"Creating target directory: {target_dir/'energies'}")
+        Path(target_dir / "energies").mkdir()
+    if not Path(target_dir / "pitches").exists():
+        print(f"Creating target directory: {target_dir/'pitches'}")
+        Path(target_dir / "pitches").mkdir()
+
+    if tqdm is not None:
+        wavfile_list = tqdm(wavfile_list)
+    for count, file_ in enumerate(wavfile_list):
+        basename = Path(file_).stem
+        pitch_path = target_dir / "pitches" / f"{basename}.npy"
+        energy_path = target_dir / "energies" / f"{basename}.npy"
+        if pitch_path.exists() and energy_path.exists():
+            continue
+        audio, sr = librosa.load(file_, sr=22050)
+
+        # Calculate f0
+        # Please note that fmin and fmax are good approximates for the speaker in LJSpeech and may not generalize to
+        # other speakers
+        f0, _, _ = librosa.pyin(audio, fmin=80, fmax=800, frame_length=1024, sr=sr, fill_na=0.0)
+
+        # Save to new file
+        np.save(pitch_path, f0)
+
+        # Calculate energy
+        stft_amplitude = np.abs(librosa.stft(audio, n_fft=1024, hop_length=256, win_length=1024))
+        energy = np.linalg.norm(stft_amplitude, axis=0)  # axis=0 since librosa.stft -> (freq bins, frames)
+
+        # Save to new file
+        np.save(energy_path, energy)
+
+        assert energy.shape == f0.shape
+        if tqdm is None and count % 1000 == 0:
+            print(f"Finished processing {count} wav files...")
+
+    print(f"Finished energy extraction for a total of {len(wavfile_list)} wav files.")
+
+
+if __name__ == '__main__':
+    main()

tts/NeMo/scripts/dataset_processing/ljspeech/extract_ljspeech_phonemes_and_durs.sh

@@ -0,0 +1,150 @@
+#!/bin/bash
+
+# Extracts the phonemes and alignments for the LJSpeech dataset via the
+# Montreal Forced Aligner (MFA) library, and computes durations per phoneme.
+# Assumes you have downloaded and expanded the LJSpeech dataset, and that they
+# are located at the directory specified.
+#
+# This script will create:
+# - <LJSPEECH_BASE>/mappings.json: Contains word->phone and phone->idx mappings
+# - <LJSPEECH_BASE>/wavs_to_ignore.pkl: A pickled list of wavs to ignore
+#     because of OOV words in their transcripts
+# - <LJSPEECH_BASE>/phoneme_durations/LJ*.npz: Numpy files for each wavfile
+#     with fields 'tokens' (token indices) and 'durs' (durations per token)
+#
+# It will also create intermediate files:
+# - <LJSPEECH_BASE>/alignments/wavs/LJ*.TextGrid: MFA output
+#
+# Note: This script will create a conda environment to set up MFA, so please
+#       make sure that conda is active before running this script.
+#
+# Note: You may need to install OpenBlas yourself if the MFA script can't find
+#       it. (e.g. sudo apt-get install libopenblas-dev)
+#
+# Example Usage:
+# ./extract_ljspeech_phonemes_and_durs.sh /data/LJSpeech-1.1
+
+ENV_NAME='aligner'
+
+SAMPLE_RATE=22050
+WINDOW_STRIDE=256
+
+CMUDICT_URL='https://raw.githubusercontent.com/cmusphinx/cmudict/master/cmudict.dict'
+SPLITS_BASE_URL='https://raw.githubusercontent.com/NVIDIA/tacotron2/master/filelists/'
+
+# Usage info
+show_help() {
+cat << EOF
+Usage: $(basename "$0") [-h] \
+          [--skip_env_setup] \
+          [--g2p_dict=<G2P_DICT_PATH>] \
+          <LJSPEECH_BASE>
+Extracts phonemes and their respective durations for the LJSpeech dataset using
+the Montreal Forced Aligner (MFA).
+
+    -h                Help message
+    --skip_env_setup  (Optional) Skips setting up the MFA conda environment
+                      "aligner".  Use only if you already have this set up.
+    --g2p_dict        (Optional) Path to the grapheme to phoneme dictionary
+                      text file, if already generated. If set, skips the G2P
+                      step.
+EOF
+}
+
+SKIP_ENV_SETUP=false
+G2P_DICT=''
+
+while :; do
+  case $1 in
+    -h|-\?|--help)
+      show_help
+      exit
+      ;;
+    --g2p_dict=?*)
+      G2P_DICT=${1#*=}
+      ;;
+    --skip_env_setup)
+      SKIP_ENV_SETUP=true
+      ;;
+    *)
+      break
+  esac
+  shift
+done
+
+if [[ -z $1 ]]; then
+  echo "Must specify a LJSpeech base directory."
+  exit 1
+fi
+LJSPEECH_BASE=$1
+
+# Check for conda
+read -r -d '' CONDA_MESSAGE << EOM
+This script requires either Anaconda or Miniconda to be active, as it installs the Montreal Forced Aligner via a conda environment.
+See their documentation (https://montreal-forced-aligner.readthedocs.io/en/latest/installation.html) for more details.
+EOM
+if [ -z "$(which conda)" ]; then
+  echo $CONDA_MESSAGE
+  exit
+fi
+
+CONDA_PREFIX=$(conda info --base)
+source $CONDA_PREFIX/etc/profile.d/conda.sh
+
+# Set up env for MFA (env name "aligner") and install
+if $SKIP_ENV_SETUP; then
+  echo "Skipping environment setup. Assuming env name "aligner" exists."
+else
+  echo "Setting up conda environment for MFA (env name \"aligner\")..."
+  conda create -n $ENV_NAME -c conda-forge openblas python=3.8 openfst pynini ngram baumwelch
+  conda activate $ENV_NAME
+  pip install montreal-forced-aligner tgt torch
+  mfa thirdparty download
+  echo "Conda environment \"$ENV_NAME\" set up."
+fi
+
+if ! conda activate $ENV_NAME; then
+  echo "Could not activate environment, see Conda output above."
+  exit
+fi
+
+# Download CMU word-to-phoneme dict and clean out comments so they're not mistaken for tokens
+if [ -z $G2P_DICT ]; then
+  if [ ! -f $LJSPEECH_BASE/cmudict.dict ]; then
+    echo "Downloading CMU dict."
+    wget -P $LJSPEECH_BASE $CMUDICT_URL
+  fi
+  if [ ! -f $LJSPEECH_BASE/uncommented_cmudict.dict ]; then
+    echo "Creating uncommented version of CMUdict."
+    sed 's/\ \#.*//' $LJSPEECH_BASE/cmudict.dict > $LJSPEECH_BASE/uncommented_cmudict.dict
+  fi
+  G2P_DICT=$LJSPEECH_BASE/uncommented_cmudict.dict
+fi
+
+# Run alignment
+echo "Starting MFA with dictionary at: $G2P_DICT"
+
+read -r -d '' MFA_ERROR_MSG << EOM
+Could not run MFA. If it could not find OpenBlas, you may need to install it manually.
+(e.g. sudo apt-get install libopenblas-dev)
+EOM
+
+if ! mfa download acoustic english; then
+  echo $MFA_ERROR_MSG
+fi
+mfa align --clean $LJSPEECH_BASE $G2P_DICT english $LJSPEECH_BASE/alignments
+
+# Create JSON mappings from word to phonemes and phonemes to indices
+echo "Creating word->phone and phone->idx mappings at $LJSPEECH_BASE/mappings.json..."
+python create_token2idx_dict.py \
+  --dictionary=$G2P_DICT \
+  --dict_out=$LJSPEECH_BASE/mappings.json
+
+# Calculate phoneme durations
+echo "Calculating phoneme durations..."
+python calculate_durs.py \
+  --ljspeech_dir=$LJSPEECH_BASE \
+  --mappings=$LJSPEECH_BASE/mappings.json \
+  --sr=$SAMPLE_RATE \
+  --hop_length=$WINDOW_STRIDE
+echo "Phoneme durations and tokens written."