storyteller-ml/tts/tacotron2-cac/utils/tools.py

import re
import os
import json
import yaml

import torch
import torch.nn.functional as F
import numpy as np
import matplotlib
from scipy.io import wavfile
from matplotlib import pyplot as plt
from sklearn.manifold import TSNE
import audio as Audio


matplotlib.use("Agg")


device = torch.device("cuda" if torch.cuda.is_available() else "cpu")


def get_configs_of(dataset):
    config_dir = os.path.join("./config", dataset)
    preprocess_config = yaml.load(open(
        os.path.join(config_dir, "preprocess.yaml"), "r"), Loader=yaml.FullLoader)
    model_config = yaml.load(open(
        os.path.join(config_dir, "model.yaml"), "r"), Loader=yaml.FullLoader)
    train_config = yaml.load(open(
        os.path.join(config_dir, "train.yaml"), "r"), Loader=yaml.FullLoader)
    return preprocess_config, model_config, train_config


def to_device(data, device, mel_stats=None):
    if len(data) == 12:
        (
            ids,
            raw_texts,
            speakers,
            texts,
            src_lens,
            max_src_len,
            mels,
            mel_lens,
            max_mel_len,
            r_len_pad,
            gates,
            spker_embeds,
        ) = data

        speakers = torch.from_numpy(speakers).long().to(device)
        texts = torch.from_numpy(texts).long().to(device)
        src_lens = torch.from_numpy(src_lens).to(device)
        mels = torch.from_numpy(mels).float().to(device)
        if mel_stats is not None:
            mels = Audio.tools.mel_normalize(mels, *mel_stats)
        mel_lens = torch.from_numpy(mel_lens).to(device)
        gates = torch.from_numpy(gates).float().to(device)
        if spker_embeds is not None:
            spker_embeds = torch.from_numpy(spker_embeds).float().to(device)

        return (
            ids,
            raw_texts,
            speakers,
            texts,
            src_lens,
            max_src_len,
            mels,
            mel_lens,
            max_mel_len,
            r_len_pad,
            gates,
            spker_embeds,
        )

    if len(data) == 7:
        (ids, raw_texts, speakers, texts, src_lens, max_src_len, spker_embeds) = data

        speakers = torch.from_numpy(speakers).long().to(device)
        texts = torch.from_numpy(texts).long().to(device)
        src_lens = torch.from_numpy(src_lens).to(device)
        if spker_embeds is not None:
            spker_embeds = torch.from_numpy(spker_embeds).float().to(device)

        return (ids, raw_texts, speakers, texts, src_lens, max_src_len, spker_embeds)


def log(
    logger, step=None, losses=None, grad_norm=None, fig=None, audio=None, sampling_rate=22050, tag=""
):
    if losses is not None:
        logger.add_scalar("Loss/total_loss", losses[0], step)
        logger.add_scalar("Loss/mel_loss", losses[1], step)
        logger.add_scalar("Loss/gate_loss", losses[2], step)
        logger.add_scalar("Loss/attn_loss", losses[3], step)

    if grad_norm is not None:
        logger.add_scalar("Grad/grad_norm", grad_norm, step)

    if fig is not None:
        logger.add_figure(tag, fig, step)

    if audio is not None:
        logger.add_audio(
            tag,
            audio / max(abs(audio)),
            sample_rate=sampling_rate,
        )


def get_mask_from_lengths(lengths, max_len=-1):
    max_len = max(torch.max(lengths).item(), max_len)
    ids = torch.arange(0, max_len, out=torch.cuda.LongTensor(max_len))
    mask = (ids < lengths.unsqueeze(1))
    return mask


def read_lexicon(lex_path):
    lexicon = {}
    with open(lex_path) as f:
        for line in f:
            temp = re.split(r"\s+", line.strip("\n"))
            word = temp[0]
            phones = temp[1:]
            if word.lower() not in lexicon:
                lexicon[word.lower()] = phones
    return lexicon


def synth_one_sample(targets, predictions, vocoder, mel_stats, model_config, preprocess_config, step):

    normalize = preprocess_config["preprocessing"]["mel"]["normalize"]
    n_frames_per_step = model_config["decoder"]["n_frames_per_step"]
    basename = targets[0][0]
    src_len = targets[4][0].item()
    mel_len = targets[7][0].item()
    reduced_mel_len = mel_len // n_frames_per_step
    mel_target = targets[6][0, :mel_len].detach().transpose(0, 1)
    mel_prediction = predictions[1][0, :mel_len].detach().transpose(0, 1)
    attention = predictions[3][0, :reduced_mel_len, :src_len].detach().transpose(0, 1) # [seq_len, mel_len]
    gate_target = targets[10][0].detach()
    gate_prediction = predictions[2][0].detach()

    if normalize:
        mel_target = Audio.tools.mel_denormalize(mel_target, *mel_stats)
        mel_prediction = Audio.tools.mel_denormalize(mel_prediction, *mel_stats)

    fig = plot_mel(
        [
            mel_prediction.cpu().numpy(),
            mel_target.cpu().numpy(),
            attention.cpu().numpy(),
        ],
        ["Synthetized Spectrogram", "Ground-Truth Spectrogram", "Decoder Alignment"],
        attention=True,
    )

    gate_fig, _ = plot_gate_outputs(
        gate_target.cpu().numpy(),
        torch.sigmoid(gate_prediction).cpu().numpy(),
        subtitle="step_{}_{}".format(step, basename)
    )

    if vocoder is not None:
        from .model import vocoder_infer

        wav_reconstruction = vocoder_infer(
            mel_target.unsqueeze(0),
            vocoder,
            model_config,
            preprocess_config,
        )[0]
        wav_prediction = vocoder_infer(
            mel_prediction.unsqueeze(0),
            vocoder,
            model_config,
            preprocess_config,
        )[0]
    else:
        wav_reconstruction = wav_prediction = None

    return fig, gate_fig, wav_reconstruction, wav_prediction, basename


def infer_one_sample(targets, predictions, vocoder, mel_stats, model_config, preprocess_config, path, args):

    normalize = preprocess_config["preprocessing"]["mel"]["normalize"]
    n_frames_per_step = model_config["decoder"]["n_frames_per_step"]
    multi_speaker = model_config["multi_speaker"]
    mel_predictions = predictions[1]
    if normalize:
        mel_predictions = Audio.tools.mel_denormalize(predictions[1], *mel_stats)

    basename = targets[0][0]
    src_len = targets[4][0].item()
    mel_len = mel_predictions[0].shape[0]
    reduced_mel_len = mel_len // n_frames_per_step
    mel_prediction = mel_predictions[0].detach().transpose(0, 1)
    attention = predictions[3][0, :reduced_mel_len, :src_len].detach().transpose(0, 1) # [seq_len, mel_len]

    fig = plot_mel(
        [
            mel_prediction.cpu().numpy(),
            attention.cpu().numpy(),
        ],
        ["Synthetized Spectrogram", "Decoder Alignment"],
        attention=True,
    )
    plt.savefig(os.path.join(
        path, str(args.restore_step), "{}_{}.png".format(basename, args.speaker_id)\
             if multi_speaker and args.mode == "single" else "{}.png".format(basename)))
    plt.close()

    from .model import vocoder_infer

    lengths = mel_len * preprocess_config["preprocessing"]["stft"]["hop_length"]
    wav_predictions = vocoder_infer(
        mel_predictions.transpose(1, 2), vocoder, model_config, preprocess_config, lengths=[lengths]
    )

    sampling_rate = preprocess_config["preprocessing"]["audio"]["sampling_rate"]
    wavfile.write(os.path.join(
        path, str(args.restore_step), "{}_{}.wav".format(basename, args.speaker_id)\
             if multi_speaker and args.mode == "single" else "{}.wav".format(basename)),
        sampling_rate, wav_predictions[0])

def infer_one_sample_s(targets, predictions, vocoder, mel_stats, model_config, preprocess_config, path, args):

    normalize = preprocess_config["preprocessing"]["mel"]["normalize"]
    n_frames_per_step = model_config["decoder"]["n_frames_per_step"]
    multi_speaker = model_config["multi_speaker"]
    mel_predictions = predictions[1]
    if normalize:
        mel_predictions = Audio.tools.mel_denormalize(predictions[1], *mel_stats)

    basename = targets[0][0]
    src_len = targets[4][0].item()
    mel_len = mel_predictions[0].shape[0]
    reduced_mel_len = mel_len // n_frames_per_step
    mel_prediction = mel_predictions[0].detach().transpose(0, 1)
    attention = predictions[3][0, :reduced_mel_len, :src_len].detach().transpose(0, 1) # [seq_len, mel_len]

    fig = plot_mel(
        [
            mel_prediction.cpu().numpy(),
            attention.cpu().numpy(),
        ],
        ["Synthetized Spectrogram", "Decoder Alignment"],
        attention=True,
    )
    plt.savefig(os.path.join(
        path, "{}_{}.png".format(basename, args.speaker_id)\
             if multi_speaker and args.mode == "single" else "{}.png".format(basename)))
    plt.close()

    from .model import vocoder_infer

    lengths = mel_len * preprocess_config["preprocessing"]["stft"]["hop_length"]
    wav_predictions = vocoder_infer(
        mel_predictions.transpose(1, 2), vocoder, model_config, preprocess_config, lengths=[lengths]
    )

    sampling_rate = preprocess_config["preprocessing"]["audio"]["sampling_rate"]
    wavfile.write(os.path.join(
        path, "{}_{}.wav".format(basename, args.speaker_id)\
             if multi_speaker and args.mode == "single" else "{}.wav".format(basename)),
        sampling_rate, wav_predictions[0])
        
        
def plot_mel(data, titles, attention=False):
    fig, axes = plt.subplots(len(data), 1, squeeze=False)
    if titles is None:
        titles = [None for i in range(len(data))]

    def add_axis(fig, old_ax):
        ax = fig.add_axes(old_ax.get_position(), anchor="W")
        ax.set_facecolor("None")
        return ax

    for i in range(len(data)):
        if i == len(data)-1 and attention:
            im = axes[i][0].imshow(data[i], origin='lower', aspect='auto')
            axes[i][0].set_xlabel('Audio timestep')
            axes[i][0].set_ylabel('Text timestep')
            axes[i][0].set_xlim(0, data[i].shape[1])
            axes[i][0].set_ylim(0, data[i].shape[0])
            axes[i][0].set_title(titles[i], fontsize="medium")
            axes[i][0].tick_params(labelsize="x-small")
            axes[i][0].set_anchor("W")
            fig.colorbar(im, ax=axes[i][0])
            break
        mel = data[i]
        axes[i][0].imshow(mel, origin="lower")
        axes[i][0].set_aspect(2.5, adjustable="box")
        axes[i][0].set_ylim(0, mel.shape[0])
        axes[i][0].set_title(titles[i], fontsize="medium")
        axes[i][0].tick_params(labelsize="x-small", left=False, labelleft=False)
        axes[i][0].set_anchor("W")

    return fig


def plot_gate_outputs(gate_targets, gate_outputs, subtitle=None, to_numpy=False):
    data = None
    fig, ax = plt.subplots(figsize=(12, 3))
    ax.scatter(range(len(gate_targets)), gate_targets, alpha=0.5,
               color='green', marker='+', s=1, label='target')
    ax.scatter(range(len(gate_outputs)), gate_outputs, alpha=0.5,
               color='red', marker='.', s=1, label='predicted')

    plt.xlabel("Frames (Green target, Red predicted)")
    plt.ylabel("Gate State")
    plt.tight_layout()

    fig.canvas.draw()
    if subtitle is not None:
        fig.suptitle(subtitle)
    if to_numpy:
        data = save_figure_to_numpy(fig)
    plt.close()
    return fig, data


def save_figure_to_numpy(fig):
    # save it to a numpy array.
    data = np.fromstring(fig.canvas.tostring_rgb(), dtype=np.uint8, sep='')
    data = data.reshape(fig.canvas.get_width_height()[::-1] + (3,))
    return data


def save_mel_and_audio(spectrogram, audio, sampling_rate, out_dir, basename, tag=None):
    # Save mel
    plt.imshow(spectrogram, origin='lower')
    plt.ylim(0, spectrogram.shape[0])
    plt.tick_params(labelsize='x-small',
                        left=False, labelleft=False)
    plt.tight_layout()
    plt.savefig(os.path.join(out_dir, '{}_{}.png'.format(basename, tag)\
            if tag is not None else '{}.png'.format(basename)), dpi=200)
    plt.close()

    # Save audio
    wavfile.write(
        os.path.join(out_dir, "{}_{}.wav".format(basename, tag)\
            if tag is not None else '{}.wav'.format(basename)),
        sampling_rate,
        audio.astype(np.int16),
    )


def plot_embedding(out_dir, embedding, embedding_speaker_id, gender_dict, filename='embedding.png'):
    colors = 'r','b'
    labels = 'Female','Male'

    data_x = embedding
    data_y = np.array([gender_dict[spk_id] == 'M' for spk_id in embedding_speaker_id], dtype=np.int)
    tsne_model = TSNE(n_components=2, random_state=0, init='random')
    tsne_all_data = tsne_model.fit_transform(data_x)
    tsne_all_y_data = data_y

    plt.figure(figsize=(10,10))
    for i, (c, label) in enumerate(zip(colors, labels)):
        plt.scatter(tsne_all_data[tsne_all_y_data==i,0], tsne_all_data[tsne_all_y_data==i,1], c=c, label=label, alpha=0.5)

    plt.grid(True)
    plt.legend(loc='upper left')

    plt.tight_layout()
    plt.savefig(os.path.join(out_dir, filename))


def pad_1D(inputs, PAD=0):
    def pad_data(x, length, PAD):
        x_padded = np.pad(
            x, (0, length - x.shape[0]), mode="constant", constant_values=PAD
        )
        return x_padded

    max_len = max((len(x) for x in inputs))
    padded = np.stack([pad_data(x, max_len, PAD) for x in inputs])

    return padded


def pad_2D(inputs, maxlen=None):
    def pad(x, max_len):
        PAD = 0
        if np.shape(x)[0] > max_len:
            raise ValueError("not max_len")

        s = np.shape(x)[1]
        x_padded = np.pad(
            x, (0, max_len - np.shape(x)[0]), mode="constant", constant_values=PAD
        )
        return x_padded[:, :s]

    if maxlen:
        output = np.stack([pad(x, maxlen) for x in inputs])
    else:
        max_len = max(np.shape(x)[0] for x in inputs)
        output = np.stack([pad(x, max_len) for x in inputs])

    return output


def pad(input_ele, mel_max_length=None):
    if mel_max_length:
        max_len = mel_max_length
    else:
        max_len = max([input_ele[i].size(0) for i in range(len(input_ele))])

    out_list = list()
    for i, batch in enumerate(input_ele):
        if len(batch.shape) == 1:
            one_batch_padded = F.pad(
                batch, (0, max_len - batch.size(0)), "constant", 0.0
            )
        elif len(batch.shape) == 2:
            one_batch_padded = F.pad(
                batch, (0, 0, 0, max_len - batch.size(0)), "constant", 0.0
            )
        out_list.append(one_batch_padded)
    out_padded = torch.stack(out_list)
    return out_padded