storyteller-ml/tts/tacotron2-cac/model/Tacotron2.py

import os
import json
from math import sqrt

import torch
import torch.nn as nn
import torch.nn.functional as F

from .blocks import LinearNorm
from .modules import Encoder, Decoder, Postnet, AlignmentEncoder, sequence_mask, binarize_attention_parallel
from utils.tools import get_mask_from_lengths
from text.symbols import symbols


class Tacotron2(nn.Module):
    """ Tacotron2 """

    def __init__(self, preprocess_config, model_config, train_config):
        super(Tacotron2, self).__init__()
        self.model_config = model_config
        n_symbols = len(symbols) + 1

        self.mask_padding = train_config["optimizer"]["mask_padding"]
        self.fp16_run = train_config["optimizer"]["fp16_run"]
        self.n_mel_channels = preprocess_config["preprocessing"]["mel"]["n_mel_channels"]
        self.embedding = nn.Embedding(
            n_symbols, model_config["encoder"]["symbols_embedding_dim"])
        std = sqrt(2.0 / (n_symbols  + model_config["encoder"]["symbols_embedding_dim"]))
        val = sqrt(3.0) * std  # uniform bounds for std
        self.embedding.weight.data.uniform_(-val, val)
        self.encoder = Encoder(model_config)
        self.decoder = Decoder(preprocess_config, model_config)
        self.postnet = Postnet(preprocess_config, model_config)
        self.aligner = AlignmentEncoder(n_mel_channels=80, n_text_channels=model_config["encoder"]["symbols_embedding_dim"], n_att_channels=80, temperature=0.0005)
      

        self.speaker_emb = None
        if model_config["multi_speaker"]:
            self.embedder_type = preprocess_config["preprocessing"]["speaker_embedder"]
            if self.embedder_type == "none":
                with open(
                    os.path.join(
                        preprocess_config["path"]["preprocessed_path"], "speakers.json"
                    ),
                    "r",
                ) as f:
                    n_speaker = len(json.load(f))
                self.speaker_emb = nn.Embedding(
                    n_speaker,
                    model_config["encoder"]["encoder_embedding_dim"],
                )
            else:
                self.speaker_emb = LinearNorm(
                    model_config["encoder"]["speaker_embeddint_dim"],
                    model_config["encoder"]["encoder_embedding_dim"],
                )

    def parse_output(self, outputs, output_lengths=None):
        if self.mask_padding and output_lengths is not None:
            mask = ~get_mask_from_lengths(output_lengths, outputs[0].size(2))
            mask = mask.expand(self.n_mel_channels, mask.size(0), mask.size(1))
            mask = mask.permute(1, 0, 2)

            outputs[0].data.masked_fill_(mask, 0.0)
            outputs[1].data.masked_fill_(mask, 0.0)
            outputs[2].data.masked_fill_(mask[:, 0, :], 1e3)  # gate energies

        outputs[0] = outputs[0].transpose(-2, -1)
        outputs[1] = outputs[1].transpose(-2, -1)
        return outputs

    
    
    @torch.jit.unused
    def run_aligner(self, text, text_len, text_mask, spect, spect_len, attn_prior):
      text_emb = self.embedding(text)
      text_emb = text_emb.permute(0, 2, 1)
      text_mask = text_mask.permute(0, 2, 1) # [b, 1, mxlen] => [b, mxlen, 1]
      spect = spect.permute(0,2,1)  #[b, mel_len, channels] => [b, channels, mel_len]
      attn_soft, attn_logprob = self.aligner(
          spect, text_emb, mask=text_mask == 0, attn_prior=attn_prior,conditioning=None
      )
      attn_hard = binarize_attention_parallel(attn_soft, text_len, spect_len)
      attn_hard_dur = attn_hard.sum(2)
      return attn_soft, attn_logprob, attn_hard, attn_hard_dur

    def forward(
        self,
        speakers,
        texts,
        src_lens,
        max_src_len,
        mels,
        mel_lens,
        r_len_pad=None,
        max_mel_len=None,
        gates=None,
        spker_embeds=None,
    ):
        text_masks = torch.unsqueeze(sequence_mask(src_lens, texts.size(1)), 1)
        embedded_inputs = self.embedding(texts).transpose(1, 2)
        attn_soft, attn_logprob, attn_hard, attn_hard_dur = self.run_aligner(texts,src_lens,text_masks,mels,mel_lens,None)
        
        
        encoder_outputs = self.encoder(embedded_inputs, src_lens)

        if self.speaker_emb is not None:
            if self.embedder_type == "none":
                encoder_outputs = encoder_outputs + self.speaker_emb(speakers).unsqueeze(1).expand(
                    -1, max_src_len, -1
                )
            else:
                assert spker_embeds is not None, "Speaker embedding should not be None"
                encoder_outputs = encoder_outputs + self.speaker_emb(spker_embeds).unsqueeze(1).expand(
                    -1, max_src_len, -1
                )

        mel_outputs, gate_outputs, alignments = self.decoder(
            encoder_outputs, mels.transpose(-2, -1), memory_lengths=src_lens)

        mel_outputs_postnet = self.postnet(mel_outputs)
        mel_outputs_postnet = mel_outputs + mel_outputs_postnet

        return self.parse_output(
            [mel_outputs, mel_outputs_postnet, gate_outputs, alignments, attn_logprob, attn_hard],
            mel_lens)

    def inference(
        self,
        speakers,
        texts,
        max_src_len=None,
        spker_embeds=None,
    ):
        embedded_inputs = self.embedding(texts).transpose(1, 2)
        encoder_outputs = self.encoder.inference(embedded_inputs)

        if self.speaker_emb is not None:
            if self.embedder_type == "none":
                encoder_outputs = encoder_outputs + self.speaker_emb(speakers).unsqueeze(1).expand(
                    -1, max_src_len, -1
                )
            else:
                assert spker_embeds is not None, "Speaker embedding should not be None"
                encoder_outputs = encoder_outputs + self.speaker_emb(spker_embeds).unsqueeze(1).expand(
                    -1, max_src_len, -1
                )

        mel_outputs, gate_outputs, alignments = self.decoder.inference(
            encoder_outputs)

        mel_outputs_postnet = self.postnet(mel_outputs)
        mel_outputs_postnet = mel_outputs + mel_outputs_postnet

        return self.parse_output(
            [mel_outputs, mel_outputs_postnet, gate_outputs, alignments])