storyteller-ml/tts/vocodes_inference.py

#!/usr/bin/env python

"""
Vocodes Tacotron Inference Engine
This is a work in progress and is likely going to be slow for now.
"""

import argparse

parser = argparse.ArgumentParser(description='Run TTS inference')

parser.add_argument('--synthesizer_checkpoint_path', type=str, help='path the TTS synthesizer model', required=True)
parser.add_argument('--vocoder_checkpoint_path', type=str, help='path the TTS vocoder model', required=True)
parser.add_argument('--input_text_filename', type=str, help='path the file containing text to run', required=True)
parser.add_argument('--output_audio_filename', type=str, help='where to save result audio', required=True)
parser.add_argument('--output_spectrogram_filename', type=str, help='where to save result spectrogram', required=True)
parser.add_argument('--output_metadata_filename', type=str, help='where to save extra metadata', required=True)

args = parser.parse_args()

import sys
sys.path.append('waveglow/')
import shutil
import json

import numpy as np
import torch

from hparams import create_hparams
from model import Tacotron2
from layers import TacotronSTFT
from audio_processing import griffin_lim
from text import text_to_sequence
from denoiser import Denoiser

# For metadata
import subprocess
import magic
import os

print('========================================')
print('Python interpreter', sys.executable)
print('PyTorch version', torch.__version__)
print('CUDA Available?', torch.cuda.is_available())
print('CUDA Device count', torch.cuda.device_count())
print('========================================', flush=True)

# NB(bt, 2021-05-31): Trying to get everything on the same device
#torch.device("cuda" if torch.cuda.is_available() else "cpu")

# Fix bug running on CPU?
#torch.set_default_dtype(torch.float16)
# NB(bt, 2021-05-31): Perhaps this is blocking waveglow? It complains of the wrong tensor type.
#torch.set_default_tensor_type('torch.DoubleTensor') # 2021-05-31 commented out
#torch.set_default_tensor_type('torch.HalfTensor') # 2021-05-31 added

hparams = create_hparams()
hparams.sampling_rate = 22050 # Don't change this
hparams.max_decoder_steps = 1000 # How long the audio will be before it cuts off (1000 is about 11 seconds)
hparams.gate_threshold = 0.1 # Model must be 90% sure the clip is over before ending generation (the higher this number is, the more likely that the AI will keep generating until it reaches the Max Decoder Steps)

# Load synthesizer
checkpoint_path = args.synthesizer_checkpoint_path

#model = load_model(hparams)
model = Tacotron2(hparams)

print('Loading synthesizer at path: {}'.format(checkpoint_path))
model.load_state_dict(torch.load(checkpoint_path)['state_dict'])
_ = model.cuda().eval().half()


# NB(bt, 2021-05-31): Trying to get everything on the same device
device = "cuda:0"
model = model.to(device)
#_ = model.cuda().eval()

# Load vocoder
waveglow_path = args.vocoder_checkpoint_path

#for k in waveglow.convinv:
#    k.float()
#denoiser = Denoiser(waveglow)


waveglow = torch.load(waveglow_path)['model']
waveglow.cuda().eval().half()
for k in waveglow.convinv:
    k.float()
denoiser = Denoiser(waveglow)

text = open(args.input_text_filename, 'r').read()

"""
seq = text_to_sequence(text, ['english_cleaners'])

sequence = np.array(seq)[None, :]

print("Sequence np.array: {}".format(type(sequence)))
print("Sequence np.array: {}".format(sequence))

sequence = torch.from_numpy(sequence)

print("Sequence from_numpy: {}".format(type(sequence)))
print("Sequence from_numpy: {}".format(sequence))
print("Sequence from_numpy dtype: {}".format(sequence.dtype))

sequence = torch.autograd.Variable(sequence).long()

print("Sequence torch autograd: {}".format(sequence))

# NB(bt, 2021-05-31): Trying to get everything to the same device
sequence = sequence.to(device)
"""



raw_input = True
sequence = None

for i in text.split("\n"):
    if len(i) < 1:
        continue
    print(i)
    if raw_input:
        if i[-1] != ";":
            i = i+";"
    else:
        #i = ARPA(i)
        pass
    print(i)
    with torch.no_grad():
        # save VRAM by not including gradients
        sequence = np.array(text_to_sequence(i, ['english_cleaners']))[None, :]
        sequence = torch.autograd.Variable(torch.from_numpy(sequence)).cuda().long()
        break # TODO: Handle multiple lines.

with torch.no_grad():
    mel_outputs, mel_outputs_postnet, _, alignments = model.inference(sequence)

#print("Mel outputs:")
#print(mel_outputs)
#print(mel_outputs.shape)
#print(mel_outputs_postnet)
#print(mel_outputs_postnet.shape)

print('Saving mel as JSON...')

class NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        return json.JSONEncoder.default(self, obj)


# squeeze(0) -> 3D to 2D (by removing the singular 1-length "wrapper" dimension)
# transpose() -> originally 80x{N}, we turn to {N}x80
mel_for_json = mel_outputs_postnet.cpu().numpy().squeeze(0).transpose()
mel_for_scaling = mel_for_json.copy()

max_value = np.amax(mel_for_scaling)
min_value = np.amin(mel_for_scaling)
mel_range = max_value - min_value

mel_for_scaling -= min_value
mel_for_scaling *= (255.0 / mel_range)
mel_for_scaling = mel_for_scaling.astype('int32')

json_data = {
    'mel': mel_for_json,
    'mel_scaled': mel_for_scaling,
}

with open(args.output_spectrogram_filename, 'w') as outfile:
    json.dump(json_data, outfile, cls=NumpyEncoder)

print('Encoding and saving audio...')

with torch.no_grad():
    sigma = 0.8
    audio = waveglow.infer(mel_outputs_postnet, sigma=sigma)

from scipy.io.wavfile import write

output_audio = audio[0].data.cpu().numpy().astype(np.float32)

write(args.output_audio_filename, hparams.sampling_rate, output_audio)

"""
try:
    debug_location = "/home/bt/dev/storyteller/storyteller-web/test"
    print('Writing to debug location: {}'.format(debug_location))
    shutil.copy(args.output_audio_filename, debug_location)

except Exception as e:
    print("Error with saving file for local debugging.")
    print(e)
    pass
"""

# ==== METADATA (1) ====

command = [
    "ffprobe",
    "-loglevel",  "quiet",
    "-print_format", "json",
    "-show_format",
    "-show_streams",
    args.output_audio_filename
]

pipe = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = pipe.communicate()
ffmpeg_metadata = json.loads(out)

# ==== METADATA (2) ====
mime_type = magic.from_file(args.output_audio_filename, mime=True)
file_size_bytes = os.path.getsize(args.output_audio_filename)

print(ffmpeg_metadata)

duration_millis = int(float(ffmpeg_metadata['format']['duration']) * 1000)

metadata = {
    'duration_millis': duration_millis,
    'mimetype': mime_type,
    'file_size_bytes': file_size_bytes,
}

with open(args.output_metadata_filename, 'w') as json_file:
    json.dump(metadata, json_file)