From b5355de94e739996eb2b481f3892ee5573b84bc8 Mon Sep 17 00:00:00 2001
From: Brandon Thomas <echelon@gmail.com>
Date: Sat, 11 Mar 2023 01:18:19 -0500
Subject: [PATCH] Add support for Ozzy's mel multiplier constant

---
 tts/vocodes_common.py         | 24 +++++++++++++++++++++++-
 tts/vocodes_server.py         |  8 ++++++--
 tts/vocodes_server_talknet.py |  4 ++--
 3 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/tts/vocodes_common.py b/tts/vocodes_common.py
index 87dae19..ab5b336 100644
--- a/tts/vocodes_common.py
+++ b/tts/vocodes_common.py
@@ -38,6 +38,11 @@ from text.pipeline_components.phonetization.english_phonetization import English
 
 from logger import LOGGER
 
+# Multiplying the mel output postnet by a constant factor scales it before vocoding,
+# often resulting in "better" (less tinny) sound.
+# This value was determined by Ozzy via experimentation.
+DEFAULT_MEL_MULTIPLIER = 1.4
+
 def print_gpu_info():
     print('========================================')
     print('Python interpreter', sys.executable)
@@ -211,7 +216,7 @@ def generate_metadata_file(output_audio_filename, output_metadata_filename):
         json.dump(metadata, json_file)
 
 
-class TacotronWaveglowPipeline:
+class FakeYouTt2Pipeline:
 
     def __init__(self):
         # Cache of "path" => loaded model
@@ -281,6 +286,8 @@ class TacotronWaveglowPipeline:
     def infer(self, args):
         assert('synthesizer_checkpoint_path' in args)
         assert('vocoder_type' in args)
+        assert('use_default_mel_multiply_factor' in args)
+        assert('maybe_custom_mel_multiply_factor' in args)
         assert('text_pipeline_type' in args)
         assert('raw_text' in args)
         assert('output_audio_filename' in args)
@@ -300,6 +307,20 @@ class TacotronWaveglowPipeline:
 
         with torch.no_grad():
             mel_outputs, mel_outputs_postnet, _, alignments = tacotron.inference(sequence)
+        
+        # Multiply the mel outputs for "better" sound.
+        use_multiplier = False
+        mel_multiplier = DEFAULT_MEL_MULTIPLIER
+
+        if args['maybe_custom_mel_multiply_factor']:
+            use_multiplier = True
+            mel_multiplier = float(args['maybe_custom_mel_multiply_factor'])
+        elif args['use_default_mel_multiply_factor']:
+            use_multiplier = True
+
+        if use_multiplier:
+            LOGGER.info(f'Scaling audio by {mel_multiplier}')
+            mel_outputs_postnet = mel_outputs_postnet * mel_multiplier
 
         if args['vocoder_type'] == 'hifigan-superres':
             self.vocode_hifigan_superres(args, mel_outputs_postnet)
@@ -362,6 +383,7 @@ class TacotronWaveglowPipeline:
                 self.hifigan_super_resolution_h.fmin,
                 self.hifigan_super_resolution_h.fmax,
             )
+
             y_g_hat2 = self.hifigan_super_resolution(new_mel)
             audio2 = y_g_hat2.squeeze()
             audio2 = audio2 * MAX_WAV_VALUE
diff --git a/tts/vocodes_server.py b/tts/vocodes_server.py
index e6e5ab4..1abb15f 100755
--- a/tts/vocodes_server.py
+++ b/tts/vocodes_server.py
@@ -24,7 +24,7 @@ from falcon_multipart.middleware import MultipartMiddleware
 #from preprocess import *
 from wsgiref import simple_server
 
-from vocodes_common import TacotronWaveglowPipeline
+from vocodes_common import FakeYouTt2Pipeline
 from vocodes_common import load_hifigan_model
 from vocodes_common import load_tacotron_model
 from vocodes_common import load_waveglow_model
@@ -200,7 +200,7 @@ INDEX_HTML = '''
 
 #tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt')
 #waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt')
-pipeline = TacotronWaveglowPipeline()
+pipeline = FakeYouTt2Pipeline()
 
 class IndexHandler():
     def on_get(self, request, response):
@@ -227,6 +227,8 @@ class ApiHandler():
         # Request parameters
         synthesizer_checkpoint_path = raw_data.get('synthesizer_checkpoint_path')
         text_pipeline_type = raw_data.get('text_pipeline_type')
+        use_default_mel_multiply_factor = raw_data.get('use_default_mel_multiply_factor')
+        maybe_custom_mel_multiply_factor = raw_data.get('maybe_custom_mel_multiply_factor')
         vocoder_type = raw_data.get('vocoder_type')
         waveglow_vocoder_checkpoint_path = raw_data.get('waveglow_vocoder_checkpoint_path')
         hifigan_vocoder_checkpoint_path = raw_data.get('hifigan_vocoder_checkpoint_path')
@@ -259,6 +261,8 @@ class ApiHandler():
 
         inference_args = {
             'raw_text': inference_text,
+            'use_default_mel_multiply_factor': use_default_mel_multiply_factor,
+            'maybe_custom_mel_multiply_factor': maybe_custom_mel_multiply_factor,
             'synthesizer_checkpoint_path': synthesizer_checkpoint_path,
             'vocoder_type': vocoder_type,
             'text_pipeline_type': text_pipeline_type,
diff --git a/tts/vocodes_server_talknet.py b/tts/vocodes_server_talknet.py
index 32cc859..85802f7 100755
--- a/tts/vocodes_server_talknet.py
+++ b/tts/vocodes_server_talknet.py
@@ -24,7 +24,7 @@ from falcon_multipart.middleware import MultipartMiddleware
 #from preprocess import *
 from wsgiref import simple_server
 
-from vocodes_common_talknet import TacotronWaveglowPipeline
+from vocodes_common_talknet import FakeYouTt2Pipeline
 from vocodes_common_talknet import print_gpu_info
 from vocodes_common_talknet import load_tacotron_model
 from vocodes_common_talknet import load_waveglow_model
@@ -197,7 +197,7 @@ INDEX_HTML = '''
 
 #tacotron = load_tacotron_model('/home/bt/models/tacotron2/tacotron2_uberduck_Noire.pt')
 #waveglow = load_waveglow_model('/home/bt/models/waveglow/waveglow_256channels_universal_v5.pt')
-pipeline = TacotronWaveglowPipeline()
+pipeline = FakeYouTt2Pipeline()
 
 class IndexHandler():
     def on_get(self, request, response):