Clear cache hack

voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/hack.py

@@ -0,0 +1,29 @@
+import os
+import torch
+from logging import getLogger
+
+LOG = getLogger(__name__)
+
+def clear_cuda_cache():
+    print('torch.cuda.empty_cache() ...')
+    torch.cuda.empty_cache()
+    print('torch.cuda.empty_cache() ... DONE')
+
+def turn_off_jit_profiling():
+    print('torch._C._jit_set_profiling_mode(False) ...')
+    torch._C._jit_set_profiling_mode(False)
+    print('torch._C._jit_set_profiling_mode(False) ... DONE')
+
+def maybe_hacky_fix_before():
+    LOG.info("maybe_hacky_fix_before()")
+    if os.environ.get('FIX_CACHE_BEFORE'):
+        clear_cuda_cache()
+    if os.environ.get('FIX_JIT_BEFORE'):
+        turn_off_jit_profiling()
+
+def maybe_hacky_fix_after():
+    LOG.info("maybe_hacky_fix_after()")
+    if os.environ.get('FIX_CACHE_AFTER'):
+        clear_cuda_cache()
+    if os.environ.get('FIX_JIT_AFTER'):
+        turn_off_jit_profiling()

voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/core.py

@@ -19,6 +19,9 @@ from so_vits_svc_fork import cluster
 from so_vits_svc_fork.modules.synthesizers import SynthesizerTrn
 from so_vits_svc_fork.utils import get_optimal_device
 
+from so_vits_svc_fork.hack import maybe_hacky_fix_after
+from so_vits_svc_fork.hack import maybe_hacky_fix_before
+
 LOG = getLogger(__name__)
 
 
@@ -220,6 +223,8 @@ class Svc:
             "crepe", "crepe-tiny", "parselmouth", "dio", "harvest"
         ] = "dio",
     ) -> tuple[torch.Tensor, int]:
+        LOG.info(f"Svc.infer()")
+
         audio = audio.astype(np.float32)
         # get speaker id
         if isinstance(speaker, int):
@@ -255,6 +260,8 @@ class Svc:
         # inference
         with torch.no_grad():
             with timer() as t:
+                LOG.info(f"Svc.infer() inference BEGIN ...")
+                maybe_hacky_fix_before()
                 audio = self.net_g.infer(
                     c,
                     f0=f0,
@@ -263,6 +270,8 @@ class Svc:
                     predict_f0=auto_predict_f0,
                     noice_scale=noise_scale,
                 )[0, 0].data.float()
+                LOG.info(f"Svc.infer() inference END");
+                maybe_hacky_fix_after()
             audio_duration = audio.shape[-1] / self.target_sample
             LOG.info(
                 f"Inferece time: {t.elapsed:.2f}s, RTF: {t.elapsed / audio_duration:.2f}"

voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/main.py

@@ -13,6 +13,9 @@ from cm_time import timer
 from so_vits_svc_fork.inference.core import RealtimeVC, RealtimeVC2, Svc
 from so_vits_svc_fork.utils import get_optimal_device
 
+from so_vits_svc_fork.hack import maybe_hacky_fix_after
+from so_vits_svc_fork.hack import maybe_hacky_fix_before
+
 LOG = getLogger(__name__)
 
 
@@ -62,6 +65,7 @@ def infer(
 
     LOG.info("infer() svc_model.infer_silence() BEGIN")
 
+    maybe_hacky_fix_before()
     audio = svc_model.infer_silence(
         audio.astype(np.float32),
         speaker=speaker,
@@ -75,12 +79,14 @@ def infer(
         chunk_seconds=chunk_seconds,
         absolute_thresh=absolute_thresh,
     )
+    maybe_hacky_fix_after()
 
     LOG.info("infer() svc_model.infer_silence() END")
 
     LOG.info("infer() soundfile.write()")
 
     soundfile.write(output_path, audio, svc_model.target_sample)
+    maybe_hacky_fix_after()
 
 
 def realtime(

voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/utils.py

@@ -23,6 +23,8 @@ from tqdm import tqdm
 from transformers import HubertModel
 
 from so_vits_svc_fork.hparams import HParams
+from so_vits_svc_fork.hack import maybe_hacky_fix_after
+from so_vits_svc_fork.hack import maybe_hacky_fix_before
 
 LOG = getLogger(__name__)
 HUBERT_SAMPLING_RATE = 16000
@@ -156,13 +158,21 @@ class HubertModelWithFinalProj(HubertModel):
 def get_hubert_model(
     device: str | torch.device, final_proj: bool = True
 ) -> HubertModel:
-    LOG.info(f"get_hubert_model() for device = {device}")
+    LOG.info(f"get_hubert_model() for device = {device} BEGIN")
+    hubert = None
     if final_proj:
-        return HubertModelWithFinalProj.from_pretrained(
+        maybe_hacky_fix_before()
+        hubert = HubertModelWithFinalProj.from_pretrained(
             "lengyue233/content-vec-best"
         ).to(device)
+        maybe_hacky_fix_after()
     else:
-        return HubertModel.from_pretrained("lengyue233/content-vec-best").to(device)
+        maybe_hacky_fix_before()
+        hubert = HubertModel.from_pretrained("lengyue233/content-vec-best").to(device)
+        maybe_hacky_fix_after()
+
+    LOG.info(f"get_hubert_model() for device = {device} AFTER")
+    return hubert
 
 
 def get_content(
@@ -172,16 +182,21 @@ def get_content(
     sr: int,
     legacy_final_proj: bool = False,
 ) -> torch.Tensor:
+    LOG.info("HuBERT inference...")
     audio = torch.as_tensor(audio)
     if sr != HUBERT_SAMPLING_RATE:
+        LOG.info("HuBERT inference... AUDIO RESAMPLE")
         audio = (
             torchaudio.transforms.Resample(sr, HUBERT_SAMPLING_RATE)
             .to(audio.device)(audio)
             .to(device)
         )
+        LOG.info("HuBERT inference... AUDIO RESAMPLE DONE")
     if audio.ndim == 1:
         audio = audio.unsqueeze(0)
+    LOG.info("HuBERT inference... BEGIN")
     with torch.no_grad(), timer() as t:
+        maybe_hacky_fix_before()
         if legacy_final_proj:
             warnings.warn("legacy_final_proj is deprecated")
             if not hasattr(cmodel, "final_proj"):
@@ -191,6 +206,7 @@ def get_content(
         else:
             c = cmodel(audio)["last_hidden_state"]
         c = c.transpose(1, 2)
+        maybe_hacky_fix_after()
     wav_len = audio.shape[-1] / HUBERT_SAMPLING_RATE
     LOG.info(
         f"HuBERT inference time  : {t.elapsed:.3f}s, RTF: {t.elapsed / wav_len:.3f}"