Add logging, ripgrep, FIX DEVICE != cpu!

voice_conversion/so-vits-svc-paul-v2/Dockerfile

@@ -3,7 +3,7 @@ FROM nvidia/cuda:12.0.1-runtime-ubuntu22.04
 ENV DEBIAN_FRONTEND=noninteractive
 
 # NB: ffmpeg for transcoding opus -> wav
-# NB(bt,2023-05-04): Installing lsof, htop, and other debugging tools
+# NB(bt,2023-05-04): Installing lsof, htop, ripgrep, and other debugging tools
 # NB(bt,2023-05-04): Installing rsync to monitor time it takes for files to sync
 RUN apt-get update && apt-get install -y \
     build-essential \
@@ -17,6 +17,7 @@ RUN apt-get update && apt-get install -y \
     python3.10 \
     python3.10-dev \
     python3.10-venv \
+    ripgrep \
     rsync \
     vim \
     wget \

voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/core.py

@@ -129,12 +129,19 @@ class Svc:
             self.cluster_model = cluster.get_cluster_model(cluster_model_path)
 
     def load_model(self):
+        LOG.info(f"Svc.load_model() ... ")
         self.net_g = SynthesizerTrn(
             self.hps.data.filter_length // 2 + 1,
             self.hps.train.segment_size // self.hps.data.hop_length,
             **self.hps.model,
         )
+
+        LOG.info(f"Svc.load_model() utils.load_checkpoit() BEGIN")
+
         _ = utils.load_checkpoint(self.net_g_path, self.net_g, None)
+
+        LOG.info(f"Svc.load_model() utils.load_checkpoit() END")
+        
         _ = self.net_g.eval().to(self.device, dtype=self.dtype)
 
     def get_unit_f0(

voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/utils.py

@@ -151,6 +151,7 @@ class HubertModelWithFinalProj(HubertModel):
 def get_hubert_model(
     device: str | torch.device, final_proj: bool = True
 ) -> HubertModel:
+    LOG.info("get_hubert_model()")
     if final_proj:
         return HubertModelWithFinalProj.from_pretrained(
             "lengyue233/content-vec-best"
@@ -233,6 +234,9 @@ def load_checkpoint(
     optimizer: torch.optim.Optimizer | None = None,
     skip_optimizer: bool = False,
 ) -> tuple[torch.nn.Module, torch.optim.Optimizer | None, float, int]:
+
+    LOG.info(f"utils.load_checkpoint(): path = {checkpoint_path}")
+
     if not Path(checkpoint_path).is_file():
         raise FileNotFoundError(f"File {checkpoint_path} not found")
     with Path(checkpoint_path).open("rb") as f:
@@ -242,15 +246,28 @@ def load_checkpoint(
             )
             # NB(paul): iirc changing this to false prevented an error at some point
             # but it also worsens security
-            checkpoint_dict = torch.load(f, map_location="cpu", weights_only=False)
+
+            LOG.info("utils.load_checkpoint(): torch.load() BEGIN")
+
+            # NB(bt, 2023-05-04): This was set to CPU! Maybe this is the fix
+            # checkpoint_dict = torch.load(f, map_location="cpu", weights_only=False)
+            checkpoint_dict = torch.load(f, weights_only=False)
+
+            LOG.info("utils.load_checkpoint(): torch.load() END")
+
     iteration = checkpoint_dict["iteration"]
     learning_rate = checkpoint_dict["learning_rate"]
 
     # safe load module
+    LOG.info("utils.load_checkpoint(): safe_load() BEGIN")
+
     if hasattr(model, "module"):
         safe_load(model.module, checkpoint_dict["model"])
     else:
         safe_load(model, checkpoint_dict["model"])
+
+    LOG.info("utils.load_checkpoint(): safe_load() END")
+
     # safe load optim
     if (
         optimizer is not None
@@ -259,8 +276,13 @@ def load_checkpoint(
     ):
         with warnings.catch_warnings():
             warnings.simplefilter("ignore")
+
+            LOG.info("utils.load_checkpoint(): safe_load() optimizer BEGIN")
+
             safe_load(optimizer, checkpoint_dict["optimizer"])
 
+            LOG.info("utils.load_checkpoint(): safe_load() optimizer END")
+
     LOG.info(f"Loaded checkpoint '{checkpoint_path}' (epoch {iteration})")
     return model, optimizer, learning_rate, iteration