diff --git a/voice_conversion/so-vits-svc-paul-v2/Dockerfile b/voice_conversion/so-vits-svc-paul-v2/Dockerfile index 5d9805f..d97c262 100644 --- a/voice_conversion/so-vits-svc-paul-v2/Dockerfile +++ b/voice_conversion/so-vits-svc-paul-v2/Dockerfile @@ -3,7 +3,7 @@ FROM nvidia/cuda:12.0.1-runtime-ubuntu22.04 ENV DEBIAN_FRONTEND=noninteractive # NB: ffmpeg for transcoding opus -> wav -# NB(bt,2023-05-04): Installing lsof, htop, and other debugging tools +# NB(bt,2023-05-04): Installing lsof, htop, ripgrep, and other debugging tools # NB(bt,2023-05-04): Installing rsync to monitor time it takes for files to sync RUN apt-get update && apt-get install -y \ build-essential \ @@ -17,6 +17,7 @@ RUN apt-get update && apt-get install -y \ python3.10 \ python3.10-dev \ python3.10-venv \ + ripgrep \ rsync \ vim \ wget \ diff --git a/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/core.py b/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/core.py index cf5ce94..081e4c9 100644 --- a/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/core.py +++ b/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/inference/core.py @@ -129,12 +129,19 @@ class Svc: self.cluster_model = cluster.get_cluster_model(cluster_model_path) def load_model(self): + LOG.info(f"Svc.load_model() ... ") self.net_g = SynthesizerTrn( self.hps.data.filter_length // 2 + 1, self.hps.train.segment_size // self.hps.data.hop_length, **self.hps.model, ) + + LOG.info(f"Svc.load_model() utils.load_checkpoit() BEGIN") + _ = utils.load_checkpoint(self.net_g_path, self.net_g, None) + + LOG.info(f"Svc.load_model() utils.load_checkpoit() END") + _ = self.net_g.eval().to(self.device, dtype=self.dtype) def get_unit_f0( diff --git a/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/utils.py b/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/utils.py index 9578b81..6b542fd 100644 --- a/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/utils.py +++ b/voice_conversion/so-vits-svc-paul-v2/src/so_vits_svc_fork/utils.py @@ -151,6 +151,7 @@ class HubertModelWithFinalProj(HubertModel): def get_hubert_model( device: str | torch.device, final_proj: bool = True ) -> HubertModel: + LOG.info("get_hubert_model()") if final_proj: return HubertModelWithFinalProj.from_pretrained( "lengyue233/content-vec-best" @@ -233,6 +234,9 @@ def load_checkpoint( optimizer: torch.optim.Optimizer | None = None, skip_optimizer: bool = False, ) -> tuple[torch.nn.Module, torch.optim.Optimizer | None, float, int]: + + LOG.info(f"utils.load_checkpoint(): path = {checkpoint_path}") + if not Path(checkpoint_path).is_file(): raise FileNotFoundError(f"File {checkpoint_path} not found") with Path(checkpoint_path).open("rb") as f: @@ -242,15 +246,28 @@ def load_checkpoint( ) # NB(paul): iirc changing this to false prevented an error at some point # but it also worsens security - checkpoint_dict = torch.load(f, map_location="cpu", weights_only=False) + + LOG.info("utils.load_checkpoint(): torch.load() BEGIN") + + # NB(bt, 2023-05-04): This was set to CPU! Maybe this is the fix + # checkpoint_dict = torch.load(f, map_location="cpu", weights_only=False) + checkpoint_dict = torch.load(f, weights_only=False) + + LOG.info("utils.load_checkpoint(): torch.load() END") + iteration = checkpoint_dict["iteration"] learning_rate = checkpoint_dict["learning_rate"] # safe load module + LOG.info("utils.load_checkpoint(): safe_load() BEGIN") + if hasattr(model, "module"): safe_load(model.module, checkpoint_dict["model"]) else: safe_load(model, checkpoint_dict["model"]) + + LOG.info("utils.load_checkpoint(): safe_load() END") + # safe load optim if ( optimizer is not None @@ -259,8 +276,13 @@ def load_checkpoint( ): with warnings.catch_warnings(): warnings.simplefilter("ignore") + + LOG.info("utils.load_checkpoint(): safe_load() optimizer BEGIN") + safe_load(optimizer, checkpoint_dict["optimizer"]) + LOG.info("utils.load_checkpoint(): safe_load() optimizer END") + LOG.info(f"Loaded checkpoint '{checkpoint_path}' (epoch {iteration})") return model, optimizer, learning_rate, iteration