{ "nbformat": 4, "nbformat_minor": 0, "metadata": { "colab": { "provenance": [], "gpuType": "T4", "include_colab_link": true }, "kernelspec": { "name": "python3", "display_name": "Python 3" }, "language_info": { "name": "python" }, "accelerator": "GPU" }, "cells": [ { "cell_type": "markdown", "metadata": { "id": "view-in-github", "colab_type": "text" }, "source": [ "\"Open" ] }, { "cell_type": "markdown", "source": [ "### Install packages and download models" ], "metadata": { "id": "yLqBa4uYPrqE" } }, { "cell_type": "code", "source": [ "# @title Mount Drive\n", "from google.colab import drive\n", "drive.mount('/content/drive')" ], "metadata": { "cellView": "form", "id": "nG64C8KKDzh5", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "8fb8ce4e-ed8d-40c9-b3fd-add97935d1f1" }, "execution_count": 1, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Mounted at /content/drive\n" ] } ] }, { "cell_type": "code", "source": [ "%%shell\n", "git clone https://dagshub.com/StyleTTS/StyleTTS2.git\n", "cd StyleTTS2\n", "pip install soundfile==0.10.3.post1 torchaudio munch torch pydub pyyaml librosa nltk matplotlib accelerate transformers phonemizer einops einops-exts tqdm typing-extensions git+https://github.com/resemble-ai/monotonic_align.git\n", "sudo apt-get install espeak-ng\n", "git-lfs clone https://huggingface.co/yl4579/StyleTTS2-LibriTTS\n", "mv StyleTTS2-LibriTTS/Models ." ], "metadata": { "id": "H72WF06ZPrTF", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "af6d38a8-3735-484c-bb7c-6fe0889c2702" }, "execution_count": 2, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "Cloning into 'StyleTTS2'...\n", "remote: Enumerating objects: 474, done.\u001b[K\n", "remote: Counting objects: 100% (236/236), done.\u001b[K\n", "remote: Compressing objects: 100% (110/110), done.\u001b[K\n", "remote: Total 474 (delta 200), reused 135 (delta 124), pack-reused 238\u001b[K\n", "Receiving objects: 100% (474/474), 139.87 MiB | 32.63 MiB/s, done.\n", "Resolving deltas: 100% (278/278), done.\n", "Updating files: 100% (47/47), done.\n", "Collecting git+https://github.com/resemble-ai/monotonic_align.git\n", " Cloning https://github.com/resemble-ai/monotonic_align.git to /tmp/pip-req-build-zqhmlktm\n", " Running command git clone --filter=blob:none --quiet https://github.com/resemble-ai/monotonic_align.git /tmp/pip-req-build-zqhmlktm\n", " Resolved https://github.com/resemble-ai/monotonic_align.git to commit 78b985be210a03d08bc3acc01c4df0442105366f\n", " Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n", " Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n", " Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", "Collecting soundfile==0.10.3.post1\n", " Downloading SoundFile-0.10.3.post1-py2.py3-none-any.whl (21 kB)\n", "Requirement already satisfied: torchaudio in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n", "Collecting munch\n", " Downloading munch-4.0.0-py2.py3-none-any.whl (9.9 kB)\n", "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.1.0+cu118)\n", "Collecting pydub\n", " Downloading pydub-0.25.1-py2.py3-none-any.whl (32 kB)\n", "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (6.0.1)\n", "Requirement already satisfied: librosa in /usr/local/lib/python3.10/dist-packages (0.10.1)\n", "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.8.1)\n", "Requirement already satisfied: matplotlib in /usr/local/lib/python3.10/dist-packages (3.7.1)\n", "Collecting accelerate\n", " Downloading accelerate-0.24.1-py3-none-any.whl (261 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m261.4/261.4 kB\u001b[0m \u001b[31m9.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.35.2)\n", "Collecting phonemizer\n", " Downloading phonemizer-3.2.1-py3-none-any.whl (90 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.6/90.6 kB\u001b[0m \u001b[31m14.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting einops\n", " Downloading einops-0.7.0-py3-none-any.whl (44 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m6.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting einops-exts\n", " Downloading einops_exts-0.0.4-py3-none-any.whl (3.9 kB)\n", "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (4.66.1)\n", "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (4.5.0)\n", "Requirement already satisfied: cffi>=1.0 in /usr/local/lib/python3.10/dist-packages (from soundfile==0.10.3.post1) (1.16.0)\n", "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch) (3.13.1)\n", "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n", "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.2.1)\n", "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.2)\n", "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n", "Requirement already satisfied: triton==2.1.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.1.0)\n", "Requirement already satisfied: audioread>=2.1.9 in /usr/local/lib/python3.10/dist-packages (from librosa) (3.0.1)\n", "Requirement already satisfied: numpy!=1.22.0,!=1.22.1,!=1.22.2,>=1.20.3 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.23.5)\n", "Requirement already satisfied: scipy>=1.2.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.11.3)\n", "Requirement already satisfied: scikit-learn>=0.20.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.2.2)\n", "Requirement already satisfied: joblib>=0.14 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.3.2)\n", "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (4.4.2)\n", "Requirement already satisfied: numba>=0.51.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (0.58.1)\n", "INFO: pip is looking at multiple versions of librosa to determine which version is compatible with other requirements. This could take a while.\n", "Collecting librosa\n", " Downloading librosa-0.10.0.post2-py3-none-any.whl (253 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m253.0/253.0 kB\u001b[0m \u001b[31m24.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading librosa-0.10.0.post1-py3-none-any.whl (252 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m253.0/253.0 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading librosa-0.10.0-py3-none-any.whl (252 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.9/252.9 kB\u001b[0m \u001b[31m33.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Downloading librosa-0.9.2-py3-none-any.whl (214 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.3/214.3 kB\u001b[0m \u001b[31m29.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting resampy>=0.2.2 (from librosa)\n", " Downloading resampy-0.4.2-py3-none-any.whl (3.1 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m62.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: pooch>=1.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (1.8.0)\n", "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from librosa) (23.2)\n", "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk) (8.1.7)\n", "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk) (2023.6.3)\n", "Requirement already satisfied: contourpy>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.2.0)\n", "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (0.12.1)\n", "Requirement already satisfied: fonttools>=4.22.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (4.44.3)\n", "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (1.4.5)\n", "Requirement already satisfied: pillow>=6.2.0 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (9.4.0)\n", "Requirement already satisfied: pyparsing>=2.3.1 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (3.1.1)\n", "Requirement already satisfied: python-dateutil>=2.7 in /usr/local/lib/python3.10/dist-packages (from matplotlib) (2.8.2)\n", "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n", "Requirement already satisfied: huggingface-hub in /usr/local/lib/python3.10/dist-packages (from accelerate) (0.19.4)\n", "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n", "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.15.0)\n", "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.0)\n", "Collecting segments (from phonemizer)\n", " Downloading segments-2.2.1-py2.py3-none-any.whl (15 kB)\n", "Requirement already satisfied: attrs>=18.1 in /usr/local/lib/python3.10/dist-packages (from phonemizer) (23.1.0)\n", "Collecting dlinfo (from phonemizer)\n", " Downloading dlinfo-1.2.1-py3-none-any.whl (3.6 kB)\n", "Requirement already satisfied: pycparser in /usr/local/lib/python3.10/dist-packages (from cffi>=1.0->soundfile==0.10.3.post1) (2.21)\n", "Requirement already satisfied: llvmlite<0.42,>=0.41.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba>=0.51.0->librosa) (0.41.1)\n", "Requirement already satisfied: platformdirs>=2.5.0 in /usr/local/lib/python3.10/dist-packages (from pooch>=1.0->librosa) (4.0.0)\n", "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n", "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n", "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.20.0->librosa) (3.2.0)\n", "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.3)\n", "Collecting clldutils>=1.7.3 (from segments->phonemizer)\n", " Downloading clldutils-3.20.0-py2.py3-none-any.whl (1.7 MB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.7/1.7 MB\u001b[0m \u001b[31m90.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting csvw>=1.5.6 (from segments->phonemizer)\n", " Downloading csvw-3.2.1-py2.py3-none-any.whl (56 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.7/56.7 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n", "Requirement already satisfied: tabulate>=0.7.7 in /usr/local/lib/python3.10/dist-packages (from clldutils>=1.7.3->segments->phonemizer) (0.9.0)\n", "Collecting colorlog (from clldutils>=1.7.3->segments->phonemizer)\n", " Downloading colorlog-6.7.0-py2.py3-none-any.whl (11 kB)\n", "Collecting pylatexenc (from clldutils>=1.7.3->segments->phonemizer)\n", " Downloading pylatexenc-2.10.tar.gz (162 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m162.6/162.6 kB\u001b[0m \u001b[31m22.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n", "Requirement already satisfied: markdown in /usr/local/lib/python3.10/dist-packages (from clldutils>=1.7.3->segments->phonemizer) (3.5.1)\n", "Requirement already satisfied: lxml in /usr/local/lib/python3.10/dist-packages (from clldutils>=1.7.3->segments->phonemizer) (4.9.3)\n", "Requirement already satisfied: babel in /usr/local/lib/python3.10/dist-packages (from csvw>=1.5.6->segments->phonemizer) (2.13.1)\n", "Collecting colorama (from csvw>=1.5.6->segments->phonemizer)\n", " Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n", "Collecting isodate (from csvw>=1.5.6->segments->phonemizer)\n", " Downloading isodate-0.6.1-py2.py3-none-any.whl (41 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m41.7/41.7 kB\u001b[0m \u001b[31m6.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hRequirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from csvw>=1.5.6->segments->phonemizer) (4.19.2)\n", "Collecting language-tags (from csvw>=1.5.6->segments->phonemizer)\n", " Downloading language_tags-1.2.0-py3-none-any.whl (213 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m213.4/213.4 kB\u001b[0m \u001b[31m29.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting rdflib (from csvw>=1.5.6->segments->phonemizer)\n", " Downloading rdflib-7.0.0-py3-none-any.whl (531 kB)\n", "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m531.9/531.9 kB\u001b[0m \u001b[31m57.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n", "\u001b[?25hCollecting rfc3986<2 (from csvw>=1.5.6->segments->phonemizer)\n", " Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n", "Requirement already satisfied: uritemplate>=3.0.0 in /usr/local/lib/python3.10/dist-packages (from csvw>=1.5.6->segments->phonemizer) (4.1.1)\n", "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->csvw>=1.5.6->segments->phonemizer) (2023.11.1)\n", "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->csvw>=1.5.6->segments->phonemizer) (0.31.0)\n", "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->csvw>=1.5.6->segments->phonemizer) (0.13.0)\n", "Building wheels for collected packages: monotonic-align, pylatexenc\n", " Building wheel for monotonic-align (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for monotonic-align: filename=monotonic_align-1.2-cp310-cp310-linux_x86_64.whl size=1419165 sha256=bb6df14370fbb68aca076ae15e60c1cff86fb299730c050ea2ddb6de51eaeee6\n", " Stored in directory: /tmp/pip-ephem-wheel-cache-96880v1g/wheels/62/6e/e2/2758abe7d7bf44cf150d943471be9836770e7c40c1daf4b7bd\n", " Building wheel for pylatexenc (setup.py) ... \u001b[?25l\u001b[?25hdone\n", " Created wheel for pylatexenc: filename=pylatexenc-2.10-py3-none-any.whl size=136818 sha256=d0cfadeb191ec0abd32921f0f0883bd849109a4d3fe58d3f3f2fe7c05976647c\n", " Stored in directory: /root/.cache/pip/wheels/d3/31/8b/e09b0386afd80cfc556c00408c9aeea5c35c4d484a9c762fd5\n", "Successfully built monotonic-align pylatexenc\n", "Installing collected packages: rfc3986, pylatexenc, pydub, language-tags, dlinfo, munch, monotonic-align, isodate, einops, colorlog, colorama, soundfile, resampy, rdflib, einops-exts, clldutils, librosa, accelerate, csvw, segments, phonemizer\n", " Attempting uninstall: soundfile\n", " Found existing installation: soundfile 0.12.1\n", " Uninstalling soundfile-0.12.1:\n", " Successfully uninstalled soundfile-0.12.1\n", " Attempting uninstall: librosa\n", " Found existing installation: librosa 0.10.1\n", " Uninstalling librosa-0.10.1:\n", " Successfully uninstalled librosa-0.10.1\n", "Successfully installed accelerate-0.24.1 clldutils-3.20.0 colorama-0.4.6 colorlog-6.7.0 csvw-3.2.1 dlinfo-1.2.1 einops-0.7.0 einops-exts-0.0.4 isodate-0.6.1 language-tags-1.2.0 librosa-0.9.2 monotonic-align-1.2 munch-4.0.0 phonemizer-3.2.1 pydub-0.25.1 pylatexenc-2.10 rdflib-7.0.0 resampy-0.4.2 rfc3986-1.5.0 segments-2.2.1 soundfile-0.10.3.post1\n", "Reading package lists... Done\n", "Building dependency tree... Done\n", "Reading state information... Done\n", "The following additional packages will be installed:\n", " espeak-ng-data libespeak-ng1 libpcaudio0 libsonic0\n", "The following NEW packages will be installed:\n", " espeak-ng espeak-ng-data libespeak-ng1 libpcaudio0 libsonic0\n", "0 upgraded, 5 newly installed, 0 to remove and 11 not upgraded.\n", "Need to get 4,525 kB of archives.\n", "After this operation, 11.9 MB of additional disk space will be used.\n", "Get:1 http://archive.ubuntu.com/ubuntu jammy/main amd64 libpcaudio0 amd64 1.1-6build2 [8,956 B]\n", "Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 libsonic0 amd64 0.2.0-11build1 [10.3 kB]\n", "Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 espeak-ng-data amd64 1.50+dfsg-10 [3,956 kB]\n", "Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 libespeak-ng1 amd64 1.50+dfsg-10 [207 kB]\n", "Get:5 http://archive.ubuntu.com/ubuntu jammy/universe amd64 espeak-ng amd64 1.50+dfsg-10 [343 kB]\n", "Fetched 4,525 kB in 1s (7,844 kB/s)\n", "debconf: unable to initialize frontend: Dialog\n", "debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 78, <> line 5.)\n", "debconf: falling back to frontend: Readline\n", "debconf: unable to initialize frontend: Readline\n", "debconf: (This frontend requires a controlling tty.)\n", "debconf: falling back to frontend: Teletype\n", "dpkg-preconfigure: unable to re-open stdin: \n", "Selecting previously unselected package libpcaudio0:amd64.\n", "(Reading database ... 120880 files and directories currently installed.)\n", "Preparing to unpack .../libpcaudio0_1.1-6build2_amd64.deb ...\n", "Unpacking libpcaudio0:amd64 (1.1-6build2) ...\n", "Selecting previously unselected package libsonic0:amd64.\n", "Preparing to unpack .../libsonic0_0.2.0-11build1_amd64.deb ...\n", "Unpacking libsonic0:amd64 (0.2.0-11build1) ...\n", "Selecting previously unselected package espeak-ng-data:amd64.\n", "Preparing to unpack .../espeak-ng-data_1.50+dfsg-10_amd64.deb ...\n", "Unpacking espeak-ng-data:amd64 (1.50+dfsg-10) ...\n", "Selecting previously unselected package libespeak-ng1:amd64.\n", "Preparing to unpack .../libespeak-ng1_1.50+dfsg-10_amd64.deb ...\n", "Unpacking libespeak-ng1:amd64 (1.50+dfsg-10) ...\n", "Selecting previously unselected package espeak-ng.\n", "Preparing to unpack .../espeak-ng_1.50+dfsg-10_amd64.deb ...\n", "Unpacking espeak-ng (1.50+dfsg-10) ...\n", "Setting up libpcaudio0:amd64 (1.1-6build2) ...\n", "Setting up libsonic0:amd64 (0.2.0-11build1) ...\n", "Setting up espeak-ng-data:amd64 (1.50+dfsg-10) ...\n", "Setting up libespeak-ng1:amd64 (1.50+dfsg-10) ...\n", "Setting up espeak-ng (1.50+dfsg-10) ...\n", "Processing triggers for man-db (2.10.2-1) ...\n", "Processing triggers for libc-bin (2.35-0ubuntu3.4) ...\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_5.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbbind_2_0.so.3 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc.so.2 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbb.so.12 is not a symbolic link\n", "\n", "/sbin/ldconfig.real: /usr/local/lib/libtbbmalloc_proxy.so.2 is not a symbolic link\n", "\n", "WARNING: 'git lfs clone' is deprecated and will not be updated\n", " with new flags from 'git clone'\n", "\n", "'git clone' has been updated in upstream Git to have comparable\n", "speeds to 'git lfs clone'.\n", "Cloning into 'StyleTTS2-LibriTTS'...\n", "remote: Enumerating objects: 25, done.\u001b[K\n", "remote: Counting objects: 100% (12/12), done.\u001b[K\n", "remote: Compressing objects: 100% (9/9), done.\u001b[K\n", "remote: Total 25 (delta 0), reused 0 (delta 0), pack-reused 13\u001b[K\n", "Unpacking objects: 100% (25/25), 3.73 KiB | 764.00 KiB/s, done.\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [] }, "metadata": {}, "execution_count": 2 } ] }, { "cell_type": "markdown", "source": [ "### Download dataset (LJSpeech, 200 samples, ~15 minutes of data)\n", "\n", "You can definitely do it with fewer samples. This is just a proof of concept with 200 smaples." ], "metadata": { "id": "G398sL8wPzTB" } }, { "cell_type": "code", "source": [ "%cd StyleTTS2\n", "!rm -rf Data" ], "metadata": { "id": "kJuQUBrEPy5C", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "01efb995-ac4a-4be8-efe1-f9acc9d011b5" }, "execution_count": 3, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "/content/StyleTTS2\n" ] } ] }, { "cell_type": "code", "source": [ "!gdown --id 1vqz26D3yn7OXS2vbfYxfSnpLS6m6tOFP\n", "!unzip Data.zip" ], "metadata": { "id": "mDXW8ZZePuSb" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "### Convert Unphonemized (only do if you're using a text based ljspeech dataset)" ], "metadata": { "id": "3m93S00hq7pE" } }, { "cell_type": "code", "source": [ "# @title Convert\n", "path_to_wavs_txt = \"/content/drive/Othercomputers/My Laptop/Voice Clones/Me/wavs.txt\" # @param {type:\"string\"}\n", "save_to_path = \"/content/drive/Othercomputers/My Laptop/Voice Clones/Me/OOD_texts.txt\" # @param {type:\"string\"}\n", "import nltk\n", "nltk.download('punkt')\n", "from nltk.tokenize import word_tokenize\n", "from tqdm import tqdm\n", "\n", "import phonemizer\n", "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True, with_stress=True)\n", "\n", "def text_to_phonemes(text):\n", " text = text.strip()\n", " print(\"Text before phonemization: \", text)\n", " ps = global_phonemizer.phonemize([text])\n", " print(\"Text after phonemization: \", ps)\n", " ps = word_tokenize(ps[0])\n", " ps = ' '.join(ps)\n", " print(\"Final text after tokenization: \", ps)\n", " return ps\n", "\n", "\n", "with open(path_to_wavs_txt, 'r', encoding=\"utf8\") as fh:\n", " lines = fh.readlines()\n", "\n", "final = []\n", "for line in tqdm(lines):\n", " data = line.replace(\"\\n\", \"\").split(\"|\")\n", " data[1] = text_to_phonemes(data[1])\n", " if len(data) < 3:\n", " data.append(0)\n", "\n", " final.append(f\"{data[0]}|{data[1]}|{data[2]}\")\n", "\n", "with open(save_to_path, 'w', encoding=\"utf8\") as fh:\n", " final = \"\\n\".join(final)\n", " fh.write(final)" ], "metadata": { "id": "Ql8NXtQGrSDT", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "a0d57bee-565a-4c87-bb0b-8b7f73d6a500", "cellView": "form" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "[nltk_data] Downloading package punkt to /root/nltk_data...\n", "[nltk_data] Package punkt is already up-to-date!\n", " 0%| | 0/140 [00:00" ], "application/javascript": [ "\n", " (async () => {\n", " const url = new URL(await google.colab.kernel.proxyPort(6006, {'cache': true}));\n", " url.searchParams.set('tensorboardColab', 'true');\n", " const iframe = document.createElement('iframe');\n", " iframe.src = url;\n", " iframe.setAttribute('width', '100%');\n", " iframe.setAttribute('height', '800');\n", " iframe.setAttribute('frameborder', 0);\n", " document.body.appendChild(iframe);\n", " })();\n", " " ] }, "metadata": {} } ] }, { "cell_type": "code", "source": [ "!accelerate launch --mixed_precision=fp16 train_finetune_accelerate.py --config_path ./Configs/config_ft.yml" ], "metadata": { "id": "HZVAD5GKWm-O", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "3f9bfe7f-61b2-45d5-c646-51abf71463ee" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "The following values were not passed to `accelerate launch` and had defaults used instead:\n", "\t`--num_processes` was set to a value of `1`\n", "\t`--num_machines` was set to a value of `1`\n", "\t`--dynamo_backend` was set to a value of `'no'`\n", "To avoid this warning pass in values for each of the problematic parameters or run `accelerate config`.\n", "2023-11-28 10:41:01.622869: E tensorflow/compiler/xla/stream_executor/cuda/cuda_dnn.cc:9342] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n", "2023-11-28 10:41:01.622943: E tensorflow/compiler/xla/stream_executor/cuda/cuda_fft.cc:609] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n", "2023-11-28 10:41:01.622987: E tensorflow/compiler/xla/stream_executor/cuda/cuda_blas.cc:1518] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n", "2023-11-28 10:41:05.923765: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n", "config.json: 100% 2.23k/2.23k [00:00<00:00, 12.0MB/s]\n", "pytorch_model.bin: 100% 378M/378M [00:01<00:00, 198MB/s]\n", "Some weights of the model checkpoint at microsoft/wavlm-base-plus were not used when initializing WavLMModel: ['encoder.pos_conv_embed.conv.weight_g', 'encoder.pos_conv_embed.conv.weight_v']\n", "- This IS expected if you are initializing WavLMModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing WavLMModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", "Some weights of WavLMModel were not initialized from the model checkpoint at microsoft/wavlm-base-plus and are newly initialized: ['encoder.pos_conv_embed.conv.parametrizations.weight.original1', 'encoder.pos_conv_embed.conv.parametrizations.weight.original0']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", "bert loaded\n", "bert_encoder loaded\n", "predictor loaded\n", "decoder loaded\n", "text_encoder loaded\n", "predictor_encoder loaded\n", "style_encoder loaded\n", "diffusion loaded\n", "text_aligner loaded\n", "pitch_extractor loaded\n", "mpd loaded\n", "msd loaded\n", "wd loaded\n", "BERT AdamW (\n", "Parameter Group 0\n", " amsgrad: False\n", " base_momentum: 0.85\n", " betas: (0.9, 0.99)\n", " capturable: False\n", " differentiable: False\n", " eps: 1e-09\n", " foreach: None\n", " fused: None\n", " initial_lr: 1e-05\n", " lr: 1e-05\n", " max_lr: 2e-05\n", " max_momentum: 0.95\n", " maximize: False\n", " min_lr: 0\n", " weight_decay: 0.01\n", ")\n", "decoder AdamW (\n", "Parameter Group 0\n", " amsgrad: False\n", " base_momentum: 0.85\n", " betas: (0.0, 0.99)\n", " capturable: False\n", " differentiable: False\n", " eps: 1e-09\n", " foreach: None\n", " fused: None\n", " initial_lr: 0.0001\n", " lr: 0.0001\n", " max_lr: 0.0002\n", " max_momentum: 0.95\n", " maximize: False\n", " min_lr: 0\n", " weight_decay: 0.0001\n", ")\n", "wavs/9_1.wav 48000\n", "wavs/11_9.wav 48000\n", "wavs/7_0.wav 48000\n", "wavs/2_0.wav 48000\n", "wavs/5_9.wav 44100\n", "wavs/16_4.wav 48000\n", "wavs/12_8.wav 48000\n", "wavs/15_0.wav 48000\n", "wavs/6_0.wav 48000\n", "wavs/5_10.wav 44100\n", "wavs/12_2.wav 48000\n", "wavs/17_0.wav 48000\n", "wavs/14_2.wav 48000\n", "wavs/7_3.wav 48000\n", "wavs/18_0.wav 48000\n", "wavs/0_16.wav 48000\n", "wavs/16_5.wav 48000\n", "wavs/16_18.wav 48000\n", "wavs/0_9.wav 48000\n", "wavs/14_4.wav 48000\n", "wavs/11_7.wav 48000\n", "wavs/9_0.wav 48000\n", "wavs/9_0.wav 48000\n", "wavs/0_7.wav 48000\n", "wavs/16_21.wav 48000\n", "wavs/16_1.wav 48000\n", "wavs/7_0.wav 48000\n", "wavs/16_8.wav 48000\n", "wavs/9_2.wav 48000\n", "wavs/16_16.wav 48000\n", "wavs/11_3.wav 48000\n", "wavs/5_7.wav 44100\n", "wavs/11_1.wav 48000\n", "wavs/4_1.wav 48000\n", "wavs/4_1.wav 48000\n", "wavs/11_11.wav 48000\n", "wavs/11_1.wav 48000\n", "wavs/5_11.wav 44100\n", "wavs/0_7.wav 48000\n", "wavs/5_15.wav 44100\n", "wavs/18_0.wav 48000\n", "wavs/0_10.wav 48000\n", "wavs/5_17.wav 44100\n", "wavs/15_4.wav 48000\n", "wavs/16_12.wav 48000\n", "wavs/15_2.wav 48000\n", "wavs/5_3.wav 44100\n", "wavs/5_1.wav 44100\n", "wavs/2_3.wav 48000\n", "wavs/0_4.wav 48000\n", "wavs/15_3.wav 48000\n", "wavs/16_2.wav 48000\n", "wavs/8_0.wav 48000\n", "wavs/17_0.wav 48000\n", "wavs/14_0.wav 48000\n", "wavs/5_7.wav 44100\n", "wavs/15_4.wav 48000\n", "wavs/7_2.wav 48000\n", "wavs/16_2.wav 48000\n", "wavs/12_6.wav 48000\n", "Epoch [1/100], Step [10/70], Loss: 0.42604, Disc Loss: 3.92244, Dur Loss: 1.39051, CE Loss: 0.06714, Norm Loss: 0.63550, F0 Loss: 2.76960, LM Loss: 1.50625, Gen Loss: 5.05559, Sty Loss: 0.00000, Diff Loss: 0.00000, DiscLM Loss: 0.00000, GenLM Loss: 0.00000, SLoss: 0.00000, S2S Loss: 1.06285, Mono Loss: 0.03130\n", "Time elasped: 49.524219274520874\n", "wavs/18_2.wav 48000\n", "wavs/5_14.wav 44100\n", "wavs/15_8.wav 48000\n", "wavs/12_8.wav 48000\n", "wavs/0_12.wav 48000\n", "wavs/0_3.wav 48000\n", "wavs/2_1.wav 48000\n", "wavs/0_8.wav 48000\n", "wavs/16_6.wav 48000\n", "wavs/11_10.wav 48000\n", "wavs/0_10.wav 48000\n", "wavs/5_5.wav 44100\n", "wavs/13_0.wav 48000\n", "wavs/11_11.wav 48000\n", "wavs/11_1.wav 48000\n", "wavs/10_0.wav 48000\n", "wavs/19_3.wav 48000\n", "wavs/12_5.wav 48000\n", "wavs/15_6.wav 48000\n", "wavs/7_1.wav 48000\n", "wavs/13_0.wav 48000\n", "wavs/5_15.wav 44100\n", "wavs/3_2.wav 48000\n", "wavs/5_12.wav 44100\n", "wavs/19_5.wav 48000\n" ] } ] }, { "cell_type": "markdown", "source": [ "### Test the model quality\n", "\n", "Note that this mainly serves as a proof of concept due to RAM limitation of free Colab instances. A lot of settings are suboptimal. In the future when DDP works for train_second.py, we will also add mixed precision finetuning to save time and RAM. You can also add SLM adversarial training run if you have paid Colab services (such as A100 with 40G of RAM)." ], "metadata": { "id": "I0_7wsGkXGfc" } }, { "cell_type": "code", "source": [ "import nltk\n", "nltk.download('punkt')" ], "metadata": { "id": "OPLphjbncE7p", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "e6968f80-741e-44a2-dbc8-554985178cb9" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stderr", "text": [ "[nltk_data] Downloading package punkt to /root/nltk_data...\n", "[nltk_data] Unzipping tokenizers/punkt.zip.\n" ] }, { "output_type": "execute_result", "data": { "text/plain": [ "True" ] }, "metadata": {}, "execution_count": 11 } ] }, { "cell_type": "code", "source": [ "# @title Set Up Model\n", "config_path = \"/content/drive/Othercomputers/My Laptop/Voice Clones/Me/config_ft.yml\" # @param {type:\"string\"}\n", "import torch\n", "torch.manual_seed(0)\n", "torch.backends.cudnn.benchmark = False\n", "torch.backends.cudnn.deterministic = True\n", "\n", "import random\n", "random.seed(0)\n", "\n", "import numpy as np\n", "np.random.seed(0)\n", "\n", "# load packages\n", "import time\n", "import random\n", "import yaml\n", "from munch import Munch\n", "import numpy as np\n", "import torch\n", "from torch import nn\n", "import torch.nn.functional as F\n", "import torchaudio\n", "import librosa\n", "from nltk.tokenize import word_tokenize\n", "\n", "from models import *\n", "from utils import *\n", "from text_utils import TextCleaner\n", "textclenaer = TextCleaner()\n", "\n", "%matplotlib inline\n", "\n", "to_mel = torchaudio.transforms.MelSpectrogram(\n", " n_mels=80, n_fft=2048, win_length=1200, hop_length=300)\n", "mean, std = -4, 4\n", "\n", "def length_to_mask(lengths):\n", " mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)\n", " mask = torch.gt(mask+1, lengths.unsqueeze(1))\n", " return mask\n", "\n", "def preprocess(wave):\n", " wave_tensor = torch.from_numpy(wave).float()\n", " mel_tensor = to_mel(wave_tensor)\n", " mel_tensor = (torch.log(1e-5 + mel_tensor.unsqueeze(0)) - mean) / std\n", " return mel_tensor\n", "\n", "def compute_style(path):\n", " wave, sr = librosa.load(path, sr=24000)\n", " audio, index = librosa.effects.trim(wave, top_db=30)\n", " if sr != 24000:\n", " audio = librosa.resample(audio, sr, 24000)\n", " mel_tensor = preprocess(audio).to(device)\n", "\n", " with torch.no_grad():\n", " ref_s = model.style_encoder(mel_tensor.unsqueeze(1))\n", " ref_p = model.predictor_encoder(mel_tensor.unsqueeze(1))\n", "\n", " return torch.cat([ref_s, ref_p], dim=1)\n", "\n", "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n", "\n", "# load phonemizer\n", "import phonemizer\n", "global_phonemizer = phonemizer.backend.EspeakBackend(language='en-us', preserve_punctuation=True, with_stress=True)\n", "\n", "config = yaml.safe_load(open(config_path))\n", "\n", "# load pretrained ASR model\n", "ASR_config = config.get('ASR_config', False)\n", "ASR_path = config.get('ASR_path', False)\n", "text_aligner = load_ASR_models(ASR_path, ASR_config)\n", "\n", "# load pretrained F0 model\n", "F0_path = config.get('F0_path', False)\n", "pitch_extractor = load_F0_models(F0_path)\n", "\n", "# load BERT model\n", "from Utils.PLBERT.util import load_plbert\n", "BERT_path = config.get('PLBERT_dir', False)\n", "plbert = load_plbert(BERT_path)\n", "\n", "model_params = recursive_munch(config['model_params'])\n", "model = build_model(model_params, text_aligner, pitch_extractor, plbert)\n", "_ = [model[key].eval() for key in model]\n", "_ = [model[key].to(device) for key in model]" ], "metadata": { "id": "jIIAoDACXJL0", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "bf9cfb66-34c3-405d-d64f-e2cedec810c4" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "177\n" ] } ] }, { "cell_type": "code", "source": [ "# @title Load model\n", "model_dir = \"/content/drive/Othercomputers/My Laptop/Voice Clones/Me/epoch_2nd_00009.pth\" # @param {type:\"string\"}\n", "params_whole = torch.load(model_dir)\n", "params = params_whole['net']" ], "metadata": { "id": "ULuU9-VDb9Pk" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "for key in model:\n", " if key in params:\n", " print('%s loaded' % key)\n", " try:\n", " model[key].load_state_dict(params[key])\n", " except:\n", " from collections import OrderedDict\n", " state_dict = params[key]\n", " new_state_dict = OrderedDict()\n", " for k, v in state_dict.items():\n", " name = k[7:] # remove `module.`\n", " new_state_dict[name] = v\n", " # load params\n", " model[key].load_state_dict(new_state_dict, strict=False)\n", "# except:\n", "# _load(params[key], model[key])\n", "_ = [model[key].eval() for key in model]" ], "metadata": { "id": "J-U29yIYc2ea", "colab": { "base_uri": "https://localhost:8080/" }, "outputId": "780e1bc9-9a4a-4675-afc2-3d797771d6a9" }, "execution_count": null, "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ "bert loaded\n", "bert_encoder loaded\n", "predictor loaded\n", "decoder loaded\n", "text_encoder loaded\n", "predictor_encoder loaded\n", "style_encoder loaded\n", "diffusion loaded\n", "text_aligner loaded\n", "pitch_extractor loaded\n", "mpd loaded\n", "msd loaded\n", "wd loaded\n" ] } ] }, { "cell_type": "code", "source": [ "from Modules.diffusion.sampler import DiffusionSampler, ADPM2Sampler, KarrasSchedule" ], "metadata": { "id": "jrPQ_Yrwc3n6" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "sampler = DiffusionSampler(\n", " model.diffusion.diffusion,\n", " sampler=ADPM2Sampler(),\n", " sigma_schedule=KarrasSchedule(sigma_min=0.0001, sigma_max=3.0, rho=9.0), # empirical parameters\n", " clamp=False\n", ")" ], "metadata": { "id": "n2CWYNoqc455" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "def inference(text, ref_s, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1):\n", " text = text.strip()\n", " ps = global_phonemizer.phonemize([text])\n", " ps = word_tokenize(ps[0])\n", " ps = ' '.join(ps)\n", " tokens = textclenaer(ps)\n", " tokens.insert(0, 0)\n", " tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)\n", "\n", " with torch.no_grad():\n", " input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)\n", " text_mask = length_to_mask(input_lengths).to(device)\n", "\n", " t_en = model.text_encoder(tokens, input_lengths, text_mask)\n", " bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())\n", " d_en = model.bert_encoder(bert_dur).transpose(-1, -2)\n", "\n", " s_pred = sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device),\n", " embedding=bert_dur,\n", " embedding_scale=embedding_scale,\n", " features=ref_s, # reference from the same speaker as the embedding\n", " num_steps=diffusion_steps).squeeze(1)\n", "\n", "\n", " s = s_pred[:, 128:]\n", " ref = s_pred[:, :128]\n", "\n", " ref = alpha * ref + (1 - alpha) * ref_s[:, :128]\n", " s = beta * s + (1 - beta) * ref_s[:, 128:]\n", "\n", " d = model.predictor.text_encoder(d_en,\n", " s, input_lengths, text_mask)\n", "\n", " x, _ = model.predictor.lstm(d)\n", " duration = model.predictor.duration_proj(x)\n", "\n", " duration = torch.sigmoid(duration).sum(axis=-1)\n", " pred_dur = torch.round(duration.squeeze()).clamp(min=1)\n", "\n", " pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))\n", " c_frame = 0\n", " for i in range(pred_aln_trg.size(0)):\n", " pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1\n", " c_frame += int(pred_dur[i].data)\n", "\n", " # encode prosody\n", " en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))\n", " if model_params.decoder.type == \"hifigan\":\n", " asr_new = torch.zeros_like(en)\n", " asr_new[:, :, 0] = en[:, :, 0]\n", " asr_new[:, :, 1:] = en[:, :, 0:-1]\n", " en = asr_new\n", "\n", " F0_pred, N_pred = model.predictor.F0Ntrain(en, s)\n", "\n", " asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))\n", " if model_params.decoder.type == \"hifigan\":\n", " asr_new = torch.zeros_like(asr)\n", " asr_new[:, :, 0] = asr[:, :, 0]\n", " asr_new[:, :, 1:] = asr[:, :, 0:-1]\n", " asr = asr_new\n", "\n", " out = model.decoder(asr,\n", " F0_pred, N_pred, ref.squeeze().unsqueeze(0))\n", "\n", "\n", " return out.squeeze().cpu().numpy()[..., :-50] # weird pulse at the end of the model, need to be fixed later" ], "metadata": { "id": "2x5kVb3nc_eY" }, "execution_count": null, "outputs": [] }, { "cell_type": "markdown", "source": [ "### Synthesize speech" ], "metadata": { "id": "O159JnwCc6CC" } }, { "cell_type": "code", "source": [ "# @title Text to synthesise\n", "text = \"This is a test using Style TTS 2.\" # @param {type:\"string\"}" ], "metadata": { "id": "ThciXQ6rc9Eq" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "# @title Load Sample Data\n", "path = \"/content/drive/Othercomputers/My Laptop/Voice Clones/Me/wavs/0_16.wav\" # @param {type:\"string\"}\n", "# get a random reference in the training set, note that it doesn't matter which one you use\n", "\n", "# this style vector ref_s can be saved as a parameter together with the model weights\n", "ref_s = compute_style(path)" ], "metadata": { "id": "jldPkJyCc83a", "cellView": "form" }, "execution_count": null, "outputs": [] }, { "cell_type": "code", "source": [ "start = time.time()\n", "wav = inference(text, ref_s, alpha=0.9, beta=0.9, diffusion_steps=20, embedding_scale=1)\n", "rtf = (time.time() - start) / (len(wav) / 24000)\n", "print(f\"RTF = {rtf:5f}\")\n", "import IPython.display as ipd\n", "display(ipd.Audio(wav, rate=24000, normalize=False))" ], "metadata": { "id": "_mIU0jqDdQ-c", "colab": { "base_uri": "https://localhost:8080/", "height": 400 }, "outputId": "bfa55741-4fb0-4d8c-be4b-2f0c9e807dee" }, "execution_count": null, "outputs": [ { "output_type": "error", "ename": "ValueError", "evalue": "ignored", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", "\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)", "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mwav\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minference\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mref_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malpha\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeta\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.9\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdiffusion_steps\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0membedding_scale\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 3\u001b[0m \u001b[0mrtf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwav\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m/\u001b[0m \u001b[0;36m24000\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"RTF = {rtf:5f}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 5\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mIPython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdisplay\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mipd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;32m\u001b[0m in \u001b[0;36minference\u001b[0;34m(text, ref_s, alpha, beta, diffusion_steps, embedding_scale)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0mpred_dur\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mround\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mduration\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msqueeze\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclamp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmin\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0mpred_aln_trg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_lengths\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_dur\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0mc_frame\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpred_aln_trg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", "\u001b[0;31mValueError\u001b[0m: cannot convert float NaN to integer" ] } ] } ] }