From 0b315f3520ed8d09a0d5f46e63955fa0c6002b19 Mon Sep 17 00:00:00 2001 From: Michael Chung Date: Thu, 5 Oct 2023 12:56:40 -0400 Subject: [PATCH 1/9] Ok with the model file this time --- tts/VALL-E-X/.gitignore | 160 + tts/VALL-E-X/Dockerfile | 32 + tts/VALL-E-X/LICENSE | 21 + tts/VALL-E-X/README-ZH.md | 342 + tts/VALL-E-X/README.md | 353 + tts/VALL-E-X/customs/ph.txt | 0 tts/VALL-E-X/data/__init__.py | 3 + tts/VALL-E-X/data/collation.py | 120 + tts/VALL-E-X/data/datamodule.py | 419 + tts/VALL-E-X/data/dataset.py | 242 + tts/VALL-E-X/data/fbank.py | 212 + tts/VALL-E-X/data/input_strategies.py | 159 + tts/VALL-E-X/data/tokenizer.py | 376 + tts/VALL-E-X/descriptions.py | 31 + tts/VALL-E-X/examples.py | 24 + tts/VALL-E-X/images/vallex_framework.jpg | Bin 0 -> 52189 bytes tts/VALL-E-X/launch-ui.py | 628 + tts/VALL-E-X/macros.py | 39 + tts/VALL-E-X/main.py | 216 + tts/VALL-E-X/model-card.md | 33 + tts/VALL-E-X/models/__init__.py | 136 + tts/VALL-E-X/models/macros.py | 11 + tts/VALL-E-X/models/transformer.py | 394 + tts/VALL-E-X/models/vallex.py | 853 + tts/VALL-E-X/models/visualizer.py | 106 + tts/VALL-E-X/modules/__init__.py | 0 tts/VALL-E-X/modules/activation.py | 612 + tts/VALL-E-X/modules/embedding.py | 97 + tts/VALL-E-X/modules/optim.py | 1105 + tts/VALL-E-X/modules/scaling.py | 1401 + tts/VALL-E-X/modules/scheduler.py | 78 + tts/VALL-E-X/modules/transformer.py | 683 + .../nltk_data/tokenizers/punkt/PY3/README | 98 + .../tokenizers/punkt/PY3/czech.pickle | Bin 0 -> 1119050 bytes .../tokenizers/punkt/PY3/danish.pickle | Bin 0 -> 1191710 bytes .../tokenizers/punkt/PY3/dutch.pickle | Bin 0 -> 693759 bytes .../tokenizers/punkt/PY3/english.pickle | Bin 0 -> 406697 bytes .../tokenizers/punkt/PY3/estonian.pickle | Bin 0 -> 1499502 bytes .../tokenizers/punkt/PY3/finnish.pickle | Bin 0 -> 1852226 bytes .../tokenizers/punkt/PY3/french.pickle | Bin 0 -> 553575 bytes .../tokenizers/punkt/PY3/german.pickle | Bin 0 -> 1463575 bytes .../tokenizers/punkt/PY3/greek.pickle | Bin 0 -> 876006 bytes .../tokenizers/punkt/PY3/italian.pickle | Bin 0 -> 615089 bytes .../tokenizers/punkt/PY3/malayalam.pickle | Bin 0 -> 221207 bytes .../tokenizers/punkt/PY3/norwegian.pickle | Bin 0 -> 1181271 bytes .../tokenizers/punkt/PY3/polish.pickle | Bin 0 -> 1738386 bytes .../tokenizers/punkt/PY3/portuguese.pickle | Bin 0 -> 611919 bytes .../tokenizers/punkt/PY3/russian.pickle | Bin 0 -> 33020 bytes .../tokenizers/punkt/PY3/slovene.pickle | Bin 0 -> 734444 bytes .../tokenizers/punkt/PY3/spanish.pickle | Bin 0 -> 562337 bytes .../tokenizers/punkt/PY3/swedish.pickle | Bin 0 -> 979681 bytes .../tokenizers/punkt/PY3/turkish.pickle | Bin 0 -> 1017038 bytes .../nltk_data/tokenizers/punkt/README | 98 + .../nltk_data/tokenizers/punkt/czech.pickle | 159140 ++++++++++ .../nltk_data/tokenizers/punkt/danish.pickle | 162767 ++++++++++ .../nltk_data/tokenizers/punkt/dutch.pickle | 97138 ++++++ .../nltk_data/tokenizers/punkt/english.pickle | 61702 ++++ .../tokenizers/punkt/estonian.pickle | 206369 ++++++++++++ .../nltk_data/tokenizers/punkt/finnish.pickle | 240379 ++++++++++++++ .../nltk_data/tokenizers/punkt/french.pickle | 80529 +++++ .../nltk_data/tokenizers/punkt/german.pickle | 181299 +++++++++++ .../nltk_data/tokenizers/punkt/greek.pickle | 89257 ++++++ .../nltk_data/tokenizers/punkt/italian.pickle | 90202 ++++++ .../tokenizers/punkt/malayalam.pickle | Bin 0 -> 221207 bytes .../tokenizers/punkt/norwegian.pickle | 162978 ++++++++++ .../nltk_data/tokenizers/punkt/polish.pickle | 245172 +++++++++++++++ .../tokenizers/punkt/portuguese.pickle | 90795 ++++++ .../nltk_data/tokenizers/punkt/russian.pickle | Bin 0 -> 33027 bytes .../nltk_data/tokenizers/punkt/slovene.pickle | 106925 +++++++ .../nltk_data/tokenizers/punkt/spanish.pickle | 82636 +++++ .../nltk_data/tokenizers/punkt/swedish.pickle | 133719 ++++++++ .../nltk_data/tokenizers/punkt/turkish.pickle | 138187 ++++++++ tts/VALL-E-X/presets/acou_1.npz | Bin 0 -> 15650 bytes tts/VALL-E-X/presets/acou_2.npz | Bin 0 -> 15426 bytes tts/VALL-E-X/presets/acou_3.npz | Bin 0 -> 15410 bytes tts/VALL-E-X/presets/acou_4.npz | Bin 0 -> 15506 bytes tts/VALL-E-X/presets/alan.npz | Bin 0 -> 50002 bytes tts/VALL-E-X/presets/amused.npz | Bin 0 -> 20882 bytes tts/VALL-E-X/presets/anger.npz | Bin 0 -> 22090 bytes tts/VALL-E-X/presets/babara.npz | Bin 0 -> 11526 bytes tts/VALL-E-X/presets/bronya.npz | Bin 0 -> 18106 bytes tts/VALL-E-X/presets/cafe.npz | Bin 0 -> 22442 bytes tts/VALL-E-X/presets/dingzhen.npz | Bin 0 -> 18154 bytes tts/VALL-E-X/presets/disgust.npz | Bin 0 -> 39386 bytes tts/VALL-E-X/presets/emo_amused.npz | Bin 0 -> 15378 bytes tts/VALL-E-X/presets/emo_anger.npz | Bin 0 -> 15434 bytes tts/VALL-E-X/presets/emo_neutral.npz | Bin 0 -> 15578 bytes tts/VALL-E-X/presets/emo_sleepy.npz | Bin 0 -> 15370 bytes tts/VALL-E-X/presets/emotion_sleepiness.npz | Bin 0 -> 33218 bytes tts/VALL-E-X/presets/en2zh_tts_1.npz | Bin 0 -> 44354 bytes tts/VALL-E-X/presets/en2zh_tts_2.npz | Bin 0 -> 24178 bytes tts/VALL-E-X/presets/en2zh_tts_3.npz | Bin 0 -> 21410 bytes tts/VALL-E-X/presets/en2zh_tts_4.npz | Bin 0 -> 43826 bytes tts/VALL-E-X/presets/esta.npz | Bin 0 -> 40250 bytes tts/VALL-E-X/presets/fuxuan.npz | Bin 0 -> 50330 bytes tts/VALL-E-X/presets/librispeech_1.npz | Bin 0 -> 15650 bytes tts/VALL-E-X/presets/librispeech_2.npz | Bin 0 -> 15418 bytes tts/VALL-E-X/presets/librispeech_3.npz | Bin 0 -> 15634 bytes tts/VALL-E-X/presets/librispeech_4.npz | Bin 0 -> 15594 bytes tts/VALL-E-X/presets/neutral.npz | Bin 0 -> 21050 bytes tts/VALL-E-X/presets/paimon.npz | Bin 0 -> 13762 bytes tts/VALL-E-X/presets/rosalia.npz | Bin 0 -> 11414 bytes tts/VALL-E-X/presets/seel.npz | Bin 0 -> 13554 bytes tts/VALL-E-X/presets/sleepiness.npz | Bin 0 -> 33218 bytes tts/VALL-E-X/presets/vctk_1.npz | Bin 0 -> 15530 bytes tts/VALL-E-X/presets/vctk_2.npz | Bin 0 -> 15458 bytes tts/VALL-E-X/presets/vctk_3.npz | Bin 0 -> 15450 bytes tts/VALL-E-X/presets/vctk_4.npz | Bin 0 -> 15330 bytes tts/VALL-E-X/presets/yaesakura.npz | Bin 0 -> 12442 bytes tts/VALL-E-X/presets/zh2en_tts_1.npz | Bin 0 -> 23546 bytes tts/VALL-E-X/presets/zh2en_tts_2.npz | Bin 0 -> 20170 bytes tts/VALL-E-X/presets/zh2en_tts_3.npz | Bin 0 -> 17906 bytes tts/VALL-E-X/presets/zh2en_tts_4.npz | Bin 0 -> 33682 bytes tts/VALL-E-X/prompts/ja-2.ogg | Bin 0 -> 31440 bytes tts/VALL-E-X/prompts/ph.txt | 0 tts/VALL-E-X/requirements.txt | 24 + tts/VALL-E-X/test.py | 94 + tts/VALL-E-X/utils/__init__.py | 15 + tts/VALL-E-X/utils/download.py | 49 + tts/VALL-E-X/utils/g2p/__init__.py | 72 + tts/VALL-E-X/utils/g2p/bpe_1024.json | 2049 + tts/VALL-E-X/utils/g2p/bpe_69.json | 141 + tts/VALL-E-X/utils/g2p/cleaners.py | 61 + tts/VALL-E-X/utils/g2p/english.py | 188 + tts/VALL-E-X/utils/g2p/japanese.py | 154 + tts/VALL-E-X/utils/g2p/mandarin.py | 326 + tts/VALL-E-X/utils/g2p/symbols.py | 76 + tts/VALL-E-X/utils/generation.py | 334 + tts/VALL-E-X/utils/prompt_making.py | 119 + tts/VALL-E-X/utils/sentence_cutter.py | 54 + tts/VALL-E-X/utils/symbol_table.py | 287 + 131 files changed, 2342319 insertions(+) create mode 100644 tts/VALL-E-X/.gitignore create mode 100644 tts/VALL-E-X/Dockerfile create mode 100644 tts/VALL-E-X/LICENSE create mode 100644 tts/VALL-E-X/README-ZH.md create mode 100644 tts/VALL-E-X/README.md create mode 100644 tts/VALL-E-X/customs/ph.txt create mode 100644 tts/VALL-E-X/data/__init__.py create mode 100644 tts/VALL-E-X/data/collation.py create mode 100644 tts/VALL-E-X/data/datamodule.py create mode 100644 tts/VALL-E-X/data/dataset.py create mode 100644 tts/VALL-E-X/data/fbank.py create mode 100644 tts/VALL-E-X/data/input_strategies.py create mode 100644 tts/VALL-E-X/data/tokenizer.py create mode 100644 tts/VALL-E-X/descriptions.py create mode 100644 tts/VALL-E-X/examples.py create mode 100644 tts/VALL-E-X/images/vallex_framework.jpg create mode 100644 tts/VALL-E-X/launch-ui.py create mode 100644 tts/VALL-E-X/macros.py create mode 100644 tts/VALL-E-X/main.py create mode 100644 tts/VALL-E-X/model-card.md create mode 100644 tts/VALL-E-X/models/__init__.py create mode 100644 tts/VALL-E-X/models/macros.py create mode 100644 tts/VALL-E-X/models/transformer.py create mode 100644 tts/VALL-E-X/models/vallex.py create mode 100644 tts/VALL-E-X/models/visualizer.py create mode 100644 tts/VALL-E-X/modules/__init__.py create mode 100644 tts/VALL-E-X/modules/activation.py create mode 100644 tts/VALL-E-X/modules/embedding.py create mode 100644 tts/VALL-E-X/modules/optim.py create mode 100644 tts/VALL-E-X/modules/scaling.py create mode 100644 tts/VALL-E-X/modules/scheduler.py create mode 100644 tts/VALL-E-X/modules/transformer.py create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/README create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/czech.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/danish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/dutch.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/english.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/estonian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/finnish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/french.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/german.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/greek.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/italian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/malayalam.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/norwegian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/polish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/portuguese.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/russian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/slovene.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/spanish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/swedish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/turkish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/README create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/czech.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/danish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/dutch.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/english.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/estonian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/finnish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/french.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/german.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/greek.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/italian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/malayalam.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/norwegian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/polish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/portuguese.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/russian.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/slovene.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/spanish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/swedish.pickle create mode 100644 tts/VALL-E-X/nltk_data/tokenizers/punkt/turkish.pickle create mode 100644 tts/VALL-E-X/presets/acou_1.npz create mode 100644 tts/VALL-E-X/presets/acou_2.npz create mode 100644 tts/VALL-E-X/presets/acou_3.npz create mode 100644 tts/VALL-E-X/presets/acou_4.npz create mode 100644 tts/VALL-E-X/presets/alan.npz create mode 100644 tts/VALL-E-X/presets/amused.npz create mode 100644 tts/VALL-E-X/presets/anger.npz create mode 100644 tts/VALL-E-X/presets/babara.npz create mode 100644 tts/VALL-E-X/presets/bronya.npz create mode 100644 tts/VALL-E-X/presets/cafe.npz create mode 100644 tts/VALL-E-X/presets/dingzhen.npz create mode 100644 tts/VALL-E-X/presets/disgust.npz create mode 100644 tts/VALL-E-X/presets/emo_amused.npz create mode 100644 tts/VALL-E-X/presets/emo_anger.npz create mode 100644 tts/VALL-E-X/presets/emo_neutral.npz create mode 100644 tts/VALL-E-X/presets/emo_sleepy.npz create mode 100644 tts/VALL-E-X/presets/emotion_sleepiness.npz create mode 100644 tts/VALL-E-X/presets/en2zh_tts_1.npz create mode 100644 tts/VALL-E-X/presets/en2zh_tts_2.npz create mode 100644 tts/VALL-E-X/presets/en2zh_tts_3.npz create mode 100644 tts/VALL-E-X/presets/en2zh_tts_4.npz create mode 100644 tts/VALL-E-X/presets/esta.npz create mode 100644 tts/VALL-E-X/presets/fuxuan.npz create mode 100644 tts/VALL-E-X/presets/librispeech_1.npz create mode 100644 tts/VALL-E-X/presets/librispeech_2.npz create mode 100644 tts/VALL-E-X/presets/librispeech_3.npz create mode 100644 tts/VALL-E-X/presets/librispeech_4.npz create mode 100644 tts/VALL-E-X/presets/neutral.npz create mode 100644 tts/VALL-E-X/presets/paimon.npz create mode 100644 tts/VALL-E-X/presets/rosalia.npz create mode 100644 tts/VALL-E-X/presets/seel.npz create mode 100644 tts/VALL-E-X/presets/sleepiness.npz create mode 100644 tts/VALL-E-X/presets/vctk_1.npz create mode 100644 tts/VALL-E-X/presets/vctk_2.npz create mode 100644 tts/VALL-E-X/presets/vctk_3.npz create mode 100644 tts/VALL-E-X/presets/vctk_4.npz create mode 100644 tts/VALL-E-X/presets/yaesakura.npz create mode 100644 tts/VALL-E-X/presets/zh2en_tts_1.npz create mode 100644 tts/VALL-E-X/presets/zh2en_tts_2.npz create mode 100644 tts/VALL-E-X/presets/zh2en_tts_3.npz create mode 100644 tts/VALL-E-X/presets/zh2en_tts_4.npz create mode 100644 tts/VALL-E-X/prompts/ja-2.ogg create mode 100644 tts/VALL-E-X/prompts/ph.txt create mode 100644 tts/VALL-E-X/requirements.txt create mode 100644 tts/VALL-E-X/test.py create mode 100644 tts/VALL-E-X/utils/__init__.py create mode 100644 tts/VALL-E-X/utils/download.py create mode 100644 tts/VALL-E-X/utils/g2p/__init__.py create mode 100644 tts/VALL-E-X/utils/g2p/bpe_1024.json create mode 100644 tts/VALL-E-X/utils/g2p/bpe_69.json create mode 100644 tts/VALL-E-X/utils/g2p/cleaners.py create mode 100644 tts/VALL-E-X/utils/g2p/english.py create mode 100644 tts/VALL-E-X/utils/g2p/japanese.py create mode 100644 tts/VALL-E-X/utils/g2p/mandarin.py create mode 100644 tts/VALL-E-X/utils/g2p/symbols.py create mode 100644 tts/VALL-E-X/utils/generation.py create mode 100644 tts/VALL-E-X/utils/prompt_making.py create mode 100644 tts/VALL-E-X/utils/sentence_cutter.py create mode 100644 tts/VALL-E-X/utils/symbol_table.py diff --git a/tts/VALL-E-X/.gitignore b/tts/VALL-E-X/.gitignore new file mode 100644 index 0000000..d756434 --- /dev/null +++ b/tts/VALL-E-X/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +*.wav +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ \ No newline at end of file diff --git a/tts/VALL-E-X/Dockerfile b/tts/VALL-E-X/Dockerfile new file mode 100644 index 0000000..78e1bde --- /dev/null +++ b/tts/VALL-E-X/Dockerfile @@ -0,0 +1,32 @@ +# Use ubuntu as the base image from the hub +# then have to build this as a image. +FROM nvidia/cuda:12.2.0-devel-ubuntu20.04 +# Optional: Set environment variables +ENV MY_ENV_VARIABLE=value +ENV DEBIAN_FRONTEND=noninteractive +# Optional: Run commands to set up the image +RUN apt-get update && apt-get install -y curl + + # Preconfigure package selections (as an example) +RUN echo "package-name package-name/some-option select option-value" | debconf-set-selections && \ + apt-get install -y software-properties-common +RUN apt install software-properties-common -y +RUN add-apt-repository ppa:deadsnakes/ppa +RUN apt install python3.10 -y +RUN apt-get install -y git +RUN sudo apt install ffmpeg +# Clone the repository + +# RUN pwd +RUN pip3 install requirements.txt + +# Navigate to the cloned directory and checkout the specific commit +WORKDIR TestingDockerIO +COPY VALL-E-X . + +RUN git pull +RUN git reset --hard origin/main +#RUN git checkout 47c491a29d10c04c6f1efd01bc7b334d11cad4f2 + +# # Allow parameters to be passed +# ENTRYPOINT ["python3", "main.py"] diff --git a/tts/VALL-E-X/LICENSE b/tts/VALL-E-X/LICENSE new file mode 100644 index 0000000..43c68b1 --- /dev/null +++ b/tts/VALL-E-X/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Songting + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tts/VALL-E-X/README-ZH.md b/tts/VALL-E-X/README-ZH.md new file mode 100644 index 0000000..357154e --- /dev/null +++ b/tts/VALL-E-X/README-ZH.md @@ -0,0 +1,342 @@ +# VALL-E X: 多语言文本到语音合成与语音克隆 🔊 +[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/qCBRmAnTxg) +
+[English](README.md) | 中文 +
+微软[VALL-E X](https://arxiv.org/pdf/2303.03926) 零样本语音合成模型的开源实现.
+**预训练模型现已向公众开放，供研究或应用使用。** +![vallex-framework](/images/vallex_framework.jpg "VALL-E X framework") + +VALL-E X 是一个强大而创新的多语言文本转语音（TTS）模型，最初由微软发布。虽然微软最初在他们的研究论文中提出了该概念，但并未发布任何代码或预训练模型。我们认识到了这项技术的潜力和价值，复现并训练了一个开源可用的VALL-E X模型。我们很乐意与社区分享我们的预训练模型，让每个人都能体验到次世代TTS的威力。 🎧 +
+更多细节请查看 [model card](./model-card.md). + +## 📖 目录 +* [🚀 更新日志](#-更新日志) +* [📢 功能特点](#-功能特点) +* [💻 本地安装](#-本地安装) +* [🎧 在线Demo](#-在线Demo) +* [🐍 使用方法](#-Python中的使用方法) +* [❓ FAQ](#-faq) +* [🧠 TODO](#-todo) + +## 🚀 Updates +**2023.09.10** +- 支持AR decoder的batch decoding以实现更稳定的生成结果 + +**2023.08.30** +- 将EnCodec解码器替换成了Vocos解码器，提升了音质。 (感谢[@v0xie](https://github.com/v0xie)) + +**2023.08.23** +- 加入了长文本生成功能 + +**2023.08.20** +- 加入了中文版README + +**2023.08.14** +- 预训练模型权重已发布，从[这里](https://drive.google.com/file/d/10gdQWvP-K_e1undkvv0p2b7SU6I4Egyl/view?usp=sharing)下载。 + +## 💻 本地安装 +### 使用pip安装，推荐使用Python 3.10，CUDA 11.7 ~ 12.0，PyTorch 2.0+ +```commandline +git clone https://github.com/Plachtaa/VALL-E-X.git +cd VALL-E-X +pip install -r requirements.txt +``` + +> 注意：如果需要制作prompt，需要安装 ffmpeg 并将其所在文件夹加入到环境变量PATH中 + +第一次运行程序时，会自动下载相应的模型。如果下载失败并报错，请按照以下步骤手动下载模型。 + +（请注意目录和文件夹的大小写） + +1.检查安装目录下是否存在`checkpoints`文件夹，如果没有，在安装目录下手动创建`checkpoints`文件夹（`./checkpoints/`）。 + +2.检查`checkpoints`文件夹中是否有`vallex-checkpoint.pt`文件。如果没有，请从[这里](https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt) +手动下载`vallex-checkpoint.pt`文件并放到`checkpoints`文件夹里。 + +3.检查安装目录下是否存在`whisper`文件夹，如果没有，在安装目录下手动创建`whisper`文件夹（`./whisper/`）。 + +4.检查`whisper`文件夹中是否有`medium.pt`文件。如果没有，请从[这里](https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt) +手动下载`medium.pt`文件并放到`whisper`文件夹里。 + +## 🎧 在线Demo +如果你不想在本地安装，你可以在线体验VALL-E X的功能，点击下面的任意一个链接即可开始体验。 +
+[![Open in Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue.svg)](https://huggingface.co/spaces/Plachta/VALL-E-X) +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1yyD_sz531QntLKowMHo-XxorsFBCfKul?usp=sharing) + + +## 📢 功能特点 + +VALL-E X 配备有一系列尖端功能： + +1. **多语言 TTS**: 可使用三种语言 - 英语、中文和日语 - 进行自然、富有表现力的语音合成。 + +2. **零样本语音克隆**: 仅需录制任意说话人的短短的 3~10 秒录音，VALL-E X 就能生成个性化、高质量的语音，完美还原他们的声音。 + +
+
查看示例
 + +[prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/a7baa51d-a53a-41cc-a03d-6970f25fcca7) + + +[output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/b895601a-d126-4138-beff-061aabdc7985) + +
+ +3. **语音情感控制**: VALL-E X 可以合成与给定说话人录音相同情感的语音，为音频增添更多表现力。 + +
+
查看示例
 + +https://github.com/Plachtaa/VALL-E-X/assets/112609742/56fa9988-925e-4757-82c5-83ecb0df6266 + + +https://github.com/Plachtaa/VALL-E-X/assets/112609742/699c47a3-d502-4801-8364-bd89bcc0b8f1 + +
+ +4. **零样本跨语言语音合成**: VALL-E X 可以合成与给定说话人母语不同的另一种语言，在不影响口音和流利度的同时，保留该说话人的音色与情感。以下是一个使用日语母语者进行英文与中文合成的样例： 🇯🇵 🗣 + +
+
查看示例
 + +[jp-prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/ea6e2ee4-139a-41b4-837e-0bd04dda6e19) + + +[en-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/db8f9782-923f-425e-ba94-e8c1bd48f207) + + +[zh-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/15829d79-e448-44d3-8965-fafa7a3f8c28) + +
+ +5. **口音控制**: VALL-E X 允许您控制所合成音频的口音，比如说中文带英语口音或反之。 🇨🇳 💬 + +
+
查看示例
 + +[en-prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/f688d7f6-70ef-46ec-b1cc-355c31e78b3b) + + +[zh-accent-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/be59c7ca-b45b-44ca-a30d-4d800c950ccc) + + +[en-accent-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/8b4f4f9b-f299-4ea4-a548-137437b71738) + +
+ +6. **声学环境保留**: 当给定说话人的录音在不同的声学环境下录制时，VALL-E X 可以保留该声学环境，使合成语音听起来更加自然。 + +
+
查看示例
 + +[noise-prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/68986d88-abd0-4d1d-96e4-4f893eb9259e) + + +[noise-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/96c4c612-4516-4683-8804-501b70938608) + +
+ + +你可以访问我们的[demo页面](https://plachtaa.github.io/) 来浏览更多示例! + +## 💻 Python中的使用方法 + +
+

🪑 基本使用

 + +```python +from utils.generation import SAMPLE_RATE, generate_audio, preload_models +from scipy.io.wavfile import write as write_wav +from IPython.display import Audio + +# download and load all models +preload_models() + +# generate audio from text +text_prompt = """ +Hello, my name is Nose. And uh, and I like hamburger. Hahaha... But I also have other interests such as playing tactic toast. +""" +audio_array = generate_audio(text_prompt) + +# save audio to disk +write_wav("vallex_generation.wav", SAMPLE_RATE, audio_array) + +# play text in notebook +Audio(audio_array, rate=SAMPLE_RATE) +``` + +[hamburger.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/578d7bbe-cda9-483e-898c-29646edc8f2e) + +
+ +
+

🌎 多语言

 +
+该VALL-E X实现支持三种语言：英语、中文和日语。您可以通过设置`language`参数来指定语言。默认情况下，该模型将自动检测语言。 +
+ +```python + +text_prompt = """ + チュソクは私のお気に入りの祭りです。 私は数日間休んで、友人や家族との時間を過ごすことができます。 +""" +audio_array = generate_audio(text_prompt) +``` + +[vallex_japanese.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/ee57a688-3e83-4be5-b0fe-019d16eec51c) + +*注意：即使在一句话中混合多种语言的情况下，VALL-E X也能完美地控制口音，但是您需要手动标记各个句子对应的语言以便于我们的G2P工具识别它们。* +```python +text_prompt = """ + [EN]The Thirty Years' War was a devastating conflict that had a profound impact on Europe.[EN] + [ZH]这是历史的开始。 如果您想听更多，请继续。[ZH] +""" +audio_array = generate_audio(text_prompt, language='mix') +``` + +[vallex_codeswitch.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/d8667abf-bd08-499f-a383-a861d852f98a) + +
+ +
+

📼 预设音色

 + +我们提供十几种说话人音色可直接VALL-E X使用! 在[这里](/presets)浏览所有可用音色。 + +> VALL-E X 尝试匹配给定预设音色的音调、音高、情感和韵律。该模型还尝试保留音乐、环境噪声等。 +```python +text_prompt = """ +I am an innocent boy with a smoky voice. It is a great honor for me to speak at the United Nations today. +""" +audio_array = generate_audio(text_prompt, prompt="dingzhen") +``` + +[smoky.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/d3f55732-b1cd-420f-87d6-eab60db14dc5) + +
+ +
+

🎙声音克隆

 + +VALL-E X 支持声音克隆！你可以使用任何人，角色，甚至是你自己的声音，来制作一个音频提示。在你使用该音频提示时，VALL-E X 将会使用与其相似的声音来合成文本。 +
+你需要提供一段3~10秒长的语音，以及该语音对应的文本，来制作音频提示。你也可以将文本留空，让[Whisper](https://github.com/openai/whisper)模型为你生成文本。 +> VALL-E X 尝试匹配给定音频提示的音调、音高、情感和韵律。该模型还尝试保留音乐、环境噪声等。 + +```python +from utils.prompt_making import make_prompt + +### Use given transcript +make_prompt(name="paimon", audio_prompt_path="paimon_prompt.wav", + transcript="Just, what was that? Paimon thought we were gonna get eaten.") + +### Alternatively, use whisper +make_prompt(name="paimon", audio_prompt_path="paimon_prompt.wav") +``` +来尝试一下刚刚做好的音频提示吧！ +```python +from utils.generation import SAMPLE_RATE, generate_audio, preload_models +from scipy.io.wavfile import write as write_wav + +# download and load all models +preload_models() + +text_prompt = """ +Hey, Traveler, Listen to this, This machine has taken my voice, and now it can talk just like me! +""" +audio_array = generate_audio(text_prompt, prompt="paimon") + +write_wav("paimon_cloned.wav", SAMPLE_RATE, audio_array) + +``` + +[paimon_prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/e7922859-9d12-4e2a-8651-e156e4280311) + + +[paimon_cloned.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/60d3b7e9-5ead-4024-b499-a897ce5f3d5e) + + +
+ + +
+

🎢用户界面

 + +如果你不擅长代码，我们还为VALL-E X创建了一个用户友好的图形界面。它可以让您轻松地与模型进行交互，使语音克隆和多语言语音合成变得轻而易举。 +
+使用以下命令启动用户界面： +```commandline +python -X utf8 launch-ui.py +``` +
+ +## 🛠️ 硬件要求及推理速度 + +VALL-E X 可以在CPU或GPU上运行 (`pytorch 2.0+`, CUDA 11.7 ~ CUDA 12.0). + +若使用GPU运行，你需要至少6GB的显存。 + +## ⚙️ Details + +VALL-E X 与 [Bark](https://github.com/suno-ai/bark), [VALL-E](https://arxiv.org/abs/2301.02111) and [AudioLM](https://arxiv.org/abs/2209.03143)类似, 使用GPT风格的模型以自回归方式预测量化音频token，并由[EnCodec](https://github.com/facebookresearch/encodec)解码. +
+与 [Bark](https://github.com/suno-ai/bark) 相比: +- ✔ **轻量**: 3️⃣ ✖ 更小, +- ✔ **快速**: 4️⃣ ✖ 更快, +- ✔ **中文&日文的更高质量** +- ✔ **跨语言合成时没有外国口音** +- ✔ **开放且易于操作的声音克隆** +- ❌ **支持的语言较少** +- ❌ **没有用于合成音乐及特殊音效的token** + +### 支持的语言 + +| 语言 | 状态 | +|---------| :---: | +| 英语 (en) | ✅ | +| 日语 (ja) | ✅ | +| 中文 (zh) | ✅ | + +## ❓ FAQ + +#### 在哪里可以下载checkpoint? +* 当您第一次运行程序时,我们使用`wget`将模型下载到`./checkpoints/`目录里。 +* 如果第一次运行时下载失败，请从[这里](https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt)手动下载模型，并将文件放在`./checkpoints/`里。 + +#### 需要多少显存? +* 6GB 显存(GPU VRAM) - 几乎所有NVIDIA GPU都满足要求. + +#### 为什么模型无法生成长文本? +* 当序列长度增加时，Transformer的计算复杂度呈二次方增长。因此，所有训练音频都保持在22秒以下。请确保音频提示（audio prompt）和生成的音频的总长度小于22秒以确保可接受的性能。 + +#### 更多... + +## 🧠 待办事项 +- [x] 添加中文 README +- [x] 长文本生成 +- [x] 用Vocos解码器替换Encodec解码器 +- [ ] 微调以实现更好的语音自适应 +- [ ] 给非python用户的`.bat`脚本 +- [ ] 更多... + +## 🙏 感谢 +- [VALL-E X paper](https://arxiv.org/pdf/2303.03926) for the brilliant idea +- [lifeiteng's vall-e](https://github.com/lifeiteng/vall-e) for related training code +- [bark](https://github.com/suno-ai/bark) for the amazing pioneering work in neuro-codec TTS model + +## ⭐️ 表示出你的支持 + +如果您觉得VALL-E X有趣且有用，请在GitHub上给我们一颗星！ ⭐️ 它鼓励我们不断改进模型并添加令人兴奋的功能。 + +## 📜 License + +VALL-E X 使用 [MIT License](./LICENSE). + +--- + +有问题或需要帮助？ 可以随便 [open an issue](https://github.com/Plachtaa/VALL-E-X/issues/new) 或加入我们的 [Discord](https://discord.gg/qCBRmAnTxg) + +Happy voice cloning! 🎤 \ No newline at end of file diff --git a/tts/VALL-E-X/README.md b/tts/VALL-E-X/README.md new file mode 100644 index 0000000..6207659 --- /dev/null +++ b/tts/VALL-E-X/README.md @@ -0,0 +1,353 @@ +# VALL-E X: Multilingual Text-to-Speech Synthesis and Voice Cloning 🔊 +# This is for inference, will require some work to get it to inference with Vall-E repo later on. +[![Discord](https://img.shields.io/badge/Discord-%235865F2.svg?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/qCBRmAnTxg) +
+English | [中文](README-ZH.md) +
+An open source implementation of Microsoft's [VALL-E X](https://arxiv.org/pdf/2303.03926) zero-shot TTS model.
+**We release our trained model to the public for research or application usage.** + +![vallex-framework](/images/vallex_framework.jpg "VALL-E X framework") + +VALL-E X is an amazing multilingual text-to-speech (TTS) model proposed by Microsoft. While Microsoft initially publish in their research paper, they did not release any code or pretrained models. Recognizing the potential and value of this technology, our team took on the challenge to reproduce the results and train our own model. We are glad to share our trained VALL-E X model with the community, allowing everyone to experience the power next-generation TTS! 🎧 +
+
+More details about the model are presented in [model card](./model-card.md). + +## 📖 Quick Index +* [🚀 Updates](#-updates) +* [📢 Features](#-features) +* [💻 Installation](#-installation) +* [🎧 Demos](#-demos) +* [🐍 Usage](#-usage-in-python) +* [❓ FAQ](#-faq) +* [🧠 TODO](#-todo) + +## 🚀 Updates +**2023.09.10** +- Added AR decoder batch decoding for more stable generation result. + +**2023.08.30** +- Replaced EnCodec decoder with Vocos decoder, improved audio quality. (Thanks to [@v0xie](https://github.com/v0xie)) + +**2023.08.23** +- Added long text generation. + +**2023.08.20** +- Added [Chinese README](README-ZH.md). + +**2023.08.14** +- Pretrained VALL-E X checkpoint is now released. Download it [here](https://drive.google.com/file/d/10gdQWvP-K_e1undkvv0p2b7SU6I4Egyl/view?usp=sharing) + +## 💻 Installation +### Install with pip, recommended with Python 3.10, CUDA 11.7 ~ 12.0, PyTorch 2.0+ +```commandline +git clone https://github.com/Plachtaa/VALL-E-X.git +cd VALL-E-X +pip install -r requirements.txt +``` + +> Note: If you want to make prompt, you need to install ffmpeg and add its folder to the environment variable PATH. + +When you run the program for the first time, it will automatically download the corresponding model. + +If the download fails and reports an error, please follow the steps below to manually download the model. + +(Please pay attention to the capitalization of folders) + +1. Check whether there is a `checkpoints` folder in the installation directory. +If not, manually create a `checkpoints` folder (`./checkpoints/`) in the installation directory. + +2. Check whether there is a `vallex-checkpoint.pt` file in the `checkpoints` folder. +If not, please manually download the `vallex-checkpoint.pt` file from [here](https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt) and put it in the `checkpoints` folder. + +3. Check whether there is a `whisper` folder in the installation directory. +If not, manually create a `whisper` folder (`./whisper/`) in the installation directory. + +4. Check whether there is a `medium.pt` file in the `whisper` folder. +If not, please manually download the `medium.pt` file from [here](https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt) and put it in the `whisper` folder. + +## 🎧 Demos +Not ready to set up the environment on your local machine just yet? No problem! We've got you covered with our online demos. You can try out VALL-E X directly on Hugging Face or Google Colab, experiencing the model's capabilities hassle-free! +
+[![Open in Spaces](https://img.shields.io/badge/🤗-Open%20in%20Spaces-blue.svg)](https://huggingface.co/spaces/Plachta/VALL-E-X) +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/drive/1yyD_sz531QntLKowMHo-XxorsFBCfKul?usp=sharing) + + +## 📢 Features + +VALL-E X comes packed with cutting-edge functionalities: + +1. **Multilingual TTS**: Speak in three languages - English, Chinese, and Japanese - with natural and expressive speech synthesis. + +2. **Zero-shot Voice Cloning**: Enroll a short 3~10 seconds recording of an unseen speaker, and watch VALL-E X create personalized, high-quality speech that sounds just like them! + +
+
see example
 + +[prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/a7baa51d-a53a-41cc-a03d-6970f25fcca7) + + +[output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/b895601a-d126-4138-beff-061aabdc7985) + +
+ +3. **Speech Emotion Control**: Experience the power of emotions! VALL-E X can synthesize speech with the same emotion as the acoustic prompt provided, adding an extra layer of expressiveness to your audio. + +
+
see example
 + +https://github.com/Plachtaa/VALL-E-X/assets/112609742/56fa9988-925e-4757-82c5-83ecb0df6266 + + +https://github.com/Plachtaa/VALL-E-X/assets/112609742/699c47a3-d502-4801-8364-bd89bcc0b8f1 + +
+ +4. **Zero-shot Cross-Lingual Speech Synthesis**: Take monolingual speakers on a linguistic journey! VALL-E X can produce personalized speech in another language without compromising on fluency or accent. Below is a Japanese speaker talk in Chinese & English. 🇯🇵 🗣 + +
+
see example
 + +[jp-prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/ea6e2ee4-139a-41b4-837e-0bd04dda6e19) + + +[en-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/db8f9782-923f-425e-ba94-e8c1bd48f207) + + +[zh-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/15829d79-e448-44d3-8965-fafa7a3f8c28) + +
+ +5. **Accent Control**: Get creative with accents! VALL-E X allows you to experiment with different accents, like speaking Chinese with an English accent or vice versa. 🇨🇳 💬 + +
+
see example
 + +[en-prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/f688d7f6-70ef-46ec-b1cc-355c31e78b3b) + + +[zh-accent-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/be59c7ca-b45b-44ca-a30d-4d800c950ccc) + + +[en-accent-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/8b4f4f9b-f299-4ea4-a548-137437b71738) + +
+ +6. **Acoustic Environment Maintenance**: No need for perfectly clean audio prompts! VALL-E X adapts to the acoustic environment of the input, making speech generation feel natural and immersive. + +
+
see example
 + +[noise-prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/68986d88-abd0-4d1d-96e4-4f893eb9259e) + + +[noise-output.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/96c4c612-4516-4683-8804-501b70938608) + +
+ + +Explore our [demo page](https://plachtaa.github.io/) for a lot more examples! + +## 🐍 Usage in Python + +
+

🪑 Basics

 + +```python +from utils.generation import SAMPLE_RATE, generate_audio, preload_models +from scipy.io.wavfile import write as write_wav +from IPython.display import Audio + +# download and load all models +preload_models() + +# generate audio from text +text_prompt = """ +Hello, my name is Nose. And uh, and I like hamburger. Hahaha... But I also have other interests such as playing tactic toast. +""" +audio_array = generate_audio(text_prompt) + +# save audio to disk +write_wav("vallex_generation.wav", SAMPLE_RATE, audio_array) + +# play text in notebook +Audio(audio_array, rate=SAMPLE_RATE) +``` + +[hamburger.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/578d7bbe-cda9-483e-898c-29646edc8f2e) + +
+ +
+

🌎 Foreign Language

 +
+This VALL-E X implementation also supports Chinese and Japanese. All three languages have equally awesome performance! +
+ +```python + +text_prompt = """ + チュソクは私のお気に入りの祭りです。 私は数日間休んで、友人や家族との時間を過ごすことができます。 +""" +audio_array = generate_audio(text_prompt) +``` + +[vallex_japanese.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/ee57a688-3e83-4be5-b0fe-019d16eec51c) + +*Note: VALL-E X controls accent perfectly even when synthesizing code-switch text. However, you need to manually denote language of respective sentences (since our g2p tool is rule-base)* +```python +text_prompt = """ + [EN]The Thirty Years' War was a devastating conflict that had a profound impact on Europe.[EN] + [ZH]这是历史的开始。 如果您想听更多，请继续。[ZH] +""" +audio_array = generate_audio(text_prompt, language='mix') +``` + +[vallex_codeswitch.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/d8667abf-bd08-499f-a383-a861d852f98a) + +
+ +
+

📼 Voice Presets

 + +VALL-E X provides tens of speaker voices which you can directly used for inference! Browse all voices in the [code](/presets) + +> VALL-E X tries to match the tone, pitch, emotion and prosody of a given preset. The model also attempts to preserve music, ambient noise, etc. + +```python +text_prompt = """ +I am an innocent boy with a smoky voice. It is a great honor for me to speak at the United Nations today. +""" +audio_array = generate_audio(text_prompt, prompt="dingzhen") +``` + +[smoky.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/d3f55732-b1cd-420f-87d6-eab60db14dc5) + +
+ +
+

🎙Voice Cloning

 + +VALL-E X supports voice cloning! You can make a voice prompt with any person, character or even your own voice, and use it like other voice presets.
+To make a voice prompt, you need to provide a speech of 3~10 seconds long, as well as the transcript of the speech. +You can also leave the transcript blank to let the [Whisper](https://github.com/openai/whisper) model to generate the transcript. +> VALL-E X tries to match the tone, pitch, emotion and prosody of a given prompt. The model also attempts to preserve music, ambient noise, etc. + +```python +from utils.prompt_making import make_prompt + +### Use given transcript +make_prompt(name="paimon", audio_prompt_path="paimon_prompt.wav", + transcript="Just, what was that? Paimon thought we were gonna get eaten.") + +### Alternatively, use whisper +make_prompt(name="paimon", audio_prompt_path="paimon_prompt.wav") +``` +Now let's try out the prompt we've just made! +```python +from utils.generation import SAMPLE_RATE, generate_audio, preload_models +from scipy.io.wavfile import write as write_wav + +# download and load all models +preload_models() + +text_prompt = """ +Hey, Traveler, Listen to this, This machine has taken my voice, and now it can talk just like me! +""" +audio_array = generate_audio(text_prompt, prompt="paimon") + +write_wav("paimon_cloned.wav", SAMPLE_RATE, audio_array) + +``` + +[paimon_prompt.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/e7922859-9d12-4e2a-8651-e156e4280311) + + +[paimon_cloned.webm](https://github.com/Plachtaa/VALL-E-X/assets/112609742/60d3b7e9-5ead-4024-b499-a897ce5f3d5e) + + +
+ + +
+

🎢User Interface

 + +Not comfortable with codes? No problem! We've also created a user-friendly graphical interface for VALL-E X. It allows you to interact with the model effortlessly, making voice cloning and multilingual speech synthesis a breeze. +
+You can launch the UI by the following command: +```commandline +python -X utf8 launch-ui.py +``` +
+ +## 🛠️ Hardware and Inference Speed + +VALL-E X works well on both CPU and GPU (`pytorch 2.0+`, CUDA 11.7 and CUDA 12.0). + +A GPU VRAM of 6GB is enough for running VALL-E X without offloading. + +## ⚙️ Details + +VALL-E X is similar to [Bark](https://github.com/suno-ai/bark), [VALL-E](https://arxiv.org/abs/2301.02111) and [AudioLM](https://arxiv.org/abs/2209.03143), which generates audio in GPT-style by predicting audio tokens quantized by [EnCodec](https://github.com/facebookresearch/encodec). +
+Comparing to [Bark](https://github.com/suno-ai/bark): +- ✔ **Light-weighted**: 3️⃣ ✖ smaller, +- ✔ **Efficient**: 4️⃣ ✖ faster, +- ✔ **Better quality on Chinese & Japanese** +- ✔ **Cross-lingual speech without foreign accent** +- ✔ **Easy voice-cloning** +- ❌ **Less languages** +- ❌ **No special tokens for music / sound effects** + +### Supported Languages + +| Language | Status | +| --- | :---: | +| English (en) | ✅ | +| Japanese (ja) | ✅ | +| Chinese, simplified (zh) | ✅ | + +## ❓ FAQ + +#### Where can I download the model checkpoint? +* We use `wget` to download the model to directory `./checkpoints/` when you run the program for the first time. +* If the download fails on the first run, please manually download from [this link](https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt), and put the file under directory `./checkpoints/`. + +#### How much VRAM do I need? +* 6GB GPU VRAM - Almost all NVIDIA GPUs satisfy the requirement. + +#### Why the model fails to generate long text? +* Transformer's computation complexity increases quadratically while the sequence length increases. Hence, all training +are kept under 22 seconds. Please make sure the total length of audio prompt and generated audio is less than 22 seconds +to ensure acceptable performance. + + +#### MORE TO BE ADDED... + +## 🧠 TODO +- [x] Add Chinese README +- [x] Long text generation +- [x] Replace Encodec decoder with Vocos decoder +- [ ] Fine-tuning for better voice adaptation +- [ ] `.bat` scripts for non-python users +- [ ] To be added... + +## 🙏 Appreciation +- [VALL-E X paper](https://arxiv.org/pdf/2303.03926) for the brilliant idea +- [lifeiteng's vall-e](https://github.com/lifeiteng/vall-e) for related training code +- [bark](https://github.com/suno-ai/bark) for the amazing pioneering work in neuro-codec TTS model + +## ⭐️ Show Your Support + +If you find VALL-E X interesting and useful, give us a star on GitHub! ⭐️ It encourages us to keep improving the model and adding exciting features. + +## 📜 License + +VALL-E X is licensed under the [MIT License](./LICENSE). + +--- + +Have questions or need assistance? Feel free to [open an issue](https://github.com/Plachtaa/VALL-E-X/issues/new) or join our [Discord](https://discord.gg/qCBRmAnTxg) + +Happy voice cloning! 🎤 diff --git a/tts/VALL-E-X/customs/ph.txt b/tts/VALL-E-X/customs/ph.txt new file mode 100644 index 0000000..e69de29 diff --git a/tts/VALL-E-X/data/__init__.py b/tts/VALL-E-X/data/__init__.py new file mode 100644 index 0000000..10b382e --- /dev/null +++ b/tts/VALL-E-X/data/__init__.py @@ -0,0 +1,3 @@ +# from .datamodule import * +# from .tokenizer import * +from .collation import * diff --git a/tts/VALL-E-X/data/collation.py b/tts/VALL-E-X/data/collation.py new file mode 100644 index 0000000..921850c --- /dev/null +++ b/tts/VALL-E-X/data/collation.py @@ -0,0 +1,120 @@ +from pathlib import Path +from typing import List, Tuple + +import numpy as np +import torch + +from utils import SymbolTable + + +class TextTokenCollater: + """Collate list of text tokens + + Map sentences to integers. Sentences are padded to equal length. + Beginning and end-of-sequence symbols can be added. + + Example: + >>> token_collater = TextTokenCollater(text_tokens) + >>> tokens_batch, tokens_lens = token_collater(text) + + Returns: + tokens_batch: IntTensor of shape (B, L) + B: batch dimension, number of input sentences + L: length of the longest sentence + tokens_lens: IntTensor of shape (B,) + Length of each sentence after adding and + but before padding. + """ + + def __init__( + self, + text_tokens: List[str], + add_eos: bool = True, + add_bos: bool = True, + pad_symbol: str = "", + bos_symbol: str = "", + eos_symbol: str = "", + ): + self.pad_symbol = pad_symbol + + self.add_eos = add_eos + self.add_bos = add_bos + + self.bos_symbol = bos_symbol + self.eos_symbol = eos_symbol + + unique_tokens = ( + [pad_symbol] + + ([bos_symbol] if add_bos else []) + + ([eos_symbol] if add_eos else []) + + sorted(text_tokens) + ) + + self.token2idx = {token: idx for idx, token in enumerate(unique_tokens)} + self.idx2token = [token for token in unique_tokens] + + def index( + self, tokens_list: List[str] + ) -> Tuple[torch.Tensor, torch.Tensor]: + seqs, seq_lens = [], [] + for tokens in tokens_list: + assert ( + all([True if s in self.token2idx else False for s in tokens]) + is True + ) + seq = ( + ([self.bos_symbol] if self.add_bos else []) + + list(tokens) + + ([self.eos_symbol] if self.add_eos else []) + ) + seqs.append(seq) + seq_lens.append(len(seq)) + + max_len = max(seq_lens) + for k, (seq, seq_len) in enumerate(zip(seqs, seq_lens)): + seq.extend([self.pad_symbol] * (max_len - seq_len)) + + tokens = torch.from_numpy( + np.array( + [[self.token2idx[token] for token in seq] for seq in seqs], + dtype=np.int64, + ) + ) + tokens_lens = torch.IntTensor(seq_lens) + + return tokens, tokens_lens + + def __call__(self, texts: List[str]) -> Tuple[torch.Tensor, torch.Tensor]: + tokens_seqs = [[p for p in text] for text in texts] + max_len = len(max(tokens_seqs, key=len)) + + seqs = [ + ([self.bos_symbol] if self.add_bos else []) + + list(seq) + + ([self.eos_symbol] if self.add_eos else []) + + [self.pad_symbol] * (max_len - len(seq)) + for seq in tokens_seqs + ] + + tokens_batch = torch.from_numpy( + np.array( + [seq for seq in seqs], + dtype=np.int64, + ) + ) + + tokens_lens = torch.IntTensor( + [ + len(seq) + int(self.add_eos) + int(self.add_bos) + for seq in tokens_seqs + ] + ) + + return tokens_batch, tokens_lens + + +def get_text_token_collater() -> TextTokenCollater: + collater = TextTokenCollater( + ['0'], add_bos=False, add_eos=False + ) + return collater diff --git a/tts/VALL-E-X/data/datamodule.py b/tts/VALL-E-X/data/datamodule.py new file mode 100644 index 0000000..9df73ad --- /dev/null +++ b/tts/VALL-E-X/data/datamodule.py @@ -0,0 +1,419 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import argparse +import inspect +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any, Dict, Optional + +import torch +# from icefall.utils import str2bool +# from lhotse import CutSet, load_manifest_lazy +# from lhotse.dataset import ( +# CutConcatenate, +# DynamicBucketingSampler, +# PrecomputedFeatures, +# SingleCutSampler, +# SpecAugment, +# ) +# from lhotse.dataset.input_strategies import OnTheFlyFeatures +# from lhotse.utils import fix_random_seed +from torch.utils.data import DataLoader + +from data.collation import get_text_token_collater +# from data.dataset import SpeechSynthesisDataset +from data.fbank import get_fbank_extractor +from data.input_strategies import PromptedPrecomputedFeatures + +# PrecomputedFeatures = PrecomputedFeatures + + +class _SeedWorkers: + def __init__(self, seed: int): + self.seed = seed + + def __call__(self, worker_id: int): + fix_random_seed(self.seed + worker_id) + + +def _get_input_strategy(input_strategy, dataset, cuts): + if input_strategy == "PromptedPrecomputedFeatures": + return PromptedPrecomputedFeatures(dataset, cuts) + + return eval(input_strategy)() + + +class TtsDataModule: + """ + DataModule for VALL-E TTS experiments. + It assumes there is always one train and valid dataloader. + + It contains all the common data pipeline modules used in TTS + experiments, e.g.: + - dynamic batch size, + - bucketing samplers, + - cut concatenation[not used & tested yet], + - augmentation[not used & tested yet], + - on-the-fly feature extraction[not used & tested yet] + + This class should be derived for specific corpora used in TTS tasks. + """ + + def __init__(self, args: argparse.Namespace): + self.args = args + + @classmethod + def add_arguments(cls, parser: argparse.ArgumentParser): + group = parser.add_argument_group( + title="TTS data related options", + description="These options are used for the preparation of " + "PyTorch DataLoaders from Lhotse CutSet's -- they control the " + "effective batch sizes, sampling strategies, applied data " + "augmentations, etc.", + ) + group.add_argument( + "--manifest-dir", + type=Path, + default=Path("data/tokenized"), + help="Path to directory with train/valid/test cuts.", + ) + group.add_argument( + "--max-duration", + type=int, + default=40.0, + help="Maximum pooled recordings duration (seconds) in a " + "single batch. You can reduce it if it causes CUDA OOM.", + ) + group.add_argument( + "--bucketing-sampler", + type=str2bool, + default=True, + help="When enabled, the batches will come from buckets of " + "similar duration (saves padding frames).", + ) + group.add_argument( + "--num-buckets", + type=int, + default=10, + help="The number of buckets for the DynamicBucketingSampler" + "(you might want to increase it for larger datasets).", + ) + group.add_argument( + "--concatenate-cuts", + type=str2bool, + default=False, + help="When enabled, utterances (cuts) will be concatenated " + "to minimize the amount of padding.", + ) + group.add_argument( + "--duration-factor", + type=float, + default=1.0, + help="Determines the maximum duration of a concatenated cut " + "relative to the duration of the longest cut in a batch.", + ) + group.add_argument( + "--gap", + type=float, + default=0.1, + help="The amount of padding (in seconds) inserted between " + "concatenated cuts. This padding is filled with noise when " + "noise augmentation is used.", + ) + group.add_argument( + "--on-the-fly-feats", + type=str2bool, + default=False, + help="When enabled, use on-the-fly cut mixing and feature " + "extraction. Will drop existing precomputed feature manifests " + "if available.", + ) + group.add_argument( + "--shuffle", + type=str2bool, + default=True, + help="When enabled (=default), the examples will be " + "shuffled for each epoch.", + ) + group.add_argument( + "--drop-last", + type=str2bool, + default=False, + help="Whether to drop last batch. Used by sampler.", + ) + group.add_argument( + "--return-cuts", + type=str2bool, + default=True, + help="When enabled, each batch will have the " + "field: batch['supervisions']['cut'] with the cuts that " + "were used to construct it.", + ) + + group.add_argument( + "--num-workers", + type=int, + default=8, + help="The number of training dataloader workers that " + "collect the batches.", + ) + + group.add_argument( + "--enable-spec-aug", + type=str2bool, + default=False, + help="When enabled, use SpecAugment for training dataset.", + ) + + group.add_argument( + "--spec-aug-time-warp-factor", + type=int, + default=80, + help="Used only when --enable-spec-aug is True. " + "It specifies the factor for time warping in SpecAugment. " + "Larger values mean more warping. " + "A value less than 1 means to disable time warp.", + ) + + group.add_argument( + "--input-strategy", + type=str, + default="PrecomputedFeatures", + help="AudioSamples or PrecomputedFeatures or PromptedPrecomputedFeatures", + ) + + group.add_argument( + "--dataset", + type=str, + default="ljspeech", + help="--input-strategy PromptedPrecomputedFeatures needs dataset name to prepare prompts.", + ) + + parser.add_argument( + "--text-tokens", + type=str, + default="data/tokenized/unique_text_tokens.k2symbols", + help="Path to the unique text tokens file", + ) + + parser.add_argument( + "--sampling-rate", + type=int, + default=24000, + help="""Audio sampling rate.""", + ) + + def train_dataloaders( + self, + cuts_train: CutSet, + sampler_state_dict: Optional[Dict[str, Any]] = None, + ) -> DataLoader: + """ + Args: + cuts_train: + CutSet for training. + sampler_state_dict: + The state dict for the training sampler. + """ + transforms = [] + + if self.args.concatenate_cuts: + logging.info( + f"Using cut concatenation with duration factor " + f"{self.args.duration_factor} and gap {self.args.gap}." + ) + # Cut concatenation should be the first transform in the list, + # so that if we e.g. mix noise in, it will fill the gaps between + # different utterances. + transforms = [ + CutConcatenate( + duration_factor=self.args.duration_factor, gap=self.args.gap + ) + ] + transforms + + input_transforms = [] + if self.args.enable_spec_aug: + logging.info("Enable SpecAugment") + logging.info( + f"Time warp factor: {self.args.spec_aug_time_warp_factor}" + ) + # Set the value of num_frame_masks according to Lhotse's version. + # In different Lhotse's versions, the default of num_frame_masks is + # different. + num_frame_masks = 10 + num_frame_masks_parameter = inspect.signature( + SpecAugment.__init__ + ).parameters["num_frame_masks"] + if num_frame_masks_parameter.default == 1: + num_frame_masks = 2 + logging.info(f"Num frame mask: {num_frame_masks}") + input_transforms.append( + SpecAugment( + time_warp_factor=self.args.spec_aug_time_warp_factor, + num_frame_masks=num_frame_masks, + features_mask_size=27, + num_feature_masks=2, + frames_mask_size=100, + ) + ) + else: + logging.info("Disable SpecAugment") + + logging.info("About to create train dataset") + if self.args.on_the_fly_feats: + # NOTE: the PerturbSpeed transform should be added only if we + # remove it from data prep stage. + # Add on-the-fly speed perturbation; since originally it would + # have increased epoch size by 3, we will apply prob 2/3 and use + # 3x more epochs. + # Speed perturbation probably should come first before + # concatenation, but in principle the transforms order doesn't have + # to be strict (e.g. could be randomized) + # transforms = [PerturbSpeed(factors=[0.9, 1.1], p=2/3)] + transforms # noqa + # Drop feats to be on the safe side. + train = SpeechSynthesisDataset( + get_text_token_collater(self.args.text_tokens), + cut_transforms=transforms, + feature_input_strategy=OnTheFlyFeatures(get_fbank_extractor()), + feature_transforms=input_transforms, + ) + else: + train = SpeechSynthesisDataset( + get_text_token_collater(self.args.text_tokens), + feature_input_strategy=_get_input_strategy( + self.args.input_strategy, self.args.dataset, cuts_train + ), + cut_transforms=transforms, + feature_transforms=input_transforms, + ) + + if self.args.bucketing_sampler: + logging.info("Using DynamicBucketingSampler") + train_sampler = DynamicBucketingSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + num_buckets=self.args.num_buckets, + drop_last=self.args.drop_last, + ) + else: + logging.info( + "Using SingleCutSampler and sort by duraton(ascending=True)." + ) + cuts_train = cuts_train.to_eager().sort_by_duration(ascending=True) + train_sampler = SingleCutSampler( + cuts_train, + max_duration=self.args.max_duration, + shuffle=self.args.shuffle, + ) + logging.info("About to create train dataloader") + + if sampler_state_dict is not None: + logging.info("Loading sampler state dict") + train_sampler.load_state_dict(sampler_state_dict) + + # 'seed' is derived from the current random state, which will have + # previously been set in the main process. + seed = torch.randint(0, 100000, ()).item() + worker_init_fn = _SeedWorkers(seed) + + train_dl = DataLoader( + train, + sampler=train_sampler, + batch_size=None, + num_workers=self.args.num_workers, + persistent_workers=False, + worker_init_fn=worker_init_fn, + ) + + return train_dl + + def valid_dataloaders(self, cuts_valid: CutSet) -> DataLoader: + logging.info("About to create dev dataset") + if self.args.on_the_fly_feats: + validate = SpeechSynthesisDataset( + get_text_token_collater(self.args.text_tokens), + feature_input_strategy=OnTheFlyFeatures(get_fbank_extractor()), + cut_transforms=[], + ) + else: + validate = SpeechSynthesisDataset( + get_text_token_collater(self.args.text_tokens), + feature_input_strategy=_get_input_strategy( + self.args.input_strategy, self.args.dataset, cuts_valid + ), + cut_transforms=[], + ) + valid_sampler = DynamicBucketingSampler( + cuts_valid, + max_duration=self.args.max_duration, + shuffle=False, + ) + logging.info("About to create dev dataloader") + valid_dl = DataLoader( + validate, + sampler=valid_sampler, + batch_size=None, + num_workers=4, + persistent_workers=False, + ) + + return valid_dl + + def test_dataloaders(self, cuts: CutSet) -> DataLoader: + logging.debug("About to create test dataset") + test = SpeechSynthesisDataset( + get_text_token_collater(self.args.text_tokens), + feature_input_strategy=OnTheFlyFeatures(get_fbank_extractor()) + if self.args.on_the_fly_feats + else _get_input_strategy( + self.args.input_strategy, self.args.dataset, cuts + ), + cut_transforms=[], + ) + sampler = DynamicBucketingSampler( + cuts, + max_duration=self.args.max_duration, + shuffle=False, + ) + logging.debug("About to create test dataloader") + test_dl = DataLoader( + test, + batch_size=None, + sampler=sampler, + num_workers=self.args.num_workers, + ) + return test_dl + + @lru_cache() + def train_cuts(self) -> CutSet: + logging.info("About to get train cuts") + return load_manifest_lazy( + self.args.manifest_dir / "cuts_train.jsonl.gz" + ) + + @lru_cache() + def dev_cuts(self) -> CutSet: + logging.info("About to get dev cuts") + return load_manifest_lazy(self.args.manifest_dir / "cuts_dev.jsonl.gz") + + @lru_cache() + def test_cuts(self) -> CutSet: + logging.info("About to get test cuts") + return load_manifest_lazy(self.args.manifest_dir / "cuts_test.jsonl.gz") diff --git a/tts/VALL-E-X/data/dataset.py b/tts/VALL-E-X/data/dataset.py new file mode 100644 index 0000000..196ba1d --- /dev/null +++ b/tts/VALL-E-X/data/dataset.py @@ -0,0 +1,242 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +modified from lhoste.dataset.speech_synthesis.py +""" + +import torch +import math +import h5py +from tokenizers import Tokenizer +from typing import Union, List +import numpy as np +from tqdm import tqdm + +_pad = '_' +_punctuation = ',.!?-~…' +_letters = 'NQabdefghijklmnopstuvwxyzɑæʃʑçɯɪɔɛɹðəɫɥɸʊɾʒθβŋɦ⁼ʰ`^#*=ˈˌ→↓↑ ' +symbols = [_pad] + list(_punctuation) + list(_letters) + +language_dict = { + 'en': 0, + 'zh': 1, + 'ja': 2, +} +def seq2phone(tokens: Union[List, np.ndarray]): + """ + Convert tokenized phoneme ID sequence back to phoneme string + :param tokens: phoneme tokens + :return: recovered phoneme sequence + """ + phones = "".join([symbols[i] for i in tokens]) + return phones + +class DynamicBatchSampler(torch.utils.data.Sampler): + def __init__(self, sampler, num_tokens_fn, num_buckets=100, min_size=0, max_size=1000, + max_tokens=None, max_sentences=None, drop_last=False): + """ + + :param sampler: + :param num_tokens_fn: 根据idx返回样本的长度的函数 + :param num_buckets: 利用桶原理将相似长度的样本放在一个batchsize中，桶的数量 + :param min_size: 最小长度的样本， 小于这个值的样本会被过滤掉。 依据这个值来创建样桶 + :param max_size: 最大长度的样本 + :param max_sentences: batch_size, 但是这里可以通过max_sentences 和 max_tokens 共同控制最终的大小 + """ + super(DynamicBatchSampler, self).__init__(sampler) + self.sampler = sampler + self.num_tokens_fn = num_tokens_fn + self.num_buckets = num_buckets + + self.min_size = min_size + self.max_size = max_size + + assert max_size <= max_tokens, "max_size should be smaller than max tokens" + assert max_tokens is not None or max_sentences is not None, \ + "max_tokens and max_sentences should not be null at the same time, please specify one parameter at least" + self.max_tokens = max_tokens if max_tokens is not None else float('Inf') + self.max_sentences = max_sentences if max_sentences is not None else float('Inf') + self.drop_last = drop_last + + def set_epoch(self, epoch): + self.sampler.set_epoch(epoch) + def is_batch_full(self, num_tokens, batch): + if len(batch) == 0: + return False + if len(batch) == self.max_sentences: + return True + if num_tokens > self.max_tokens: + return True + return False + + def __iter__(self): + buckets = [[] for _ in range(self.num_buckets)] + sample_len = [0] * self.num_buckets + + for idx in self.sampler: + idx_length = self.num_tokens_fn(idx) + if not (self.min_size <= idx_length <= self.max_size): + print("sentence at index {} of size {} exceeds max_tokens, the sentence is ignored".format(idx, idx_length)) + continue + + index_buckets = math.floor((idx_length - self.min_size) / (self.max_size - self.min_size + 1) + * self.num_buckets) + sample_len[index_buckets] = max(sample_len[index_buckets], idx_length) + + num_tokens = (len(buckets[index_buckets]) + 1) * sample_len[index_buckets] + if self.is_batch_full(num_tokens, buckets[index_buckets]): + # yield this batch + yield buckets[index_buckets] + buckets[index_buckets] = [] + sample_len[index_buckets] = 0 + + buckets[index_buckets].append(idx) + + # process left-over + leftover_batch = [] + leftover_sample_len = 0 + leftover = [idx for bucket in buckets for idx in bucket] + for idx in leftover: + idx_length = self.num_tokens_fn(idx) + leftover_sample_len = max(leftover_sample_len, idx_length) + num_tokens = (len(leftover_batch) + 1) * leftover_sample_len + if self.is_batch_full(num_tokens, leftover_batch): + yield leftover_batch + leftover_batch = [] + leftover_sample_len = 0 + leftover_batch.append(idx) + + if len(leftover_batch) > 0 and not self.drop_last: + yield leftover_batch + + def __len__(self): + # we do not know the exactly batch size, so do not call len(dataloader) + pass + + +class AudioDataset(torch.utils.data.Dataset): + def __init__(self, h5_path, ann_path, tokenizer_path): + self.h5_path = h5_path + with open(ann_path, 'r', encoding='utf-8') as f: + lines = f.readlines() + ls = [l.split("|") for l in lines] + ls_T = list(zip(*ls)) + del ls_T[-1] + self.h5_paths, self.durations, self.langs, self.texts = \ + list(ls_T[0]), list(ls_T[1]), list(ls_T[2]), list(ls_T[3]) + self.durations = [float(dur) for dur in self.durations] + self.tokenizer = Tokenizer.from_file(tokenizer_path) + + self._archive = None + + def __len__(self): + return len(self.h5_paths) + + def get_dur(self, idx): + return self.durations[idx] + + @property + def archive(self): + if self._archive is None: # lazy loading here! + self._archive = h5py.File(self.h5_path, "r") + return self._archive + def __getitem__(self, idx): + archive = self.archive + h5_path = self.h5_paths[idx] + sub = archive[h5_path] + audio_tokens = sub['audio'][()] + phone_tokens = sub['text'][()] + dur = self.durations[idx] + lang = self.langs[idx] + text = self.texts[idx] + # tokenization should be done within dataloader + phones = seq2phone(phone_tokens) + phones = phones.replace(" ", "_") + if not len(phones): + cptpho_tokens = self.tokenizer.encode(text).ids + else: + cptpho_tokens = self.tokenizer.encode(phones).ids + assert len(cptpho_tokens) + return { + 'utt_id': h5_path, + 'text': text, + 'audio': None, + 'audio_lens': None, + 'audio_features': audio_tokens, + 'audio_features_lens': len(audio_tokens.T), + 'text_tokens': np.array(cptpho_tokens), + 'text_tokens_lens': len(cptpho_tokens), + 'language': language_dict[lang], + } + +def collate(batch): + utt_id_s = [b['utt_id'] for b in batch] + text_s = [b['text'] for b in batch] + + audio_s = [b['audio'] for b in batch] + audio_lens_s = [b['audio_lens'] for b in batch] + + audio_features_lens_s = [b['audio_features_lens'] for b in batch] + # create an empty tensor with maximum audio feature length + audio_features_s = torch.zeros([len(batch), max(audio_features_lens_s), 8], dtype=torch.int64) - 1 # audio pad with -1 + + text_tokens_lens_s = [b['text_tokens_lens'] for b in batch] + # create an empty tensor with maximum text tokens length + text_tokens_s = torch.zeros([len(batch), max(text_tokens_lens_s)], dtype=torch.int64) + 3 # [PAD] token id 3 + + language_s = [b['language'] for b in batch] + + for i, b in enumerate(batch): + audio_features = b['audio_features'] + audio_features_lens = b['audio_features_lens'] + audio_features_s[i, :audio_features_lens, :] = torch.LongTensor(audio_features.T) + + text_tokens = b['text_tokens'] + text_tokens_lens = b['text_tokens_lens'] + text_tokens_s[i, :text_tokens_lens] = torch.LongTensor(text_tokens) + + batch = { + 'utt_id': utt_id_s, + 'text': text_s, + 'audio': audio_s, + 'audio_lens': audio_lens_s, + 'audio_features': audio_features_s, + 'audio_features_lens': torch.LongTensor(np.array(audio_features_lens_s)), + 'text_tokens': text_tokens_s, + 'text_tokens_lens': torch.LongTensor(np.array(text_tokens_lens_s)), + 'languages': torch.LongTensor(np.array(language_s)), + } + return batch + +def create_dataloader(data_dir="/root/valle/egs/mix", n_gpus=1, rank=0, num_workers=0, num_buckets=10, max_duration=120): + train_dataset = AudioDataset(h5_path=f"{data_dir}/audio_sum.hdf5", + ann_path=f"{data_dir}/audio_ann_sum.txt", + tokenizer_path=f"{data_dir}/bpe_69.json") + ran_sampler = torch.utils.data.distributed.DistributedSampler( + train_dataset, + num_replicas=n_gpus, + rank=rank, + shuffle=True, + ) + dynamic_sampler = DynamicBatchSampler(ran_sampler, train_dataset.get_dur, num_buckets=num_buckets, max_size=20, + max_tokens=max_duration) + + + train_loader = torch.utils.data.DataLoader(train_dataset, num_workers=num_workers, collate_fn=collate, + batch_sampler=dynamic_sampler) + + return train_loader diff --git a/tts/VALL-E-X/data/fbank.py b/tts/VALL-E-X/data/fbank.py new file mode 100644 index 0000000..4c302ba --- /dev/null +++ b/tts/VALL-E-X/data/fbank.py @@ -0,0 +1,212 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from dataclasses import asdict, dataclass +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +# from lhotse.features.base import FeatureExtractor +# from lhotse.utils import EPSILON, Seconds, compute_num_frames +from librosa.filters import mel as librosa_mel_fn + + +@dataclass +class BigVGANFbankConfig: + # Spectogram-related part + # Note that frame_length and frame_shift will be converted to milliseconds before torchaudio/Kaldi sees them + frame_length: Seconds = 1024 / 24000.0 + frame_shift: Seconds = 256 / 24000.0 + remove_dc_offset: bool = True + round_to_power_of_two: bool = True + + # Fbank-related part + low_freq: float = 0.0 + high_freq: float = 12000.0 + num_mel_bins: int = 100 + use_energy: bool = False + + def to_dict(self) -> Dict[str, Any]: + return asdict(self) + + @staticmethod + def from_dict(data: Dict[str, Any]) -> "BigVGANFbankConfig": + return BigVGANFbankConfig(**data) + + +def dynamic_range_compression_torch(x, C=1, clip_val=1e-5): + return torch.log(torch.clamp(x, min=clip_val) * C) + + +def spectral_normalize_torch(magnitudes): + output = dynamic_range_compression_torch(magnitudes) + return output + + +# https://github.com/NVIDIA/BigVGAN +# bigvgan_24khz_100band https://drive.google.com/drive/folders/1EpxX6AsxjCbbk0mmAhE0td6eYiABr8Oz +class BigVGANFbank(FeatureExtractor): + name = "fbank" + config_type = BigVGANFbankConfig + + def __init__(self, config: Optional[Any] = None): + super(BigVGANFbank, self).__init__(config) + sampling_rate = 24000 + self.mel_basis = torch.from_numpy( + librosa_mel_fn( + sampling_rate, + 1024, + self.config.num_mel_bins, + self.config.low_freq, + self.config.high_freq, + ).astype(np.float32) + ) + self.hann_window = torch.hann_window(1024) + + def _feature_fn(self, samples, **kwargs): + win_length, n_fft = 1024, 1024 + hop_size = 256 + if True: + sampling_rate = 24000 + duration = round(samples.shape[-1] / sampling_rate, ndigits=12) + expected_num_frames = compute_num_frames( + duration=duration, + frame_shift=self.frame_shift, + sampling_rate=sampling_rate, + ) + pad_size = ( + (expected_num_frames - 1) * hop_size + + win_length + - samples.shape[-1] + ) + assert pad_size >= 0 + + y = torch.nn.functional.pad( + samples, + (0, pad_size), + mode="constant", + ) + else: + y = torch.nn.functional.pad( + samples, + (int((n_fft - hop_size) / 2), int((n_fft - hop_size) / 2)), + mode="reflect", + ) + + y = y.squeeze(1) + + # complex tensor as default, then use view_as_real for future pytorch compatibility + spec = torch.stft( + y, + n_fft, + hop_length=hop_size, + win_length=win_length, + window=self.hann_window, + center=False, + pad_mode="reflect", + normalized=False, + onesided=True, + return_complex=True, + ) + spec = torch.view_as_real(spec) + spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)) + + spec = torch.matmul(self.mel_basis, spec) + spec = spectral_normalize_torch(spec) + + return spec.transpose(2, 1).squeeze(0) + + def extract( + self, samples: Union[np.ndarray, torch.Tensor], sampling_rate: int + ) -> np.ndarray: + assert sampling_rate == 24000 + params = asdict(self.config) + params.update({"sample_frequency": sampling_rate, "snip_edges": False}) + params["frame_shift"] *= 1000.0 + params["frame_length"] *= 1000.0 + if not isinstance(samples, torch.Tensor): + samples = torch.from_numpy(samples) + # Torchaudio Kaldi feature extractors expect the channel dimension to be first. + if len(samples.shape) == 1: + samples = samples.unsqueeze(0) + features = self._feature_fn(samples, **params).to(torch.float32) + return features.numpy() + + @property + def frame_shift(self) -> Seconds: + return self.config.frame_shift + + def feature_dim(self, sampling_rate: int) -> int: + return self.config.num_mel_bins + + @staticmethod + def mix( + features_a: np.ndarray, + features_b: np.ndarray, + energy_scaling_factor_b: float, + ) -> np.ndarray: + return np.log( + np.maximum( + # protection against log(0); max with EPSILON is adequate since these are energies (always >= 0) + EPSILON, + np.exp(features_a) + + energy_scaling_factor_b * np.exp(features_b), + ) + ) + + @staticmethod + def compute_energy(features: np.ndarray) -> float: + return float(np.sum(np.exp(features))) + + +def get_fbank_extractor() -> BigVGANFbank: + return BigVGANFbank(BigVGANFbankConfig()) + + +if __name__ == "__main__": + extractor = BigVGANFbank(BigVGANFbankConfig()) + + samples = torch.from_numpy(np.random.random([1000]).astype(np.float32)) + samples = torch.clip(samples, -1.0, 1.0) + fbank = extractor.extract(samples, 24000.0) + print(f"fbank {fbank.shape}") + + from scipy.io.wavfile import read + + MAX_WAV_VALUE = 32768.0 + + sampling_rate, samples = read( + "egs/libritts/prompts/5639_40744_000000_000002.wav" + ) + print(f"samples: [{samples.min()}, {samples.max()}]") + fbank = extractor.extract(samples.astype(np.float32) / MAX_WAV_VALUE, 24000) + print(f"fbank {fbank.shape}") + + import matplotlib.pyplot as plt + + _ = plt.figure(figsize=(18, 10)) + plt.imshow( + X=fbank.transpose(1, 0), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + ) + plt.gca().invert_yaxis() + plt.savefig("egs/libritts/prompts/5639_40744_000000_000002.png") + plt.close() + + print("fbank test PASS!") diff --git a/tts/VALL-E-X/data/input_strategies.py b/tts/VALL-E-X/data/input_strategies.py new file mode 100644 index 0000000..0b576e1 --- /dev/null +++ b/tts/VALL-E-X/data/input_strategies.py @@ -0,0 +1,159 @@ +import random +from collections import defaultdict +from concurrent.futures import ThreadPoolExecutor +from typing import Tuple, Type + +# from lhotse import CutSet +# from lhotse.dataset.collation import collate_features +# from lhotse.dataset.input_strategies import ( +# ExecutorType, +# PrecomputedFeatures, +# _get_executor, +# ) +# from lhotse.utils import fastcopy + + +class PromptedFeatures: + def __init__(self, prompts, features): + self.prompts = prompts + self.features = features + + def to(self, device): + return PromptedFeatures( + self.prompts.to(device), self.features.to(device) + ) + + def sum(self): + return self.features.sum() + + @property + def ndim(self): + return self.features.ndim + + @property + def data(self): + return (self.prompts, self.features) + + +# class PromptedPrecomputedFeatures(PrecomputedFeatures): +# """ +# :class:`InputStrategy` that reads pre-computed features, whose manifests +# are attached to cuts, from disk. +# +# It automatically pads the feature matrices with pre or post feature. +# +# .. automethod:: __call__ +# """ +# +# def __init__( +# self, +# dataset: str, +# cuts: CutSet, +# num_workers: int = 0, +# executor_type: Type[ExecutorType] = ThreadPoolExecutor, +# ) -> None: +# super(PromptedPrecomputedFeatures, self).__init__( +# num_workers, executor_type +# ) +# +# self.utt2neighbors = defaultdict(lambda: []) +# +# if dataset.lower() == "libritts": +# # 909_131041_000013_000002 +# # 909_131041_000013_000003 +# speaker2utts = defaultdict(lambda: []) +# +# utt2cut = {} +# for cut in cuts: +# speaker = cut.supervisions[0].speaker +# speaker2utts[speaker].append(cut.id) +# utt2cut[cut.id] = cut +# +# for spk in speaker2utts: +# uttids = sorted(speaker2utts[spk]) +# # Using the property of sorted keys to find previous utterance +# # The keys has structure speaker_book_x_y e.g. 1089_134691_000004_000001 +# if len(uttids) == 1: +# self.utt2neighbors[uttids[0]].append(utt2cut[uttids[0]]) +# continue +# +# utt2prevutt = dict(zip(uttids, [uttids[1]] + uttids[:-1])) +# utt2postutt = dict(zip(uttids[:-1], uttids[1:])) +# +# for utt in utt2prevutt: +# self.utt2neighbors[utt].append(utt2cut[utt2prevutt[utt]]) +# +# for utt in utt2postutt: +# self.utt2neighbors[utt].append(utt2cut[utt2postutt[utt]]) +# elif dataset.lower() == "ljspeech": +# utt2cut = {} +# uttids = [] +# for cut in cuts: +# uttids.append(cut.id) +# utt2cut[cut.id] = cut +# +# if len(uttids) == 1: +# self.utt2neighbors[uttids[0]].append(utt2cut[uttids[0]]) +# else: +# # Using the property of sorted keys to find previous utterance +# # The keys has structure: LJ001-0010 +# utt2prevutt = dict(zip(uttids, [uttids[1]] + uttids[:-1])) +# utt2postutt = dict(zip(uttids[:-1], uttids[1:])) +# +# for utt in utt2postutt: +# postutt = utt2postutt[utt] +# if utt[:5] == postutt[:5]: +# self.utt2neighbors[utt].append(utt2cut[postutt]) +# +# for utt in utt2prevutt: +# prevutt = utt2prevutt[utt] +# if utt[:5] == prevutt[:5] or not self.utt2neighbors[utt]: +# self.utt2neighbors[utt].append(utt2cut[prevutt]) +# else: +# raise ValueError +# +# def __call__( +# self, cuts: CutSet +# ) -> Tuple[PromptedFeatures, PromptedFeatures]: +# """ +# Reads the pre-computed features from disk/other storage. +# The returned shape is``(B, T, F) => (batch_size, num_frames, num_features)``. +# +# :return: a tensor with collated features, and a tensor of ``num_frames`` of each cut before padding. +# """ +# features, features_lens = collate_features( +# cuts, +# executor=_get_executor( +# self.num_workers, executor_type=self._executor_type +# ), +# ) +# +# prompts_cuts = [] +# for k, cut in enumerate(cuts): +# prompts_cut = random.choice(self.utt2neighbors[cut.id]) +# prompts_cuts.append(fastcopy(prompts_cut, id=f"{cut.id}-{str(k)}")) +# +# mini_duration = min([cut.duration for cut in prompts_cuts] + [3.0]) +# # prompts_cuts = CutSet.from_cuts(prompts_cuts).truncate( +# # max_duration=mini_duration, +# # offset_type="random", +# # preserve_id=True, +# # ) +# prompts_cuts = CutSet( +# cuts={k: cut for k, cut in enumerate(prompts_cuts)} +# ).truncate( +# max_duration=mini_duration, +# offset_type="random", +# preserve_id=False, +# ) +# +# prompts, prompts_lens = collate_features( +# prompts_cuts, +# executor=_get_executor( +# self.num_workers, executor_type=self._executor_type +# ), +# ) +# +# return PromptedFeatures(prompts, features), PromptedFeatures( +# prompts_lens, features_lens +# ) diff --git a/tts/VALL-E-X/data/tokenizer.py b/tts/VALL-E-X/data/tokenizer.py new file mode 100644 index 0000000..55ff809 --- /dev/null +++ b/tts/VALL-E-X/data/tokenizer.py @@ -0,0 +1,376 @@ +#!/usr/bin/env python3 +# Copyright 2023 (authors: Feiteng Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re +from dataclasses import asdict, dataclass +from typing import Any, Dict, List, Optional, Pattern, Union + +import numpy as np +import torch +import torchaudio +from encodec import EncodecModel +from encodec.utils import convert_audio +from phonemizer.backend import EspeakBackend +from phonemizer.backend.espeak.language_switch import LanguageSwitch +from phonemizer.backend.espeak.words_mismatch import WordMismatch +from phonemizer.punctuation import Punctuation +from phonemizer.separator import Separator + +try: + from pypinyin import Style, pinyin + from pypinyin.style._utils import get_finals, get_initials +except Exception: + pass + + +class PypinyinBackend: + """PypinyinBackend for Chinese. Most codes is referenced from espnet. + There are two types pinyin or initials_finals, one is + just like "ni1 hao3", the other is like "n i1 h ao3". + """ + + def __init__( + self, + backend="initials_finals", + punctuation_marks: Union[str, Pattern] = Punctuation.default_marks(), + ) -> None: + self.backend = backend + self.punctuation_marks = punctuation_marks + + def phonemize( + self, text: List[str], separator: Separator, strip=True, njobs=1 + ) -> List[str]: + assert isinstance(text, List) + phonemized = [] + for _text in text: + _text = re.sub(" +", " ", _text.strip()) + _text = _text.replace(" ", separator.word) + phones = [] + if self.backend == "pypinyin": + for n, py in enumerate( + pinyin( + _text, style=Style.TONE3, neutral_tone_with_five=True + ) + ): + if all([c in self.punctuation_marks for c in py[0]]): + if len(phones): + assert phones[-1] == separator.syllable + phones.pop(-1) + + phones.extend(list(py[0])) + else: + phones.extend([py[0], separator.syllable]) + elif self.backend == "pypinyin_initials_finals": + for n, py in enumerate( + pinyin( + _text, style=Style.TONE3, neutral_tone_with_five=True + ) + ): + if all([c in self.punctuation_marks for c in py[0]]): + if len(phones): + assert phones[-1] == separator.syllable + phones.pop(-1) + phones.extend(list(py[0])) + else: + if py[0][-1].isalnum(): + initial = get_initials(py[0], strict=False) + if py[0][-1].isdigit(): + final = ( + get_finals(py[0][:-1], strict=False) + + py[0][-1] + ) + else: + final = get_finals(py[0], strict=False) + phones.extend( + [ + initial, + separator.phone, + final, + separator.syllable, + ] + ) + else: + assert ValueError + else: + raise NotImplementedError + phonemized.append( + "".join(phones).rstrip(f"{separator.word}{separator.syllable}") + ) + return phonemized + + +class TextTokenizer: + """Phonemize Text.""" + + def __init__( + self, + language="en-us", + backend="espeak", + separator=Separator(word="_", syllable="-", phone="|"), + preserve_punctuation=True, + punctuation_marks: Union[str, Pattern] = Punctuation.default_marks(), + with_stress: bool = False, + tie: Union[bool, str] = False, + language_switch: LanguageSwitch = "keep-flags", + words_mismatch: WordMismatch = "ignore", + ) -> None: + if backend == "espeak": + phonemizer = EspeakBackend( + language, + punctuation_marks=punctuation_marks, + preserve_punctuation=preserve_punctuation, + with_stress=with_stress, + tie=tie, + language_switch=language_switch, + words_mismatch=words_mismatch, + ) + elif backend in ["pypinyin", "pypinyin_initials_finals"]: + phonemizer = PypinyinBackend( + backend=backend, + punctuation_marks=punctuation_marks + separator.word, + ) + else: + raise NotImplementedError(f"{backend}") + + self.backend = phonemizer + self.separator = separator + + def to_list(self, phonemized: str) -> List[str]: + fields = [] + for word in phonemized.split(self.separator.word): + # "ɐ m|iː|n?" ɹ|ɪ|z|ɜː|v; h|ɪ|z. + pp = re.findall(r"\w+|[^\w\s]", word, re.UNICODE) + fields.extend( + [p for p in pp if p != self.separator.phone] + + [self.separator.word] + ) + assert len("".join(fields[:-1])) == len(phonemized) - phonemized.count( + self.separator.phone + ) + return fields[:-1] + + def __call__(self, text, strip=True) -> List[List[str]]: + if isinstance(text, str): + text = [text] + + phonemized = self.backend.phonemize( + text, separator=self.separator, strip=strip, njobs=1 + ) + return [self.to_list(p) for p in phonemized] + + +def tokenize_text(tokenizer: TextTokenizer, text: str) -> List[str]: + phonemes = tokenizer([text.strip()]) + return phonemes[0] # k2symbols + + +def remove_encodec_weight_norm(model): + from encodec.modules import SConv1d + from encodec.modules.seanet import SConvTranspose1d, SEANetResnetBlock + from torch.nn.utils import remove_weight_norm + + encoder = model.encoder.model + for key in encoder._modules: + if isinstance(encoder._modules[key], SEANetResnetBlock): + remove_weight_norm(encoder._modules[key].shortcut.conv.conv) + block_modules = encoder._modules[key].block._modules + for skey in block_modules: + if isinstance(block_modules[skey], SConv1d): + remove_weight_norm(block_modules[skey].conv.conv) + elif isinstance(encoder._modules[key], SConv1d): + remove_weight_norm(encoder._modules[key].conv.conv) + + decoder = model.decoder.model + for key in decoder._modules: + if isinstance(decoder._modules[key], SEANetResnetBlock): + remove_weight_norm(decoder._modules[key].shortcut.conv.conv) + block_modules = decoder._modules[key].block._modules + for skey in block_modules: + if isinstance(block_modules[skey], SConv1d): + remove_weight_norm(block_modules[skey].conv.conv) + elif isinstance(decoder._modules[key], SConvTranspose1d): + remove_weight_norm(decoder._modules[key].convtr.convtr) + elif isinstance(decoder._modules[key], SConv1d): + remove_weight_norm(decoder._modules[key].conv.conv) + + +class AudioTokenizer: + """EnCodec audio.""" + + def __init__( + self, + device: Any = None, + ) -> None: + # Instantiate a pretrained EnCodec model + model = EncodecModel.encodec_model_24khz() + model.set_target_bandwidth(6.0) + remove_encodec_weight_norm(model) + + if not device: + device = torch.device("cpu") + if torch.cuda.is_available(): + device = torch.device("cuda:0") + + self._device = device + + self.codec = model.to(device) + self.sample_rate = model.sample_rate + self.channels = model.channels + + @property + def device(self): + return self._device + + def encode(self, wav: torch.Tensor) -> torch.Tensor: + return self.codec.encode(wav.to(self.device)) + + def decode(self, frames: torch.Tensor) -> torch.Tensor: + return self.codec.decode(frames) + + +def tokenize_audio(tokenizer: AudioTokenizer, audio): + # Load and pre-process the audio waveform + if isinstance(audio, str): + wav, sr = torchaudio.load(audio) + else: + wav, sr = audio + wav = convert_audio(wav, sr, tokenizer.sample_rate, tokenizer.channels) + wav = wav.unsqueeze(0) + + # Extract discrete codes from EnCodec + with torch.no_grad(): + encoded_frames = tokenizer.encode(wav) + return encoded_frames + + +# @dataclass +# class AudioTokenConfig: +# frame_shift: Seconds = 320.0 / 24000 +# num_quantizers: int = 8 +# +# def to_dict(self) -> Dict[str, Any]: +# return asdict(self) +# +# @staticmethod +# def from_dict(data: Dict[str, Any]) -> "AudioTokenConfig": +# return AudioTokenConfig(**data) +# +# +# class AudioTokenExtractor(FeatureExtractor): +# name = "encodec" +# config_type = AudioTokenConfig +# +# def __init__(self, config: Optional[Any] = None): +# super(AudioTokenExtractor, self).__init__(config) +# self.tokenizer = AudioTokenizer() +# +# def extract( +# self, samples: Union[np.ndarray, torch.Tensor], sampling_rate: int +# ) -> np.ndarray: +# if not isinstance(samples, torch.Tensor): +# samples = torch.from_numpy(samples) +# if sampling_rate != self.tokenizer.sample_rate: +# samples = convert_audio( +# samples, +# sampling_rate, +# self.tokenizer.sample_rate, +# self.tokenizer.channels, +# ) +# if len(samples.shape) == 2: +# samples = samples.unsqueeze(0) +# else: +# raise ValueError() +# +# device = self.tokenizer.device +# encoded_frames = self.tokenizer.encode(samples.detach().to(device)) +# codes = encoded_frames[0][0] # [B, n_q, T] +# if True: +# duration = round(samples.shape[-1] / sampling_rate, ndigits=12) +# expected_num_frames = compute_num_frames( +# duration=duration, +# frame_shift=self.frame_shift, +# sampling_rate=sampling_rate, +# ) +# assert abs(codes.shape[-1] - expected_num_frames) <= 1 +# codes = codes[..., :expected_num_frames] +# return codes.cpu().squeeze(0).permute(1, 0).numpy() +# +# @property +# def frame_shift(self) -> Seconds: +# return self.config.frame_shift +# +# def feature_dim(self, sampling_rate: int) -> int: +# return self.config.num_quantizers +# +# def pad_tensor_list(self, tensor_list, device, padding_value=0): +# # 计算每个张量的长度 +# lengths = [tensor.shape[0] for tensor in tensor_list] +# # 使用pad_sequence函数进行填充 +# tensor_list = [torch.Tensor(t).to(device) for t in tensor_list] +# padded_tensor = torch.nn.utils.rnn.pad_sequence( +# tensor_list, batch_first=True, padding_value=padding_value +# ) +# return padded_tensor, lengths +# +# def extract_batch(self, samples, sampling_rate, lengths) -> np.ndarray: +# samples = [wav.squeeze() for wav in samples] +# device = self.tokenizer.device +# samples, lengths = self.pad_tensor_list(samples, device) +# samples = samples.unsqueeze(1) +# +# if not isinstance(samples, torch.Tensor): +# samples = torch.from_numpy(samples) +# if len(samples.shape) != 3: +# raise ValueError() +# if sampling_rate != self.tokenizer.sample_rate: +# samples = [ +# convert_audio( +# wav, +# sampling_rate, +# self.tokenizer.sample_rate, +# self.tokenizer.channels, +# ) +# for wav in samples +# ] +# # Extract discrete codes from EnCodec +# with torch.no_grad(): +# encoded_frames = self.tokenizer.encode(samples.detach().to(device)) +# encoded_frames = encoded_frames[0][0] # [B, n_q, T] +# batch_codes = [] +# for b, length in enumerate(lengths): +# codes = encoded_frames[b] +# duration = round(length / sampling_rate, ndigits=12) +# expected_num_frames = compute_num_frames( +# duration=duration, +# frame_shift=self.frame_shift, +# sampling_rate=sampling_rate, +# ) +# batch_codes.append(codes[..., :expected_num_frames]) +# return [codes.cpu().permute(1, 0).numpy() for codes in batch_codes] + + +if __name__ == "__main__": + model = EncodecModel.encodec_model_24khz() + model.set_target_bandwidth(6.0) + + samples = torch.from_numpy(np.random.random([4, 1, 1600])).type( + torch.float32 + ) + codes_raw = model.encode(samples) + + remove_encodec_weight_norm(model) + codes_norm = model.encode(samples) + + assert torch.allclose(codes_raw[0][0], codes_norm[0][0]) diff --git a/tts/VALL-E-X/descriptions.py b/tts/VALL-E-X/descriptions.py new file mode 100644 index 0000000..531b955 --- /dev/null +++ b/tts/VALL-E-X/descriptions.py @@ -0,0 +1,31 @@ +top_md = """ +# VALL-E X +VALL-E X can synthesize high-quality personalized speech with only a 3-second enrolled recording of +an unseen speaker as an acoustic prompt, even in another language for a monolingual speaker.
+This implementation supports zero-shot, mono-lingual/cross-lingual text-to-speech functionality of three languages (English, Chinese, Japanese)
+See this [demo](https://plachtaa.github.io/) page for more details. +""" + +infer_from_audio_md = """ +Upload a speech of 3~10 seconds as the audio prompt and type in the text you'd like to synthesize.
+The model will synthesize speech of given text with the same voice of your audio prompt.
+The model also tends to preserve the emotion & acoustic environment of your given speech.
+For faster inference, please use **"Make prompt"** to get a `.npz` file as the encoded audio prompt, and use it by **"Infer from prompt"** +""" + +make_prompt_md = """ +Upload a speech of 3~10 seconds as the audio prompt.
+Get a `.npz` file as the encoded audio prompt. Use it by **"Infer with prompt"** +""" + +infer_from_prompt_md = """ +Faster than **"Infer from audio"**.
+You need to **"Make prompt"** first, and upload the encoded prompt (a `.npz` file) +""" + +long_text_md = """ +Very long text is chunked into several sentences, and each sentence is synthesized separately.
+Please make a prompt or use a preset prompt to infer long text. +""" + +long_text_example = "Just a few years ago, there were no legions of deep learning scientists developing intelligent products and services at major companies and startups. When we entered the field, machine learning did not command headlines in daily newspapers. Our parents had no idea what machine learning was, let alone why we might prefer it to a career in medicine or law. Machine learning was a blue skies academic discipline whose industrial significance was limited to a narrow set of real-world applications, including speech recognition and computer vision. Moreover, many of these applications required so much domain knowledge that they were often regarded as entirely separate areas for which machine learning was one small component. At that time, neural networks—the predecessors of the deep learning methods that we focus on in this book—were generally regarded as outmoded." \ No newline at end of file diff --git a/tts/VALL-E-X/examples.py b/tts/VALL-E-X/examples.py new file mode 100644 index 0000000..205210e --- /dev/null +++ b/tts/VALL-E-X/examples.py @@ -0,0 +1,24 @@ +infer_from_audio_examples = [ + ["This is how this machine has taken my voice.", 'English', 'no-accent', "prompts/en-2.wav", None, "Wow, look at that! That's no ordinary Teddy bear!"], + ["我喜欢抽电子烟，尤其是锐刻五代。", '中文', 'no-accent', "prompts/zh-1.wav", None, "今天我很荣幸，"], + ["私の声を真似するのはそんなに面白いですか？", '日本語', 'no-accent', "prompts/ja-2.ogg", None, "初めまして、朝武よしのです。"], + ["你可以听得出来我有多困。", '中文', 'no-accent', "prompts/en-1.wav", None, ""], + ["この文は、クロスリンガル合成の例です。", '日本語', 'no-accent', "prompts/zh-2.wav", None, ""], + ["Actually, I can't speak English, but this machine helped me do it.", 'English', 'no-accent', "prompts/ja-1.wav", None, ""], +] + +make_npz_prompt_examples = [ + ["Gem-trader", "prompts/en-2.wav", None, "Wow, look at that! That's no ordinary Teddy bear!"], + ["Ding Zhen", "prompts/zh-1.wav", None, "今天我很荣幸，"], + ["Yoshino", "prompts/ja-2.ogg", None, "初めまして、朝武よしのです。"], + ["Sleepy-woman", "prompts/en-1.wav", None, ""], + ["Yae", "prompts/zh-2.wav", None, ""], + ["Cafe", "prompts/ja-1.wav", None, ""], +] + +infer_from_prompt_examples = [ + ["A prompt contains voice, prosody and emotion information of a certain speaker.", "English", "no-accent", "vctk_1", None], + ["This prompt is made with an audio of three seconds.", "English", "no-accent", "librispeech_1", None], + ["This prompt is made with Chinese speech", "English", "no-accent", "seel", None], +] + diff --git a/tts/VALL-E-X/images/vallex_framework.jpg b/tts/VALL-E-X/images/vallex_framework.jpg new file mode 100644 index 0000000000000000000000000000000000000000..c8dea042c2e3919003721a6adeb67cb658ed76bc GIT binary patch literal 52189 zcmeFYcT`hdw=Wz-dXwIX0s>N1n$)036ESp(^z1E&{t-0o!^EcvzX<++FOo|{8p$arsiyxMFRF=8 z2NM4WU1lU>x_0X>IkUl23Vsik+iwzcCVI(N`9Y` zn)cyi`j_0i{DQ)w;*zTBn%cVhhQ_AOuI`@RzVAPN4v&nEjZaKYq0pGcrR9~?we=0$ z-u}Vi(J^px`WIg$L_YuR`d7yOfiFfPUzaXlCcRAY7hfcoe2I;e@iN)9TjWf44Je*^ zF!SGjL&AOi z)430(-)~EN0x`IKX&JUfDWR7uLYHtFJ4lpGB7GrW!fRAPmbZgS{!-XIb(NONuivWO zHTj;2pf;vZ?H|0QP|$YNLpin)8WYF;FEw)~9l z;3&Sztj3qC;WrTnaRieB6y}Y?wbq)aE}Kf;&Bz3up$hM7g`MpwN$X`fiLQwum2|h% z?%m?Kag87hoG;i?Ih%5M&s9t8d?`P(E@fpEpG|HmzCUbtxBGI%WMmh#$Y`a_|6uu!>^zerv!{FjhFC zH-y_f!!+5diZDEa#u75TOLbMUei2fZQ5s_6U_}Pm;D_Hh`yGOQt3pOqMc~{c$~wgx z2Ak~czNqN@EitaVZ+(HvuTu8f4V7!uD0voz1&LE+8ZDB2>2Iw+q8p|1=ZhEMZOQ50 zb(`C`QC;N4A9M}R(vu~hjtVAhT2yUJRs!?-zH|NXB+c06Sj;#FSqwBlECx0P*b7h}*IXmcvO8U6d-rqU6Tupdl2jT(MZ+X*m#WqYRi5Omoj8@|=C^6+e%s zeAO$kvdo?Q#q;7ti22HjrU}3F;3s@zx!I!G*2K$9Myyp3`H)ok6YPA{qt?K@b<}oG zcz&thw|=tjQ2qU3)24316F+-fY5R#+T&)!CSv3wJ1i{ULJVzRgOH9L#V9m#!mTRR` z)6@N2?y!PP7vpGQQm|OnrxadoijgAH<>`bFdBwE1UMB~TOR$K$qr$j9F{K-9NO7F} zQ7gV~Jo%mlM(24;g9)mn!%oLMMdQv7lpMs7feNTQ>os(~05QO?V_YCS2nP+T2Z`)W4^n!?UkaBrn*upL?RAQ7yb=*l5DYeJovD z#PS%w_vD1GXX_Bl=d+t6my0+^+gNuj{?rx%zPgR0hc0K!t9mDH*l&l3YOl;Tm-$eyJ(m7bS*xbHZ zH63aFBc5|;Z;jKO{H{BZl6f9L4iZXgIi%Vr98sIaYP}J9X5t<+z`Im0@p8_y?V~D= zAKuNrD#^0S*Y9wq31c>#xHj@q%G*`LzSUY)#k8*r71#UznC@!i)lnu=m+pvP!u{>9 zjNi3KAN&LX>n(IRd38Vudxq}50MQ?o*>xpZ+s*)uUe@Zm`}uY9o;q~VC&wO_l6Q^7 z#CSBRy*;b#dgo~lXKgW~h$|~tiuekz&vsqoC}SiTI}tN-e@rH7yYTb72NH)O606bo zKb=o4iReEK^649^(WmGSVg&**3~!Liwc{5p+m1*{;lTz&i)(D%)Kk zpLzttPugy~rfK=kGR#~Jwcg!oUbp};0?Z4@lss5HG(s15X>LQF-L!6kMp@e)nfYSs z>e)|I4S}mXd}<1$W1F9`?(yQvubQU5MPsD4;tUtHv$m;An;f-ie}yX;H9YwQeyYaH z%3`2K_HWJ8)RwVDr3e;PBNnLsd(2elnjGPN=!%+dT#pZ~=6Ku29i0W}UVzY^{YP!v zZCl{F+3o$7-Kvv-=X=|}nZmpoPb+-S6CE)uiGJ*2~^tO$_t~~ET zm?m_bxz5ci{W+6LtAk2$p7@;4_^d~VqM`USDGtQ+D&c*;-v!7p)=q9YKtly$RJcjk zkXPpW1TeY)UH{b!K03Jo<=@ci%sP>@xp@JqA2`49K$YbJ^y4)y;v9)se_((uBFF+TdyXgsk zzpX9{%@;zM)<_iwJ}8V#uRUxjBhJu}&Gr9WRw`U8m>IPuWg08IJqHVoP(s@-L|L@z z?fAcVx={4b)8St{#qPF4YUHD3HCyzQv~X^+88?WgKzIj;so^Z6IDGJ^H6El3T`T_Oy4?#Dha8--S zGwO*hf4x|)SUHq+N{eD_m6c5&Ym?}#_5y{U8d&Kjh-pfYB)tPW(9s40gdkE~a}?OQ zP;iQjl2{F*%cpG?l6}bpmlCHoil9qVgLCt3rBpamvLkLb@J2vE0|w)sdB|*I;Kh0MrjRt&4Stt<^o#qXjH} zJfHAhuVBBcTfAi1Coe6&%g65_pv(XBJLMhG=-iEjC_efIB!FOuP*y^NsbHbHwxTFo zTHqabpqz__;bq+_s#KM?nm=hJ8}u5i!fk995bY~i1)P0Oxg(Nm7~CyJ_W|1%j+xDO z8owzX!`s9beNAprd{~LYj`Mx_FuNxReuB$DSH#M+y+Lrewm|*sY`7EDLJew%UlRB? zP7h~We=5Hz^xJtFV&E=lM=s>jy4A`~B)B1&I15wyjg@5&+g5tx2GViFGxRCl`m*k8|0ikkzn1DK@+wcSUSj z2YgeR$8;dPMeGY8rCv7#hvb<%TBsC>k!KZ5UvF(uOX?PMyi~3u{AM&Abh59LVRrsO z9M0~Lvzf{#2dIDDsB2P9e#-x*WWCg9t?mM3->A8_>pGa`S{@_Xt9vs1vSa_Vu6~w2 z;Bck}47*3E3PG6xsqUusA=d>QS-0Hj%S4s{OZz%4#u>qi%KM&Qqwi;2p?w#Wi_L0J z2+t&$Ag@IyL}apHSH7{nbLM&!SOZ-k=Q%-bxY@8GeQap0jtJnQYwNJy|rzY0O z$6f6Xe=hJctS#o1!mb)u>KHs6Jy7N|Zc7hW zDFmWwrvo9s#w>J%RLv2iec@AFa*tWPqW5g;r+JsX^6PI> z<@-AsYa=_pUyk*uaC#1Q|VMpT&`PoZ((?!lRP*)yojihLriWgacZ6CscNPoC@6t*on~y(UGw z0Zy(;y!u_8eEu^cU!u{9e|HUUBxws+rGo-*72Mw#Gt>=Fjt?C)HYV}hcG^Kut4?Od z8Lws2WW9uH=59x=jXjD6H6C>zIREz_|9_v7vf|CUbkz)Bgik+j*;lA?IZNHBiFhAr za=SA13lK621$;3^2i?Va53-kMy6 ze>gilll@CB@MnP11?WZuuzCS{=GsPx8rt0buZfJ)4urZ;1LP?F{hTZ7af+ZX9k}5@ zh9oHZ`D^i*L})g7Gv#RIy0PH9?^Cj}OCO(nju0vxB(d>0({$mJ!mhUOa&D?#O66#I zT;Q~4J4F+D`Ny>wCglJI!|&y}vtj-!0wUg`?Y9#{7}V9k3PJ4`pqi8m(6-7M1O+5p z2l(JL;to!IN+j$Ua}u>-Q%2uz>3}g@4Up;Gqp!HO4g?GQHkQ3}-%dCVZw)tkvE1-U z{NXySZq32GKqg{6u|nWH!p?mv<7ge|!rp%EdZ1^gSP;bxe*gCTKoprb{ct*Rmt4sM z(rM`qI70zZNL-8C1!z5p4X|@R&v-sHljW#}y~-PTJLAXugdHiH8j^#;?fSF(&fu!~ zt$AOJu2N}07CZDh@?*gd56ZperXU-2K7Yz#v%ZJHV$m0%kO0CMQUbUDIV{f8;&tLt zBYgyx(YI!7Sj!m1jGU3v!>o_jqt&_*eh@J0d9|`r4&M13smcNu8~6UK^bqSCYV959 z1qaKrr3UhF->sghq5WFFxUI`t(`i=EwrwH-MFtOY=6hmpnIQfP(c`p!fu#%nK= zYBqe>e?@k6)8+z%ov>=cX?0~$Z_4*r92*Rmxvh#c8~+d#-F`FqJX37Z77)+ErId#xm98rVQB`9b)%|N z&O+jm;~hRr_(G<(bz8fMm18{XK0z~KYL(C>-*(EI67+76#LB%lfyEXEUf7MtYvor7 z73QwVpN5q_$atsA+Wh@({Nn3)u@Ip3%mvN?&|vn_2wvE)GX`t6b?+nFt>)>3S)@-L z@YH+NX;A@FjrYLicC_)eJ%>bRieM!x&l_{VbenQ*&S&D|7oZkh1$oIIiQI8rpX`Ws zAI$=^FJao{C>AS%^9{AfTAki%N*eKAZqaZ1@L_f@bN=QEa`*f*0_}w8tc@VBQye+q zg%D0@zX{)`m;4m?Y~*mC9Ar;w&q(q1=h=7~ETh*zcow+?+#6l7t@{yCTQF)!z8|TF z+m=*(>euIgt^(Y`K-po<=**~6*lw2xpw*Kmb^|%|?8C#l?gtyecLaH|H8=6bB&~}y zDt7wV@-kQS_B~!Po1}4#Z>{8>K#W0|9G3-6iX`lIp<4%_tZUi;V z+ZR@CkvV9^BGUdR5=wnnjB_b%zPva5YbVy?AKwe-{G1-Rv!LV?eLDQx!X!S0VhE*xoCE? z_=n%pFWURTFFrhf6H97&JjXv=(^hw0ZB$)zM%U2XyuZV!IzTcT_58<){)ziO4?|)r z9rSi(qx~;!Hs_Axi|Zc z>J7k7XUNuM!n;hq%Tz2-E>mc%R?K|SDcAR2PIta9Z>$d!&y#SItHF)A0aoy@n<6C^ z;k8fm*Xk8p$7b}`Q;)X2C5B%1R*LROB~Cm3aY-Yh#XXeXj#83`dZgyq)HJe+>13mc zBudx(dfw-ANo~o&Cee=s;Na#x8rgVGu?CMkIaOZV6dgo28*eXM|X|2_@c|mHy_lF;??t=CKRhLkK{DBMhmw*GbW*~CcSn&Eo@+p zuV$VX);Zm-$~-E&*=$)lDL$rCUrlNMuDV3D{K({oxZ*dygOhh(`!~d5ux;Id>}-%L zjnUwtJJn{?OLx12T`dzUwVjy&aF=-`<^RexE|}{xcmIWA;}0?w1WNWE!T7 z*>KqFymyZyjdb3ZM$3w50%a9;wlK1ssDS}xi0`%V zd6P8bH?H=t%vp7Gx9!O+gFW}M9u(rlFsl(rF2H`@pk>Ufw~g?pp>5>g$06lHn!(kHKqQgs_;CuiDgekgisFDzV_#$uORLW))=UR?!1V+alKg^0V zmQubeK8az8%Ofqf=STke5)2V-v$LH%dw5<^TM=!`s(J_JX}o{v`szI7=i}8X$Ju(_ zTkWF1gLSB-o>oS<$zOn6=K-OCU{zi`=_uR`o6*&jQXh5T(j4%SKl%u>qq803_gYjy z8vfH-@cSxb`neqO71G3B1FW!^#bZ2kGCH-%OYaea$;w4e-Kf&?hVnr}^xV#kPkr^T zi@n8fD=MCj&j(+C77CeQ>g{v9g9QNl;;$&LR>irV72VgYs5>b%Afp_y^mEp~Mn)_Y zr}_8A%YdgwdYa6_yI`#~zav?gr}Gf?C3?|TE6$8ZnSZXo%FYn94!l+S*NL!I8b8s9gn7RaNc>YIoe!NWsVGlvk5V*A3%?LuomDMuCI!t*h$n1V| z(XDfm>tV@sT!0t$Pamq=G;4r7vl!vtPq72-sQGB`JLQt%2hH{kFmcT7{##(5JA<$P z{J!^QqbnaSc1DI5!)f(Fp6{`qALK21RQpZs`*(27Ikv~K!p`!PO+qWz(R$Y z@oHb9#Re6KKu`|0q8Gy4;$4~%sQYk8Sq90Md_UssL9gy^P3=(sXk_I~*K&$ETED+x zH3-C$soR~&2&5Imq^+tOv7ge*8!-%*yU=Gbk=``M@!Zp%K3qC9-|2t6-T1D5iz0&Y z5d3h+yh*2S$FX2&zdjbp3luEslkl6X*%9jVZhOX!7dE0;0PsCq>_M zg4vz4?!216mZpP+M&lVV=J0!SR~m5lZC!A*kg2*f(-@f>enGNrR=_14d8q1ge^48IP%~FH8j@Ok3AophVHYZg;3Ex7bOjT=%}s#ihhj8 z)HWuoH|t6ZS86ptzNyzVyz^C-^J|4Hw_JiTOwul8q5g*0r991UnK`rm<&Xw(qp7u$ zBj<5;Y1P1tj!d!S?$AEFj^k@Yh(ga8fNq;+y>#sTG;Y68|M>`^6hJY(a^4M=+f8jO zg}g=H8MM4VE>Tlvt1Wk&zNeE9GYk~c&4BGnnZh2c&J>Vx-bYP8xbyRs=7MVBm%_>N zMWUVOP6JSw0KO#_S`G`_>OG2bIp}S9O^wm=t@|kVM8Tfi=95kioMP62$JO62F9i+< zQY_j>6vL9y@wJ8PY*NL}w<&G7wU^@M(w=``R@eTSjfT!WQQ2(xQg&AmBk2&3NrxkC zM~VZp=f$s1S&+2R4D`+x8gCU_dzxpV-ahhvbd{(|6@;-fkj+D)e zYjGU7poB#tEIVhFQ#~p!G-%iSjs06o8Hr_@&{LARt0L1cd}x*rPf}nKx(oBMaL9s+ zOV<$05X@%fVnY>rFIwAedw5>2;jPup$d4b*yJ^>#tHj_#bRDXPflv+lq=0Wmjwf}C zXmFHSpW;_`98k4*cU)Fa-iXPOX{~cyC`8Ho(!xwp6S;!KanSMymfo3L{H-o%tuQyd zm;>f%!1Gj3XTED~qb$Yu?BjKG;x$E~*DoC|fk=AA8ordsj9Paup}rYtuojcpL}3#M z&*qPQ5q+o%rHcE2)&=O6``L>NQ1t|xG{AHc)JjW`m|sNla6Msn{g zVMV?&L`u(D05sjzP8${kmD|hPqx4mjAF}Z{H9kA+ucMgS2_z$sfk}wjks(H30e(7* zLz#?j9U7g2F8L|U$6oUvf>i=3=A+M*olE*s*OjGk@fVvgj)eDK*MD1P?(k(8r{bY=7=m9hirL|3w=gdbwrg|0ra8y}W zH4*LVuX1SxL!*tkC3v%^s3u77lh2+X(zzxvygO@CK4ku`7JAc`f-niDItZ{7Q;{2= zKpz#mu-3k%A!~ZyST$40AIw7EthT@fI<=2Jq3M{D!KQ`PR-o#waHcsP;Bbi+6DQUg zl+_4TOkl{LYfQ3@OzyWaDkWNip}j6*?H=YZsiOfbZ($gLWe(0W0#`=|>d8vn!i^(;mM>4LWG!hFc8bX%?ZEVHP`cN5_wG-ZSHMSEl&oQ@?e;NOkg6+>T=bY0tdg zxYhsEI%sH)wHjD=$Chh*Gi*10;TXN|asI}QR!dQyE>k+WRl! zeWP_qbsE#CA~RBEUTg7X-l_mHyL#tqg}3%GyrQ;{hhr^L<+c{T&e!sz`v(aR#Vn7C zh-SyH1&Z_c-0tTB&DjjuF24xZ;jlWucku#53rNm)+FZlAN4DH`@(yrv$;l9;VQoz4 zmpnGRDFSJ+0zL98Z|g;hjJAjsc6b!Pv>xMPhjo{03u0v7zrNof9KhWi$?T>E7GqAj z)2n@whN+#1#v7bj;%OTP7b7~j@N+y9Q)PIAC+TTZqq5&ba3~vowyT$@AE^x}tbtT? zF3`VYBbS$a(rs`7Xi%5zfxORJ(=b9I>&t7uU^j{Gsq;K=puR{a=%% z{z{no-`DJ6pYo@*~bN35OIwQ5VN&v<=1rr z?{N8{is|X}!x2{jr*bJNA@iJYSxyJvXP1h2b?&gPg0W*+L|AR@8$fXZq7INEC|P#a zK5!oL(R^J~aaLkCHDy5calX6aptp)N);>aOMlpm#zgc z{>D}tu>BtgG&?Mr{|&7yoLqzaokA_V=x1RdYjb_VGi15ysb>WtaUA-T#5!<~|n+NDxB1cqjz~oIw%f33Kk9R?FWO>K(x4*tL6v@@0 zQK|-&lu!uxmK*Fo_;bN(dAawm2pOMgmnn` z+$sNuCBs0fA-%ho@WKEXjUd6R0mNJq8lbQ99Wu1YDZQ@V)kWFWRiHDWO3QQ-_eHyq zr?j}@(f>|v|D%9tjh7;B25<$&hyG2vWDuRj!-!SPG2skK5Nf;t0nP&u<_5que+lNe5&`PQf?n}PBCa}eV^ zOc+gGi5=)wlqGKPh?;Gv{8-(`QR#ybWUCXUD(*Gp-`b;Eu zdHWTXm-(@M4iHyTZj=-^eCn=5!lHg#vpaH-Cye7DT?9l*3VLGpev0rr=<@KP*x9x` zfhq3VZpHHG&XZyMi?1k49Z#{eBq+aFZLHff9gs>0RF2ynl~K$YE{r7eSIyPA#$h9Az>iPIQEzT@u2?OloX@;al-1gUZJ0n{w2{ ze!hd-RbZV&tAK2fLP~!SgyL ziuYHks(($7xXs;Kw_wqyNY*(M$KP=k>r@Oxek^r*2RuT#NU&K;b^E8hKYI1R_y6qluv)6d~YTl`GILb$&5M6AFebCPSmZP>J^0M zO|GY2DXWcpPvz=PO)n6~B3g6=A~9Y2hxGFQ9dOqT&X8ik1$rGBlI5(LOXbB53sCx> z%?lICmWAr#4!QY_0Y^q#8mW{!x0NPJ%u-=p-Tv&6wFC4s{qDZEu!KBsC7o-D1{Uts z1Md5m=$EHFgn4+vRK5iY=#EJoW^n9m+rpun9TW|!hc#=X1@H5|#x5G(_KA@g($Mno z;Qic^{19^|*YJ1$Ac=W8lYy1YJnL5KLdj!2L504L4VMCO7odsq=D)^@5!optjx|#G zZ(~)++T9tAo&EC5%Ru9qBa5kgl(LZ%`h!6sonTeq_vGR}qb)I|uC^;L4>x;i9c_kv z&fa{uU0)aI@*Gym+0<6k)j<*f$I7sr%zfe&W}GffyAOSFs(^5 zthzVyBJS+VgP=&r>0zy+tSFv(<#jO!{NavcNxDt3``5=>-p*&3pwE#i9ml2K$&Vy| zH_sch7?^lVJeb$1PTE+;H`wuHu1*FFCcAx?8%r%JXM@DB>3P(CEim^?kbWh+nLudu zsjD}zneJ1j7mU`J`KlB>p3n6jDf>*oP#SnF!D3*ZPUhzjj84!14sp7=!IT(e^$`9Z zt^%s))mY=u_smQ$0kX5wN@Fuo23L9J<`&hVWWb(kwX-Ko#{fnztqF{^vJ6mOgc zrltt!Ty=z(I=SV=kb+9tS~7$M1Lr;nZga^B^EsB$E*sv0mu>ia|I17N4+P=txt**m%h4zUV++=;+QI9I9P2)Q>o zU)d>+_Y_h_dBVPvg5OoVN4(sf9-laJ0!FaEI^|ZBVjk~=9Y?>)LGh||-H%2lB#k_I zLRhBIl0DKW8+h+lduRDw(Ej6tq`Tiie15VUX=eJ9sD4im(Yx-{oPw#E z)J56PD}}Q`4zjqmExR*+6k&LtD@X{?Tx>2T3bV~WTaN9qz2wQ7zq1z7 zoE!7$#+gjKlYVUBpTG=a?38NQ0SMJ~bo$Gf-gI$aC*+y(*swWCI?tb!M3d_>`Gd$kZMg}(kovNUZ3$c<#{I1u ztOt+vz34pdt+bPv!qOc!afVgmqBBaw0^wnXaZ+qS00)!aB6MeR~p~MP=?3Ao?;+)|s zn7{n@82>#V|1BT?yA|kU`k$ga?AvQd@!e%2#AW>O8s3%{_F2LXJy)s_v8g;&#wQ`b%hw>Om`7Ut7@c zEosf-hW25 zvvq0OCvEF+a>vGuUQulNU(nJ>N8sd4o(Nz?;VjX+k+M>hPKt*+g%}0qZuu<>!=tEN z+erKoU0AFh@0XBQ@{|+BphKb~TL$+4BH?~L3-kO!emnOmdh0^qfErJuJY;{#~)F$TD_NLMcSjbfv>Hdorh>--T> zZ7+9Y%$oMpz023+{9hm=c>i#gy_{njZV~N@{XQ=(bq^Tm|4vbfzX`OW!Lg|8f65lN zt(=e8Cg_}GvK&&$}+nFnZc%Q0c&jK;XH*7U3{y@SEr+1&G6g8 zOb;vHzh3J~Iq|i->a}i_5UUKu*>)-pyb0owkE@xkU9&z&PjKw9 zwIoi(b93ezf`-M>>W#T3R(6iAMMl@vW=MpBRa?Fwt%rjoV4G<6gdkyI1~&=!a0Xp>s*-D!%#ukLRAg<4 z@8$kzCz?9^`S%uUUHu2})ym7PmxBX~HDJVz1`n)o&Z=4M22;)2`XRx?HGy&)e+GWU z`WgOe`|XPM<=N(8QtRWWxIIWvahZUS)=KCSBVbLok!~$d{c@brO#S(7XMYweQEo5P zX_)v?-yr#PMX}ER3PTQ32!C$`$pX+=;M~gS#(FdII*y_+RT@qoUz4*)iwI{f8KWW+ zyj;QUhOj=v&s#DaFUj*|hVTCwQvarKUH_IK>5%7kI?&t$W`)1N?7l(NK;Ocj%H;&g zM~I8(*S3s%Z3fUB#V@%}%gAjF#l9rd?&yNI^HCu2&n`g8;yCbmRNMt9k9b6=cOjTs zWHA>YlAc8sZj%Ik?C;b8pL(3Y;;)Ch*C}VHx1SitwTIkFZ(Nw?Z~|XZmMVnBs5PGj z#T9i6)i&;BLT*tBC|Qy7D!sAg23{sim3I1|>{6i-Sxm}uLe=(y1s}_5Bc&L!b>IBF z0M$`~NVLL&q+o>|i0b)vFa_*3cVZdvrT6uOkK8-!*}Ipit?`i#Nq5rK8(wb*v&N4v z=u^zdn0;4_=#h%Ep#h|Eb_Tdh&Z$X`M>l5yM7QGj$EWVFo!@NcOCeO(--azZ^<9hi zTVJ1NRuzOLbe;CKugRQiZQ)?aX7Ljyw)}Eu_wC0%Rfzh)+j$D;KmlQ_`(`0{1DraB zqE9N)0&XK+J&0#V^ZL&y8#=ukQ29;H;#*65@5E7a<#y6QD(DGm7Q`JT-&==N#M(FJ z!Lm$o4k7Tn7?{^RH@Fm8_`7H6T6d3JLgt+kN1VE@7)0N?gLWj{0F&U&hE=b@VtN(j zKD!6dO@!5w6wI{_^XDb8&VKmWc-><*B+J?MpU{+q7#&862vvN zP7pe;MoQRh{-7Ig&YmKMi+FtR*w7@nWzguY*A@W!G`mkM?{`dccDJwLxJ>~&^env;b%&ih#+Ge!Je{@m zeCg?zdcW7oCoETdJGg%G!UyAdQ&0vaOQGYu3QyUWD=xI}iVk)H-S=j~xHgS& zrCG&|aHc-8wOdh_qMgX3)MyuFbT!re6aFWNCmnP~AANEQDT}wkwsn10(waC}9>Kd- zz)Ga(*Zg~%ooJP|^Q|!(Fo`8tcQosS^EX2utlt0&?Vg(T@9v#DF$M@yuc!QO)75P)vmS3xzHW~t zAE@TN$V=Tn(CV`NQDuEqiq!59l!)DU_9)tM91CDCq5xO?0v(&>hY6A zJ87Y#gd#{YLoMg)dC6sp_+%ZRcCq5EjSBD?-Npe#L%=9UJ{F=zEa^S00bV+Q*i7G~ zUS$dS^?6on4W#5b*8M|_qk!#eM(xF-`yhS z%uI?@ysv>ISR4+tZ|fS4!|f!~Fx(X!HjWlA#xf(*j$fBN>QsVBB%0XVi&k^EOkb86 zkv5EQPT&LB+b=+zeUVI?kH?j-ou_*>;x!8I7S2@L;7cT*)+5CEKix#BvqIz;)EH_V zD^ipVO7V7pE@t5EIBKH%7nFGt$m%vd;!0^4+}_Sp`~G{VdxQR&%8yHGCfYA=k!DUH zW#CVMM9cuM{2Mcysu5+Sk{QQwwo{>)f;r2Xeh^9AOuZQu|BgF4j(qgV>)v41?+p#{ z<8zOfT@y9)XTLoHaSAmMgPsUZI5)0Zy14jmKoSM94D zkT4zexsl~Kff+RqBtNtUce8UnI=|QP$du~0mVZ2yyE-1q+YdK769z*1j(msl%oy(D zGvT**KXgoS1#i%Xvqiw~vv?TU_8z(FzSN=`*l8O79P2&NLz5MuzbPghTULgglKb#Y z_kE0W-NOv!7WC-$Y^E0L)I=AU#7nG9{^%XnJyy2J2Z-pkVINc0DMI*}IxXJR7*Kl` zIl&w6Qaq#_h)cj|b+p%Xgi^sr(R6(d0u5nN!38FjI~^*fDR*?dqbJ&u?C)eL}jUxaSH!`h>M9#vVbrtmJa@ z9gDxpMP!t!3l=QtF4Kw&LEqP_pDf^PT(A{Awmi8&B6{s`8p`Nm!#;XBs#LZ4%X%u~ zByU)5(rlUAb02PkVHNoa)N0@6=>j;?<|^QFCRSdEZdE^1JYB@d)ZNxy>XTf1(IUo= z9t-}yL$38qgmhO_Y!$);3&ohlwmC_$z!{Bw6pe{^~Xl8U|mPSP1{@e4Osum$0k+if}o>CUsJx!Q+~d&tM3<&)}=v*u?~N zHp92IigcO&08B(M#EBtdjsnNt&7u4>dsERT$5$F*XE|lnlBh|^UoQ;-E;{x=A^(A7vkPzz(8pKiHgT19rkBB)Y zYO)yb?aY4!{isF01XGLT3fvix`S;?PO0*)cvJuYP}LhwV(lTs<+aHG*hVivoxK1G7g+08ZiOwzQ@ z)3hjMld?Qi8L~pIxMbtr%8^da^XG;;VfWy{t|mMV9GljVo`g_;_Fg)9XoN|I{)jBy z?Zs_>x9Z%P43|w#kB}jq(Aap&tD$|Ip!khw(4H*~q2He(+z!Y4%VkbzTv@BaA~1Zl z`!uz%%_y2NsrYh9NhO)^tI6DVRDZ}!1}{J_&e?TGYgH`HdNOa~7$V(e`i8Gf$m$t$ z|Kj#limZ2ByvyxIQok+E0Qa!@p#k!9@0y?5)y7K2X7!{I)3StP;W|gsgWL&D zYUZ0y{LTM|xAzQas@vK{gQzHo6zMH0O^FndCLj@!E+D;wQlx|;y%PliL5hHYf`C$_ z6Dgq=>AjcGAt1da)IdVK>)qu$ulu`a@BN*9@A(1AlIdD=%{k^+V?FbE#)!>`{NBZN zZCYZn77E4*bS#Poc??i&XwAsxY%;hSYH%0ub?0vAe7I9c(zJB_3=$Xt!U|p@_IS?G zL^+1FHoi4#|4<>k-l8(*;r*?gMw;ik*d;Q)97sP2Y4C7`$oynWi;-S1W?m@*uLpA` znK>(XVv~GxU##6k{jvA+FsQqr{h`q<(g9>lFcDmZ?Re|&=tIAgE?8~M($AQvE=K1K3TKj+v+8#)EIn>wSNKYYIW z`h5YFNeCwkbdszoG5zY6{|(FB%_-}K)S`yQ9D6;laQ5w}k-N|uxS-P@Q|^_brl0YB zej#)uig_IT2g7*96moWU%XJ2PL&=b4iZje1Zo2YnHysEvNKnO~ zo?tavDCW_9UVr!U#- zHfsy9q>NB83>*S~$S>(Gsyrt@b=7m}!FLyvvEwZynsqlckPh>*e!K{Zrf(WTZb~&> z%y~gr9@lwTeSIP5qqNI=N1bxcl}Z z`CCm*OLBwQ5=CBKo@(c-#7;BTL8SR5dvO^EHtekc0pczo@@?GXaq6>&A=0lJo@#Ok zse>$$vJSUMXKlY+6>o-;yTdQyg4=zOStzf2xKgaM^J0y=?NH(kj)~^n8!FMu(SE2; z>+lkH`7T3~o|(p#oV<2n*l6w-Do$`2I8;!EeZ!0fC2CINogsxs zHU^j*jj;qMK;D29?`VUhOH&jEzg+zibort=h}bX58&qdX79mKGU36x}g^te_-9b{( z+n?Ls%6LXvsFE9zUD88sus^_4Zx#VMIo&+Eg|V9QdMjcpEw_lLBH!e26AyJ14M=t@ zk8`z(;14^0_YVC#o-NuOQg-M&_>1(b-Wl`Fh>JxCoSQLbEY5D6g&5+`zuVB8PuIrj zaBE}Yd{C-eq!p@2^C5?EZeV{PGfIZ72hHw^H^H8FHKSC5NhW3t_3tR3|9mA4QjzgW z8sLO%^?d{2ihH;zFbl4y*=Q!&-`6S%U8DffiKFv;$?ASbd1kI^!9_RgEWzE-qlni4 z&WxMb6!YYLQah?p%n+bA+HFl^DiO&5Dgk|K?_geFIRY@gj=Fnzew=nQkm3NIF*(0U zX}6H7;;5}|_u_3`T&LBYG8*bDnSBly$i3Op&HzRdyycif0!tNzxLH5C184^E7k5>; zS>|9*dzW>^LItKH_Hfq^*kOYJW;W2Lie|;ls}D(&Eeu@sH)Y_|Mt+7jdz+z@2Kg;N zN~b-1b{fK{wq>2_w6hac^FD^^(wrTy*W3?2u@DgX zF_2;nJx@lsH!AIdRYlhs$s}MOEh%D(ztyD9)no6b%^0_^St-I`6nwjPCT6C^{qrfP zC60fAr~`XWIkJ~1ja!YxBkIf-3R0i9)?_0>Jv1Gi0|ezDnVj^l5_dv00_#vELl2pc zYUqoS!oChY-)HD%)qIiix%O?d1M`b6UiN-i*P+Jby&tEmP3BYtp;53_EruB8!rqtN z8n1X%)WZqwZGMm?Tq@z@<$jG3EQ-;+D3^^Ivm}A|3*`<$6D^zt zwE9llKXTidu6QCOM&Xvum&^w8jg4_(ZBrT{NY&8ldQ5a`!HJ|!@2W7>9iBsd=>$d1 zX^@NAKJd^|i~&H)fh)Lq9SpI+I(O6T^0>Q{`4;|xYI7ybMyPz-VNyXalH*arBA^`J zs$7VnT2GT4u`*RP8mW6>;ixySUk3}?3a@nSdd;)KRsG_HRP}jxgy|RRAJndtARqmf z^vRP#Yz-t-_RLN@xe!b@C)d7QmD9kPXWKQ>A>kZtF5Xk^c{#G* z)JSVsJtBBsXcbhxv@rOY$T^{(K#<2#EHsk)gHZ?X>IK{8xJnwZ=hLo}$;zkel-4Jz z%>{f15+K<~prbGMZ=7C@0O9Zp=`FLyt@Gs!!AqA85FBz}z1>u6E0RobF`PV^xf){h zH>1y8oQU;eZ=$0$zAv5r-S&^j=->P~95@H>EU*NGiO>T}{aDoRu&(&p-@-Y_UgOc* zv+6ooG!%DB;I+p$*<7^+4Kk5r`t5S+TK=D|r4%QxNLOwxKB!s}>pEL9&z<^~eK97d z&&Rdx*<0(k%nx3I2ucX{aX?Ry=Wm8dkI0C&9yv(m)E!xk8ucmS?yV4H*8EI=Zv4s5 zAmJ)9UN&Tjj;)MQsdB3O%LMG@?hmYrWmB_q0Wiw8x_2ViuE*V{UN%(fynaffHKqp9 z2{s%-*Hu^ys$AA;Aq$cPe-w49;pFDZX3V^w&B(CC{URuWPwL&t_z&!C7h@2@VqBDp8{AW0>n>5jbydf>{M8cC z@B0Sw^!@^rd3~NnnlZyWUD?J_T)!-0kA!dAL;vZtp`TFyr7c!Q#a+fx2Z-7vqE=e+kFs9_MPg;Me8i_d#GS%zVUFoi^RT3vTVpB*VI($W^i+ zMN5+p*pI%$KXXJTC$%qxXL9(yw__F2l=ozlHD~<=YSNOCGAHn!rS3>S_ zW8ZN!-q|Xja`a8~C#{lSplK~X{YmOI5S7;dink@U4*6x{i zk`KN~PN<9An+!BgUoi5eI={PfEiGXh+IGjCol_SU^$X`eG3iC9fXi zt?s=pmZL9s!8v8JNBlHZFs ze4W-J%5xM<66invp|pDcw`iQC!`}jzndMK9GhZ}kgT+*;4LQuMk9e}!th1@W>vnX> zYROMHiI?BuEdf+jEDFFJui&}~Cb$;$`V_s&7tl?IV(vSdil*N?@np~99%p6;4x;#4j2+=-HHsCXG~yGh zV>>GqC+}_Sf|Qg@2$|#$IZ4H}$W5YE&T5kRJlK3jO>3v+CIOz-VDBFl2$XgsI0Y7m=`B_3?^jwV%gE#!Xc;NfzHt*(W4g&MBbNJ)`s_#x!*`ACl(_ zfwnH#u)NO;ROYo*9!ZsQYb!Wrm#5PUeQr(x*n)bagQND5&rsbBbo#~lcg{@q;@>w= zS8~pD6Y9PO32j!`U3hJ=i^=)V7SmqOC%oLO!z~ zvySh3f!$@*?@aYqV~R zG4k5x#wfNlNR;d?*3`V~9CcW7I7{4ghgWvFDRqIpth2sf8t(QQOt}%w&Te_fcn`16jOT(ax{7?hx2B4ZVF$AC1rL zJ}pJMG8ao{U+E8t2A3?jH?8aTK+Kv~j7<(BZe?UWH!zdNR>WiGn|xOzkkhYWnQKR` zbF->53K0gcq)OUBJ4)=Ep4H}W=NK?CtyEWLy)E1IjwBP``F$VAnO8;fOrcRPJ3FuB z2Y%63uBS#g4x~aC(xDYP^GL5Z1YL%j?^E6dilfUPrrKUnzS+As9XldpFiRt`&T_2> zHICdj--^L-qERgrYTAZF{&#b5@V0v}-?NB5X4}?t+r>RgFaUIQT_0fZRv8;b)+(YO<=cvHNz zrXALVb>MQvV9|pQvK9#%1+)w=uc~vucb7Fc!4;?tdkkFNkW{bI#zQnKyuwDGLjqcuVI zW3BOF&hI2guZDnpWz^~Q=IXfVMbsN2+sq$8v!0mA31==)if6-qzvij;h{?!L=L8m&Y}vOW4A zDt_F|X_{mxMql0|A^;I#3!-kfOo zga`ts@C0s+<)1cI2psV6sN8B4lhNd$Nw1jf_c6lUyHa7$ou6e5g9O?PSyyKXiKO0F zIQAkmcH&9Vjf8MtufIYvru@Q8mX+Za5uy>IOgi!)lC29e^~oY<3AaYJ;_xxF^%{

;lr5z*OG9`Vk~drKvBm-_}w6nY)Wyprg-Y9exqMa`zX= z0e<9*X~f-9+~&^->`tdVFf8YXsHC?`j%Tt=P7lCX1t-6H#4S%)el;!eU2%x1HCH5X zW7ESLxcsFCrOnaqAtH{h{ZbSDS-PArmzk&*wG%i?v>>ZOK2I>nB_nNh6rnIcOiCgY zLdADCSif@B4FO2v*JBdNPmbF_F|-?#Ous;?fM`Tc^nl(Ui+Ayd4a04quL7@QZOcpI zTT##Krf#LpvZw7md{lkqDw-wqeQ*=aQoYapP;!F_g}Vube#}mi63pFNQ9IEEXevTv z67+Cw{**<(7!@7Z1!rIOyJLOk3De*a~woJtf|{cUbW*~WuKy29nF-o)igK(hBt zHoX4b1$MEc{nkWnqs+JH_tK@WJVSOaA2Tx@Bdo_o{RFAj8G*>ljXf>Zl5}Qfru${= z;|Rmqm}hjffe8!dB{ACp*@5G8Oj!9tWa)ZM^t*ySr}6tIeJQn~S*=zJ<m8?sk z*JkV{#iA}ygH^vAXN`OFnJFrXj2koWULZDq23!LOmQ~&)h8zMBEAT5DcPDqmDQSL~ zWx=UAaDA{L!_lzv;4lwM6}Ka|$l=MElY+F=&3S27%25u+k+}VrpA|cuzF2LO5lJIX zmKMdg0~U6H4T}oipayD_i3yR@zJh>8T`SWguQGZu&1)NI18VqcIq&rs4POvo!5%Ym z*OY|auiGp4Gddm#Jw+mcNVmS_2l>g8N$3Wp1y)clprFqJU;Fv+RDbYrKWV!f|I`** zI`GQf)bkfe8Hw_UQq>6>9??jpEBjH5a9?^L*7lz2;hy5>$cR0+D-`YRw^jN$+Kpoj zn7gWG?~OE2w|HB!^vc{2D>0p% zkvrxJ859;e;*+;Syz_190;uiKcSjIL(03vl8!=z{Fib3X7nuOH%n9oK;wSR!jfgmaJ% z@yppaT&|72%moXxsx|#;<<7j$*0ku9c|!wBeu%+!yXWO9OJxQXb7-f;*$jTrl=mU? z{#yw8cj%a8Z+9DlE-Cic8<{CytdUk7A*N^$MME&o&9F>@J*IWq^SzTxM^Sm^DQXDL zD6NTxhsltl{GAsIj15J$FSe`ZStrwZO=aE`xg)bLMt61Vhki1^fn}?M;??}whY23A zOgmto|9PA$rPkCi{l#&#%lo3JX;Wr4w;}ArAX(!-bm;FL`!_y)j@weT_SJ~pc#}qP4Th71S#nqWiEVtycX)8c3uSjN7um{XXYokm@jLH^xtj`5EE(?!Uv7;zd+~A z3AV(!96}^??6h$o%M7i6Z|!3C7j41-A|xFyy~SnYbi1M8>-y8#J~zn|&ocX?p&lmr zmtTa{Zt*go)L`!JfGGo+a5lhby>avxTsY%*cRV*XyHKK>mB1;`mSPwC@s&Y*mtS59YnE{B|~8oPr_%?`uj}}d_;%jBy!9O?QV$zS%Tt5TZR+#MYens z=+hT(n}Mk|Totz2!eJ0dUXvE}Lhz;;Gl1vF058h2uwBAsIJUUZ$&!ukgEi~S&mYTW z9ST~JMV4!lkkx_T1_%86s@aEE<^lfi0rW5sK_DpR5}x6VzY*+lZiP6AVg0CsNR*jq z=#H9zb*N{gkj3ln3i^`JrT0p^=3>MaFmUZ4BKd~Q4Qx#4A-j+0wfs#p4nUBO<(PsQ zJF3_|7YGtk4S9WaL6N(b7BUF`ZG~#6G}H`kF9&PXRF#GruGGXy@K{lbl#0Z4K7{(D zehS|)NQVBUZ&ZKn+R(v2U-JK_tA>LUYe~Gkj+yTOBNbwD?Q1$BevSe!4h5U2N|z1I ztk{|9foC^pXc#*q1WZW!&wabp0rfO$9zXitI->k?=!=|4)!ff3P7WYpUknJ5`l6wx zYOF|9*{PUowCG0S+}!#%OD@gMr=1a#omVU(W9KbSb^l>TrbmIC1Yv)7RR3baejAfP zk3GiiWjr?yx@4UeyF^86&w_6J-Z1QL3{y2Oj~>6}gjDtBeHJS0->y8pGN7w2R8_o5 zqz$kn2BGG0w18Sz=Mc78t9 z05W<1liwQ@cc7h0;JMyGi&6!fDIePkgp{*wK)dQMHO$o9pZ4EI6|ksKnyj{3A%zaI zb^D|^vgPT~U`(9-`~e*Rb^ZW?GM3Xi8TLg%8yFa(m!AzQ8t}@f<0R6m854t&8gbOX z{VNifitzfSUK%c0c4TbuEwaqi^lJ@;oCd>PFce8B2L++>`d>wWv>Sb$+j7oQ*V^y z=Wpmo1`E591zAh;5> zD{$wTp_%vK4h?(jbJaJN(dxbKe-WiCi#nf@1IqFRIlzhj!$3lTacMftJ(6A`UPj_2 zW(edssAHkr;ySWOu-%ivXlVFFDGh&AODhH4%T-UE{Lin``8C*9H#bi3Dh<_x?mebb zYE?<14^CCg-0i>>O3{o-P=3z z0RY4O&t1!}NJ2>lj2-qBwN{|ShYL||kCr(WAHHMqxwQQ8=FK2i&y46-nrG?I6{;e| zk^{>BW{^-c*I47)Z6LM9Ybv4$+-Be;cvExBul~O^+ze#dK%q{zurqTeCJOwYf_flA zWUg*O0ev>wq9R5{hjTOP6agZ)J~2o?{rC(p;<}N-qYX$aKmZ?uuW|bT zilO8?vlDUT7s7Mp%OKYz4B3?ekOL)T;5KXhJt!W&M;~Y!vk`F?^ z<{8e-WFGV6KYU8WhH1w}uAZw!{B|Wk+VrP>l1}F$Xn;8n{!?fF;Hm$w+k5q( zCISooAD#jskMr51-v7r1gW9^JfWyV;%#n>&gDfNz;eZXIGE9ua@f&R z!5dh=S2+$;>o)98q^=)ngU~nEv=;Wf+EBiJLA&}b8!FU0v%{0)K9KK8;9aqNvF%Iy zKsstw8@JV|UCA%|SDsv!`+Uoy9z~XLT|2}bSZ?$FkHSANA$sQC6LkM7f3ie95st&D7by7VhQwg`)4i$TcFgOvD#^k1GIpY`vqnaj*xPcHW~3K>fd2=34~c3iRI zaQ992aeW4EsGWv+utrNJPoX$bL=K8o%nG7-)2R6C!shJbjy%(@K(o_pWJw}V8}!6v z{8l7%HNSu1Js19bY%A@Ua84o#hD0{f1(3-5mwWU{t|JS~L+8X%{#x7^nLG*G!|ABU z{Cg+oc=Ww2**p4(MM{%Z)Zw)&TY?SZYm&va9{FpMgoc|#z;Ileo11O>VsfWdsnj?& z+Vk?5>dSfI(34k`-QI@ogwPVr%@d+x>89^t48YRZ)lAq5Ocz^4Q9;dW?F1)oLSXG+gjd4b7>~Nu>1i!r!0L%zncS$)v0OaxG_?6 z2r*TZ57xvWXRAxGYN)qp+O7hJ_eHbK5UaOOnMOGXqQ)He3O^kscpQg6Wwr8wzZREgfhDc9-j*HfVDuZ}V13 zZVbF)-P7fhD1^&^D}Erhtz!q}+a6*@E$e?R?t%@DYB49ILC||uX-vDFgZ<*{3t}r~ zwm9cX&717O(5h5YC&j&%*W{5IWsDXbR*r+E&I#MZz{mN=c5b|g`HCsv+5AIF<0`=c z6B8k-swUHZbiz~T!&#n)SFS6stab1I8ETiIwET=ddelNm@A@-ZseFuJkQwPXz~vFV zu)#83YbhRfkdk*uLCH7p#P-F0Yy3^-9TuCb`Rx6)5_{9i66e7Ig$2!Yt370A0i9EC zR<+nhjDLYlC-hw~VMGPh^wR_9GfGHU@zS3WJ#|y!vBv0fZ{y5{FGs>6{Dk4q`Sc*= z4@%npcEE*y_y^yUW9*GJZvvFSm-bZ`$*(Z4IO2rZ`MKxFZ{f2NZ|9i%rHqXv{uPx< znaO-B;)K_X?-%I1E{Yg#fAX=|jw|3v`W<5dZ?Co~=sRhC>qRv_BBcK6-h^q3Pxp&;ouZUS57Nh9;NM~t>wy&7p6IvdC`bIjUE zV>d3@SDfJDS^|U)YF=~>MZt7DqOtERZKteb^2k<}BqXv6yf@{C+*d}OinAo`*Y`F; ziE~qi+acxl%;|kmhWE;Y3Q~K8b=cUvW5QOP+56Dns%si+8K)Q{5?&aGf33QGmgzuE zrd4`_oDOv9I!I#Oq<1PI&Yx!Zjyu1TqpET|x}xg%mZ-9`rTA_7xY80Y2N8$8TTh6) zcfQzT!>?q?IDNo2%~B3)7&VPbHmGkoP<5Pgty6Mg#F zE({9YBt*%S!Dc-w50_U0*ya6EfD>jnvgJ!I3=|mU#i*w*xjCIS^|xrpx4N!P`MVT-(0X-@U0w`{ zsMfee!zmn7^Kbf@exV$%|rP7?a4I}bgo=L8H#nzosvZ#~PqvCPCxjMMYrC15 zM^)ncBRUpWLk}qhbsA~y6HDxZ?zvGC-}%AZ@Xe2(Ur(fud$dKKEAQAV7MXUN6W56L zk-k>|8}RURe4bU);m1Cnk1NL&mv&~4}zO;KL0_oUOO0k}Hqw%kc8kB&T zfaPg9o+IxI$$1X$y(*rw__3>MRDG|GGM8d&abb~ZZvo*w;|fVSCDvi#82D=s(|c_! zG2#BoMMj&x%VULlG!^MNtgJ9Us~0Z|SXCj3HlBCdA4xo=f9d5p)}bk{I^zpB3$qp% z2M55;e2LFZJDR~K%mVPup~eG|VvgFFH_$y{6Clw^CVb7#wAu?l3)K!2{snT&Tl3xa z-!%t7x=)5J*70Rr-x1Q$(ec#dp%KNlJ?3f?SlVeBURu{Z*%4G za>)GEDGiRs1y~gIK*oGm!0N_Rd5EIO^7dB)BWIcCaVm+r7hbmqDEGBx#kMicg7x)9 za|jPdkYF17DUp}v6h&gh2a@{6`f>L-r@1>FyJ{^QUWzN1JDvC@3cPla5FkZd;j) z$#axiejPPwVxRckXC+sNaQ>GpdJ%Kt9+9*puIGXt_N#d}{+PegVYxU!K4TY1HwMT7 zrq;@umMG5T)DltnN0^TlyPwA7e}?LQYVEWlQT1=>1s${W7xA{tN3yin=dDZD6JrW( zk31_-$ugf@a?9@-_q-gS=r||!<+YdqA2ICMNx2eX27V)-H6gjH_M6s$Vhd7>e3x=6GxWmJthq2o=v@I)CLS@gO zAMZUHXetGS6@faK=pt}*5y3onW}|`AQB3oE;sE9Dr0Gt@RO!<`PV5V!W?37!8@fY`46$U$j#mUS6K~EcniMm0VSla*?kuainhBi(fT+ zJf5rYJy4SZ66=GJSoa1NQvI5-uZg!_Fc`5P_EL*PFrQDNX&&Oyea-zDbX6Vl9dQ|% zwj_%?S*+eU1T)(Kud)?9a|2H>Ka+eNG59;SM#ix=0zX*!l#+$SelEouun4yJ^$8jc z`O7~4GGmBKbY_DoyW}Cu!b8(}r>BIr6y!_XC|(7s^%h=S=~BsTXjxVqF4ym_u?cZ_ zj?=ocMW862!iCa}ghd&;e?lkeKdzSZ4Shs5J^LfIQgLNR!GjV%Sh((g!|EpfaUl7q zKAHxrVp16j|88@V!9dBJf*0@EmySVhISY2~6PW!QwM<=GEz zZR|y7U zr#B_#(@qE@oY9O0(9PL4Z}Ni`Yvwn z=k-wXdZv))m1T0=H6pH%Fl*3z)drAERz`y^&TolA^~@4oH)Rtg8fDWy^`?@ybxLHa zZeNaau%y~E)K4*>B85jQkGAmH8)>m0dIMXti&5~_v^yi#by%Lb(|Cgc!M6v>{?9#f z?QbrW9**z?iGJV9x)Z{AA$LIgj1&G&W4H@BRkkF7@DIkfW?6GFdPD1($rt zW9}Y$&Pz-{>0Jtf5m4tA`E6)hK(G+$3>i?Rv{9)Y;Y}8LIY4VF&#IFDlIhD0fPR5a zXUEmM(SHVDm0|&IrRxzr4}G{}gYs`BKQf5_0rDy?Gr@WWW1TzFNMf3NLt?^zfl54- zMWtpfRX#a67CvVfd2DGna*OMI?-%)6m#4-|CWB;Us2u;hKpGj!jx+yC;;V!e(e?Q& z@SmhEz&<8Ffb(#JJP!VC=7xHdmr~~3E8e=8_e*QAZsI)vej`r9w-PoBzkUcFL@_SK zkfzfiKZ>VYGvG1)Mfgg`W2W}`@w%<%$W3&PdUCBNVQ76B`8BNVQMRHL;^bb-Erbdn z^Sbv7q!~j<04fr!{dTmu1TA%G?7|`TW#;jgq`+nwl#HmDJY)tmCAT4Ff=?g-RRsY1 z{&a8_A4Sw?gMWlOJcF;U0)c<$z7b@&5&h&BXcS=maKrJE(BBUJb_J^KNGbw>06yV> zJ)L`WtQX(H-kKPv6#35Ok`^{jh*|EUNp$8*&S+W0)8G2|dym(MUG0D>aj~%%S|J48 z-^ajzdTWFPf@7#)*EWt%iA9T1W=a?G%{Jz3w6CyMG7a&gBZMfPB~DkNWi99*ZbIai z*Z}o271~dO+PkHVRDHw=2{9G8FVlTL(dS%n&)KNy-rcR(lAnX49sj*S~(|7H^jnXIcX(sT=2@bj^ zJsxp@z-dDd0oF*DIk7+H&)>H`JMRTT2rd7~9PG#dIC^Inkf_!y>KEv1Hc;m2(=SkN z2N5uUBadn{6@bHIA~v*U&l4aZag40mq86BXQv+}rg+&k_TV22G7xN>mN_MM$Lo%X zbOUD!&^>(syGJOBHKE$n1wJ@i&j#kC#4mlv8M#H-8CFgb-~MxJ!NC7+kiq}SVEhja z(olwH$XRq-#Mlrv6&A)b2i{J$@*EJj7-(W5#|Hq7wP}(OM^t~Bu`2UOSDf|&@SY|u zyF^7tDWz63TI62!6Mt*P@FzKy;IXDxXGf_*N1TvRcS+uSJl1@LH)X-7vYfz8ROAH~ z&82B?@h6Gn6e$Jxcj0pzJH(Tg0S~Dq_h=oy;8!4BU0w2XxgheHzpP%-G=-8_U|Q$T zQo)#S@D0G<#5JC!QbF3gW?O(WQ;7YLH`D-Df=?0sy7`}4{JFK@-)yYotmHM2YBoI> z*lp~y{iojH|7ETHvnzlnCFXwuD8ej+JSE@@?6-P(PewgD`Chx3;)y--HS>1K9X~J7C8ynA_~=icp&dH zPvC=|zJvb+a7P>S>}Lg!V#KA8TY=7RbNCXM6;p@I0Pjs6L(G^z0o?FkT-bjdpmGko z!F{-UF&l?c9A5Iiyni|skU{#tPOFHz1s7jv`A=F&yLvcKogV$+cu$w9RtQW1W;vGq zYXwj;!MfYnrRL+n>lk==Xy&)64Sge%xPXr?cWCK(!|=tmT-vi-+D#Jdf4(`zoHJZV2)D|y)4i$dz6Z0s_-$s8+X;M>Qs*&t7D_08Hqee z9?8_Yr;}>}|JV0d(&7p@`=dav%JD-u-DcYDnljpeQ2wVz z9h9HmJ9<8@4C3$AU!LXJBYAddW^x+>5MKTdFC@Woup0{9BdG$sDu5vk(v|@b`=9QA z1E>`H3xrcRIN`4Ys!KY3=&F4&V?`2ab^7#T{lA-S(y%NhZgq{1DN|UCOnn8B|K=%~ z>_ZAQ>#J3qH&f+7T5)6X2`6EjPsdY5V~uk!zPguNFmDrsZ_LZw0v`BYhaw~j5%ov5 zfb%;|mgxf@!?hpM*%r*d_^ru_EI?mR9cs94F8<~)<)h?#4`-E8t$E(_g1C4+iqIIHlbgS|AQ2E>5kP@b}&&eD5}Z)2Mwd z>$2vp_a6aHoEySt2m zXgp%YAD%Y;Z8U*;lN6fdiugM?gWM)ykF6iocqTFblx%puv>^&h+5#n&zW0To7CI|! znAvoe&fkU{tTO&+dGh?jcjCpe*#`OLF)dbenXdlJ8NEY63(K|&J zXH8j)D>Dej%Rt_xe08cA?Aq(QWl4|~ztC~xBgfblAk+G9Ms?l-Fe-z}ccLk_jyr+O z_S(phA=nq5iqS98_i5!L5A~u0-b801jv7?2);BM;MO2SF`&Iq%J0U zy!%%JITuCm_cxRNtA)I}HKgD07mHT>ztX}$Y>t0eP+Hdmy_PX*{uKEmS!1i`t9@2R zC*w8`l&b=zT`uF(ey>e6E~@xgG)E!Z%toDmub>t(=@pRl>8rf;8c}#ywC5epaa;7` zT$|ytcry24>G$)&%jG4vK45=DmORKy_KJ3Q)K+L1yHSRsof`3ri~OvCy6d?A7c*u>bcGX`=lc7>hC-3XepA;Dr@+3QqyGs52}racZ(+SM|} zm8q6uxg^;VYs$>}DjmMB9mg3(Mn<6^G$8?dOYCb$?{~rtq*aykUCY$Aj*J`S#|}hp z72-7*?7SMdQskmhdd3r8PKk~jds5o3<13aks4FYjR*{OCxE>{xfFQcEeB9EH-sZ!c zZPIx64_d0k;noc8j&s7TN10P{HDtl9S<^WXuCxK0vV#DywpK*vNS#}zeX{6L9%Wip z;y8TMvTD$IA+>Syv*4vTbKE&381s2;9s`8fP606{c*c}vAM*+EUm*M0&o*0g8mX0M zMIVndr&4ag02LvBTWj)lGLjW|PcZ;XFFmh}Pc>9P(jXrJZQ$HzWDm2{#H{FLCr7lPW9 zv+`7prxiy6Qzef?fe~qBu=6zI3X_XN>A@x|fK3*TJyp%v*qUVvAw+|TWj#aMo#9=7 zWw9{!;=;=BJF|DD9{lvoA?0=Kd3?j!;r@XAgDgQ*@SUc(1n7C4}#$EvB zU8b|GN%)4$nzYyBPVV13Ulo!So?#LF^}~l}Kvp#LE6%UyipO$1++9_K-+Z|j>17X!4w^wXwYHrdnVdt5jBu}?yD ztZv!Jrt8rk)zdvC`BzL2b2@mFkqjfQb`iC;itgiVN@E+X)5w=wtc6trRT@3h9=|c1 zyZt^tGUUqIy^ZDu)W4}v1xl+i;f`D8sScJKuna|x*~WLLqNM!$6`$q{_XfTiYZuPr zW`a~a$;v?RTKNMnC+EX)et_VyvCKJKWPs7z#sE{CE&k5Q$+B#Gi@WU8qL`;W*X5ge ze1>dJ?ZR1}qDN=0X%~S_9B8=mUCem3W4;Kddo0h-1hPB@l2^y=kb)eo&+Dpj8Foj$ z=dGEGZEL>(f)e4sD>csjO|kJ`pWprOdlj0%IBv&ig>{!!j>bLUuOL}5W|eyJVgNpm zY+gP4>{+zYgjxPp;#}zcO-;pFtb0jISp+#lmE>?|J0Xd^F|d*{=BR8g!?*)=;Fs#| z*rj)b^WDWNqW1xLmXiTiugd>E8s8^xaZTig=U0jIiRAaT`+RH&0!7Zm0D$y>2i{x= zroPZtewig5Okk!X?3$X?c?e5ne3+gDfdp|CSi2g)_V*XDufQrt5b5{tBmH$Hhjffp zW~FA+<>S$q1lw-iVW%@cLyq|cK{~a?S(@V2kQ>SOBkyqbkA!H)rltFq6XX~9#hZ<; zel3DAx=h#^`BS-hcnElQu$(iDzAt?>IDV?S@lB2CTmt%2W?g06xUxX9yz++v$Ejcm znT7GBR!30}1j*(&tu4jk$6|+{5o+`~27y!AO18lz{m5qw zB!j{ve9@6j#uq|d^ni4FhXshq)BWagIE7Q)jioE=a7?`Yc<|#yIMXE?N}p0Yt`F+N z7K&i!DEi(M6VSjbs?tr;_Br8F49B~xIl1yKqIH!=MlX%6^_+z_HXycL=U2W zAp@mR1)F=*`AeOJ5jodNCN@uXRMFc`IwN(8XURznUl%@Z>5RaKEMx9lD^-zHgRmxb z=^zgnx8H-e6ugab7oiaqkz%qb>_Pklq4>A~gYFEq-tsf3Zqz@9j9 z?qQIz(^50Mj0ww=jlSd$TYxEI98(c*=cumOYqxkfXK|0s#!niStF((vvT1$S*}ob7 z;k?QdxW(Pk%S z{>Ta~hWC)r{?P$#7@lWxljH6n5&#py8jXx;+*t08x8G{Kv+*xdAGGyW(DaL^OR~>f z$3DN2z^)rnaPA@5xQ>2n9$w75`MAxT3gNGt<6Wky29Y1yD%|mh@&t<{bGMa8+X(js zvY<%=mm?Fk7-Ro1!cV7iM{hCifigJ#JX<&0nz=^$-tnz*88ES5T(H#Yy|pLjum;N-Wuqg6!f%SNt$tjpI>8{ zy|4!CO6YZb@|-6(AAVVWh>hb{R8|9S#n%1pCG65wJ*hSNv7Z-m7ufqTsXY{9U{t}f~sum?_ zhve@V`gewkj(!$Xk2S%3Q$C!P2MoA>1Il@E0~dDT($KCVc=F_YWvh6HwR4(a!uo6JKc(}NqNa54Hw zoadCxyF3c~yGB~?xDF-aM52$AmVD^kUN?Frc{Hk*vnEohR$sV+LG)DRm5@KFSD`tM zu?0w?6t*=*m~t#nj+8og;>E))Tuk=l{_}{rLNI6w#sYuse;>BU+%SS@`8KVX+gBz2 z$@Qit#LA;5^nTGsf9eT6-a``WdD^CKZ2XA;y?Q2H>jUV)jM#vB@4r1@$p+@`j|--> zC1;$KzRbIKGr#j8=p~3gSmVo0@c# zND;Vq$5s6@TY7UHmC4Y`peQU=W}!fp;#q0hrR&siYI(oP7Q_i{IpTB$e!yuqt~$W> z1TFLKP%$LURh9B`OTiiQFrcM7vT6OhnC`bv@1!RPUNzv16I| ze;WJFsHVDgT@(eR3eu|z0!mRpK)Q-F5fGGKqSA?>cLGt7-UI{$lp?(Zq=Z1|gd$CP zFCjrXfzX45e0P24?0xUp`|dl={lQ?YKgk+%t<3qp@6!TazD%|n1pN*?=?I$HZ4!oi z$o%S+`Cc1q7RK4u`)A7DO1>R)%4%>9ZQZxn0UwIfxzXvh{3I>k(|NObYaxu)(7GoJ z5a9jCfI_y@tuq&JT3}lDRwGw|17ZyRDtb|wRLU(S$Qzi1$vLbXE+ws=SE>DO1p^7y zeN%wWa=Ees`!P!aIePa%E8?r%kFAN2LgaQ&uG1e-EJ>uYXZcJ*$t>gdTyIXM^RsF( zg@isy)fY{#!~MDVx&UF@!#m>QvUqj0yb)xkWW!@lrPJ({ z{)h*Cl_wSBZ{=&Tyv&H8b1h&3B^t} ze8`C%*(>*3uL4$O_QzZwukcMoTfCvJPl@2qotQ(Uc^_yASm#OBe`$udwGzO`-;KN!Q@#C@NM&TnGk+K`dgDUM%+q?7ot&$fAf;Y`2@vzBIV%os%42f+q{ zLpI!5QDPDyS-E1JI)1ap$wo~5$~<$WEemxDjg4tQpD1;+A?Ut2c}AXN5U~CZ5!f6h zIzApd0?+ALLyl;cQD6pal8$|qu2W~B{PdgpuLSu!U*8u9;BPPU2rw^hpr+74e5wq0 z4s;WYQFVJFv&u1$$4{aswUf8|SNkgL4s36+UA&m|%#wm~-9ikl%1R#N0I`wGKAkNk)1b5yM67aUB~I7QI&P+K<90Wqf)VU7`8+>bD3@TBV3uQ z1mh1gXeJM~?lHg%^X|<3lIR@x4NpYun|j`MK$%N>tt5W%x^Z5L*n%w0z9mM0ZRsidoYWk2VqbC~j+D%pvZXF6JhjswdYpwS5l zHrgNU(a!2q^3Bq0tRI)GfQ+(ydL*9ve9l#Z*HeoAakut`s#?sfdUuvk0-Bv$zT=o* zXi@^9oBs9)ueqg-L|Wd4jQN9O*y{aPIK-;rP<+#uA17^IIAmkPEiw_$%1H*}BsHc` zsEECXvrq0u1{p5bP_FVh^`6bHlopEy>Kfb2P_h*)yS)vol&+li4 zXY7>ZZA{L7MnR4WQuFl=>s)(RYE+^vD>HtcDM#&aE?s;dKYZcPhiwfQtcM6&k&Fm~ zlLY|+5%>!H&g`V63bkqGbGZR#&Mr?8L|y74R`+#I()Hf%ZI*|Jv=nP0yzBc!7IHPN&zY93WgQKc5_0F_T+K@1jb}pzsN;=|VPG>I#{Ik3l&a@m4=HZ86*z1T z3IPb05t9E&IKY&-H#j1ke;#l`@f?pda#TJ7A>?L0o z@WORJ8l7%@SxIwx@Ss~$i<&=$Jp!kvg+>}#nOSk>e}X{YwYTI<#;P$wwLEB_X4r1~ z&^P95JW%bbz70U^B0-aT9T*aJIE^ zB-ZO%VJBlC=IZB~Kh;7~1h7pp0IRQX{EOnT>~GM&eULF;_3G&A2vq*Hy*ovgC;_Yg zP$)T;EW2ZO2!7mDWF%qjV}MfT-D&3j;{W(_a?Sdb!&{)XIa=%fmzw6E=#f8QF?!19 z3XC;je`qTGD@BM2OV*BNdxDG8v*(3Z1PxOXUzSjEUQe{)_t5uo-*{o_X9*pG-72Fs zOWoAD<=FZYcOS#)@<%gWGTYgI#G?0W#jy5}6|bG+?Dt`)L5aVM4#=QkUwN1^F86(p zKFy}IOA9>TdfhL4^3S!)Z5N{1sYAzd;VNov7UU8~zKOc8`;V^oN%>eMSg|lb?Lw+o z^B7&7{w%-}j}QKozgkhXF%LBViy~dKr;|H8yNpktF4c1K&7jz;ogu5xDOvW)avLym z&@Wgj%a_9SIt9f#1>a?0dfid8clZ3_9GTqn7e$vLfCvwDwjR>Cot+5)VUniXQm)GJ zEOo~<^I?(gC0-@nXEs}|J^4^TY?k;0AhGrXcx1(D11v(34HMBMS2tBG_^rU1mWtZx zL7R}rEGqhp#A;73`=Hl+nzY3yI>csV=%@DsC>Lt6|Nbuj^I8>lj!bnU8kGqtMWmEI z{MF}y@cr39`Y8|xZ9Fqsg~g05R#+^a=mYz|{-`Eom`WfzyRY?O6KttbATo<#0;*7Q zQNj%?m1daCbCLGJi3u%w15j3=wN#l$wgG>WP&7AUAwFLDn@AT~pdEI->ZY)>suoD& zt#mxan(T==;~lBkH;B(xwVUJc(p^VBQwQj(qoxt$(06JO1hf2-UGo*jv_mpq`Y9>U zHMqwtZj8pCjk`DaNosGSE^!?YWFI7T(+j*B2x`I6jYFNEi006sS zv0l980Tbkj*-GKnm7!pi`87;oxGl3-saQ$VQLq~@BXlyd(CBS>D_Ez0qqk2Ekr#xh zo3yEHSO-c?DDz?NzjZzTd&Fop^bvrV)sRij=b3B4$TJHyyn&2T$k;KRUZdK~zUmNh zry`>v`^u)G&vEmz_S^E1UN6Tot;>f1(IBBDE-e3pLC5Wz3FMhc&qdYNJSF#&V;?vy z41(<8JFA==SYHj3yy0JsDWKB(PuSut+#-nI{c=yRA6 zMX(HDw4y_(MBEBIKF#yQS9${lUr3oKuUvI=+If{Ctk+?F;Q~SZsz0vWUkl6hvFQq9 z1bamOdDMNg9o&z>fln>-RpO18qa-foduY;k7<)pN>MOC_9HuNN)7kA!K*OX7muy{*BI#l!9?8kwlv@biE;es?K)X@^KPbNGCogj;xQ zz#1haS-7Mvq+ZWSkop>v#X;1-s$K5Q@6C7CoP|g(?o7@)He{I7F6iR=QDrCEY^oS3+%MQaQqdnWLm6jiF!_LwVKqM~+&+uUNRW*?vhRNK zoY`E@VNwF#70RVbna+3Sn;-f+pIckHC|YbGm-Q1}b>uIL1G=o6BAx?X%NmehJqmC1 zyK@PZ9e)Z|;XXbEr!|{}or`L-xA>iFk_YORx~=3@1}p+AF*DD!e)QW;sMAfl}8JFvbtiKYZ>O{Qx?q- zrGXSuubnRYmpRSt(>J}Gx67>+zBhnlw0Ozlr!(wohXx# zL>a858kc<3HXgyCWSH!nv4}Qns{2uIANiOYEV^RpY^GV9nI-q&3D+qHZ)(Dsn7dnL-B4w?OYQrV!;iDOpc z+xbwUSGjd259|I+)5b^XUoWm-ZEXxU>3;>S|D{cB^iCP6cb*=?w-IPl_L@DpSRa+o zo_Zp0ids1zoUt_Lw=_zV{xzmKq8$5keY1CeDXuI&6PQcVH33ob--K6ZWW^c>C1*E( zO&*v`?8?%Gh2ymI)4469tT7Uo%ocT?zmufW_}hs2t2$se#@}~>p7Lc^OilF2HMiWT zbIadC9$VsHDTMR9r{9z`yz*B3tY_daip95sZUnIoU-{XEiNvWx4F0|REWGwFifb;u zlci&-r*`2qJM5xDj{RIWUj`4XDL4$}0GY?F@5tQiY7B6<+>x8jcY`0)H#Sbliha}E z#CF`d9Tbx;l=9Bln4*rAo@^Vc zg~d3`pH1mm4Waq8^e0t${Pq8$(5?bp(p+mZb+)&?vTHhq^Rxs=m$e1LyeCrbV!Ia- z=i*Ua#zxJj1ud0+jk}n?O%BKZuey#F=iN#nlqEhc3x8zta4fa%rr(lmtke4&e^F$WQ^Q{W znQ|&+dtBVcOL5z&{Psu~hX`Bwx0w->=BIs9nWJ}p0{1t+_Rq8V|Mg#?k`Jm)SVQZn zE}nIwk~UJ2ET>XGM|6bO+8=b>R=H3q3|PbIL1m^?`G`e&ReMjo78jjOO$bjUB&kmM z*^RBfv~dYBf@h)XSUrxex~e&FhDLmq8_#JJ+Y#zn>9n9bJtO0lwZ$K5*rz0*rOy>W zw-Utwf?ibR3%VE7dYZeGPu*0)oiGL`a}(fBhVXbJ^LDtAAIi0B1yg!_a|@DCan8Hbx4Q~d6!wFani48c6J12W0pbXd{8Xcu|3&zYmIX>{}1+6zS0y7)ASkE87uQ{;jh5BDEWn0 zSNvgXxpfiBR`Sa697K+_AfW!dak8zC%S&O`o_?L!{fPggi680aANli(;xGc2T1WF+ zx!cb}V@#DZ#;tX%Wn{3Gq$tIII@hUa(M!!esZ!_eVu`(T^Bu+U#rz7|q_xa6uR^iZ z+X6RrL}Kp))jo$4i0*KgrAbh4N-{6Blb;)`bEx-Nr>QL?Bv0J=@_<+0K)bv_Us*sd z#Zx;D{_8)+pc=3xPO|9G(iwBEjRAF3vPrPMIc_{J=^SE2Ib5if&l=`V{*8XNQ zq~&`~-s+W1AaMx+QFkPXWQc_O2nblx{N$L`=sDIN5yT*ObT2V^mNU)Ym_DCT;Lfn} zOT}<2zkrk_@WzEK+DCL;2AL9D=?^Btr%b4>rJu`2Eb|P|j(qFWNd4)~mhw={swm8n2yiSJ zZZWKU_b==oT{fOr8uzxGlL!n}r1Obhn*BMI=I8dRUp(qkbe+3MT5JCz#SL=T_jju5 zg!zo}Y)V%f<~>8GdylTFEJvsX#|7mh@(6k0TF7s{mJqd?hB;*aT7MaezX7L4+Z^9m z*uKZTi!IDvrdbq~)m1S%FqLB}7-H?huQI??>YI~pvcGgNDi{xN2@9zNzC(PkumsmX zM7{oJa9i!_&Tg{t9xH*U$OB`Z?k`psE)tnWZ*sp99>~9;jHoxiDcW{{^Bi}?ogL;Q zz(~wowq8uIUydtChpnUS%{3|QoL$a-K8iq4~D(nE&GA3(q4m0HFQc4p$0QA;FeEO7AC6PIx- z-6_P>3VzLcUf z;oj2O1!96;YebhtlL^5BNxIb{sCbk!ULb7Uhw`n^LHKaaL(*}Am*BEqM%^YK*agsJpb9=X&XB>% z!NNJ6V)T!@X`XLh+*$HQr*Z<|NOrq`dL?I4=eUkgL)NdYkc86hKfql2U$GJDyUU}}E>*5cF2l948}irk<_w0Cg>ByEM~@7!-j+(iSpBNmQ|lyOM-uaJ z!2;dMMCqP356il5fUy_+4K4^Qa@XE-!UpTtOcF+?|mIlvi zy(cSfGL>l+x~THhBdzZ!KPeM;oWDEqI8xxm`-_i+@K04PM^mA-bL|n?~cI z>Ov@kcX$$B9t{S&)=lYq|6FxL^u2M8b=lz!FyP`09_V{qOg#c%5!#~;^(*oj{|nL|d$u9nWpHwPKCo$Ydi{CR=4 z+OG)Y{HIAi?P{BkW{o@m{SGBcwAJ__&7WT7eDXLcKkT{hJDP1Zr`?aU6|Ib6_Iv8> zX&LiV0(LQa$i2&X^G#iKr@EZU9>MbvN(EdqOH)SQ3cv$&8WNb-qO;QB2=z8+XWRU3 z?b0c_-1>B{H8h(=Rf42RSjdj5bUQkyMst%xeTo-|F{U6U1{w8CrzEPXtGYsXmXJMO z8?ic87TZ^C?}CFgpfR{$aAzxb2p2KP2DKNSiv>48oxfZ132N5XQS0BtU8M`InD{NZ zs!?y<#{qsUm$J~h1@@oi6pUL5Sa^9m^&zDKbv+d|*O1^Oa;sBSkvL{Fc4N3!fN^tC ztY&JZD*Gkajb_eS0+5AiCfu*ytgq8HTzOV}(?qf$#5i!Zg@^QHueIx&dIxu&f9kTv zD5edi7G3Gx%>Qd*@p#AaOOw)SywML-;b^fsAyep?+!9j(5OhUi*pum&u57PlwT5aW zcV1aOLN%iZr8DMq{wdyPF^QM+6CVp-Vg2x6x^KVghIIddm>n>1nCfbcvAGs%(d@=L zLv-vU3p6z~PPlTQJH~X=KPl4{#smm|`|P0o#YQNuBe^Q{HpQR){e6snP1eqiKOXWb zt{^qw>rKsQ>45=%3JM*OLa_)!CH^v@u_y31w|sbu5K*gBu@f6>5t33dyI`fwrYmK( z{Hdz#(pN?bS*>Ho(Vbe@J2fV3HUwZ+QX0$#JUU z8~jXDM{z)aNT*z&L}8|5>GYP(U2DqV^Xas;s;K*QDSFKvoD+-~1#_~g=Sw>IPg{Rc zFe@7q)H^+iyCy$U&#cCag4OhDtIf*y?7cGOu36M(<}uf@u-`!5FZ!*d{DY}@{XcN3 z`u5bKYYi@~`zcOEB4QXQH4(Iqb;ua6%hvk}v<(gvcKh0V8&DbCH;(QMzKR+w z-Ue+mvh)6)3%7lrmE`~>+byBT#y=kM^5;x3?e(;A*eF*K#xvv)MeBsB#s+yOfJ_hG(8)SZ}Q^hE22 z*YtXuokm^UFhcS8K8|mBHP5*Ft)1tgihOq~YZC`yrktZ|QF0h6KDFKNAPd*$k@4(D zx+e_SFKVv*l8Hl4qTD^D71)+QJhf-L5g~V`$P(*D>!|razlh64(NLLk>Aa!B$F3q; zb=ld%&zfd**z&%J8rh1q)Sr9dv|@ZB$dhlvPx?AYytcT$g#me~)($=WlEiueG_SEKb zuz#+UMDcr~lz8^CX%7Zupu}D4k#D^gf(gcb9IgCLI5RU>O?j=Hk&;}QB&fY(XLeKC!1Tkc`;MsK{ij zDzY-yXG)P~d$6fOb3)bBLuOq7SWL$;XRsQ_a|fOfwS{gqi=!VEA7YLhv-Ipuwmu!g zsZA4XMkU;qjbfzNoYyY#GM|PTO#E(*&}~fCtwT1ArwYqWo}@9j(I?rim?j!Mj@FlM z=Y0QbY&1?#H$r0!vYj)OLW{q2q|&SQBH2`ID5W7=vgf6neee(m0n}Z|i4lqNVd5Z) zJR3QbpxW0{#DVBH)Bc=C3?c%9KVB(yBAjP;L2#j9U|h;nnD*-9 z3=gOV?OI(Q7l5g+Ve$H!WRdS^UK}zKa(R?w{TiE@IcP2mo+~bYGl#`%;3rqLp+_Hd z!`^kVez?0RtyFgNgPAB*!!BFMWETAhR3ret@oCEfGgI;Q&Ds4x#sZ!phQYQd*H={2 z>DX>ubtkv{jhAxPUaK4+poS_A84LVX=7$xi^EWV@80g;LT*>Ma^R3c#tdEr}|E`oR zu1G~5aO`K3fe0Ip`->c_B(yD3PBbYL>Mg}duHdQ?3?>iVE=$C1@#u^?%{})93=E%% zk7%{N3Ka?9@ngv0-zxPVGZ-NOWaulr$A4-wU@5$ddKC!)9zlq{Faf)G;cWML=@y%n zHFwSjD?@ndB@gT)923$7@?Rdt z^PpRVfYAX?On{{yrz@`SH^7H^9RrMjLpze91T#vC$|b%~H( z&5a-f0&;ep67B41?odQnSrR*V#q3J;vog|@s@eJ6Ulbly!&XouLUVawev(Vs%0|Yw ztOW0V`dF6DEug}4jJ?`=4~Cc{^{4PV3capcBi|(oF5hLgc1bg_Q5rogOJH&OvECvC zIP&`xnop|T5172ZkWk%V8KrqkUhIT%41pY-Xmx0|&~w=5Wd_Fe*Iq z@kBKw3&Liklqg;LWN*5I_Dg!GedyA54#V!XkkLhs5t96}#0&kJsiQ)`^nYO-P95QM ztEN84S9Gt-Um{kT`TFd$w`Y$VnGu|EDjdrWiKUTu9mVq2;W;{$269zw3vm-ohP~CA zNmuY=cCYMq9qHX^9vAs|qymaplM)Y{0T&45pQ$tE-O9Xy3XAr=*AaX2y)2 zodQ>iVTvYBpRG!N`8bEbm|3`yX7O#ZvIwydy(yzE65(_6PjJJVET5$_Rme^8geb{N zF6~_J@8ri7Lz!^dSD|uyOkpH2PEQPQv3R@o!&+T^#BMBAH2-6_uEwp2?zKxbcGB?= zXQh(SF0B+KuVPCKjp_|&c6sid81^t6mmmRMtw*nEA8#oO?tm2R?+6*$%hPC^Oy4tp zrwC($x{mFHT=&mJTObz7U!dU?nR)7ttV#V|G-DEXZ)NB;{)n6s;?g!7zyDN!l_wGA z+!`S;ZBbaG+Tdkduz_)KuBbeWnKMpOiu!m@k13t@l+YB*s>e<|rS?@QXSjQ8KISdJ zzr}xh6yaM{KaruNwB=KGDp_0b16g~+X2o?AUe6Q zw`BUNcro}*Gg&>#_L9O)d1kXkv?u$qeZyTRA*bY&Vr zWdlWq5YdevL_fn|L&?;;3g5vOTwPyAR*4K=<)N)}f210gzX|L@)>*l(Y0T|Dv_st1 zb2<>|GKz=(1;+k!)G0pnOqVEa)Rr+bKsnY$-Nj&A>hIptP|;O!dygq%wN;Kq>-FXD zDmCaJHzAplcR!Idn;WfkALLC7iA-oe8zzEHMDF4iN67WYv2q+D?Pcqmu?1;SDlMT@ zp~SB>GJ!R`Nf(;`(X~)9y@SZcskOE6BKGJp&QsViNZz=^JkLM_vuH}S;tHB~@(Ad| zy#t_XfA`^5CmW@Mwgt0uDx|mOuI4w4r9T+lh+N19s7#8mOTmmywg%S>3o}-O#3cSm z##H|e#X0?NQ~JL#XWoP;@>Sxs?#1g>Q%xF6m5f_|Rw6ZO$B*>x&}C-2$6H_WM6v6Y z>(VoS(Ajzc&Zbl5z$LIcFW#Dj-pJWp6dZd@C-LU&*#~gHb*!~=NxE(!XDW|&yU-Ls zFY~myNz_^bfX@$nel`S91ftoKlcClkHLu{2hv%nHZQa!1D zN$YclE6wZwaxOi#9*5_3<%^nNaN43+D0=x95>&U z`PS)Tg?ZEi-XuM+;X^jt(em#TBI8NF7G$}w@r#ySQBOvn#BN6p#iqOv`gqZuu1+!I zcOH?4u&4quop^yg2}^!CNg4O_)Z6m4%6b61)-~0^QTqw29TD-IBepqU%)iq}SnCK8 zt((9fWz##&Hn;q^Vf|DLuX=lw_{8h#7wN?7>L-4VZf{MFnww01XkB8?zt^41@?iW< z>HBvSd=z}8Pem!dNB`?zDJm=75$=q%2+4E4d>%Hhcex8b4iNMY}q**8o5@b%f=o*|AeMYhngz9oW+n7R(3B`)VwZ6;77 zKI1hazWZ*qrI3wvQ@}pfI_Hh-{M>VQX?F$qQ&GxJWx6wfN7f4u*WEB_5=%G`Q}KnLuSzipRdL{zOC6%nShC!Dt$ zj5MnnT?*gd3ZjNd+dtfGcxPMx_0yGA6fmU|=xAXa2C{t$ld*6nJ;L$l_)&@Y6lcD2 z+5yi2mV2x$Izr3IrV#H;9=ezrd^sM!*7DiFPu%bhqVqlbz5T1w1I)~x|K%!c418K7 zVXX}lPG)lEc|;7xLYVXG!}oq=exI{U&AQorTfdXbTXMy+bmxJfa4q6)xDDfE?!FJx zm32R$uXJXx?Qa-+oftAtb$OD@VE5f$6sg%kizthKeIEaKK9rQ**-WIndxB3BboboC z4-;57s`?~ZhAe(W@3VcnNcl0*jOxM{M=rF0MQ%Ct20mSS&8ji0$!vp%DGb&raYc@ecKN}009(UEptOMzI%qgw%+<^Vn%8RB%+adFs^mWL3}aD`v{M_o#-qDdkl6 zD86P)et)OnMl=qg1%TXzCV<956ApkFS3)Jna7^1j)oN*GR+jNR(1Ew7AtwvV1tK3p zSOKv5YuXUthd>RadwZ zHWCphw`Mf@IyLy#=;$csY9BUTe@i3T6R&XHcBS_ac%~^p960b!Av=MP&7&b|f0+3m zV`j~^MO|(|yUIdEk7U-%im)+o655$lC~=U1kloI({C-)J;1@9zt7UNYn*E9~d)$+N5Pr_@e zZ^YBzgxE8TQ(8X>wEn6X*PY9k&y1PZ{50rYC7Sx=aTi-A*A3kUimwd#_(CAC^FLC0 zJWTl+ro0>RfhTCi#c!{@;4ZqwR)B!h8Q3iGwa7Y}4@!~RxOq7pHUC&su*Q>|qJ}Z` zYcJ^MiXg_{MxD!wN}b`Oj!NZb#A{rp!yDSG<8Q8+Nv0r9f^Y+`h+HUM^ zy685lU2BcHkuhaD`g)iOBi9YNV=kE)?>kSIV%*y;!`##CW~e(;sn-0t%9!Qp$cQYV zr?MYN5E{v&8XfA~!08mCCJxhUDI5E$BA%uy6)eY1QLUu>@cQ~h25-22tSTOkBW-3( z#s$!o(cP~ZSw%Woico2SOO)HFq$!^LiC!t>r*43Efn9ca)70jtIhrxWi87ZB)fJ}oh6wELGVG21;jP4hv9#T#U)WX-B*wDm9^ zVF&O(8q4C?opUnD4V{uBc_h2Qjeo37_e=fb_!HudRx%|rwg0=q!W+QK-O7v)FhMKF zY8d*bWhhTE$k^F+_xlK>()_x>l<>mHdWQIoq(eW6YDs2%CwE#W<80LHJ?~YwWK~+B zFZCr}Wh*dRCF9_1MvTo+2+ns6L?1{(Zl73l--C#g^s|IAz;KXG=0l~XGEVOjTa?X% z1$S3-OKOEqb=3*}|9SB$qN5c!vKQNNrS~WDN_DXb2C3(ZyJe0tMb2cY1kILcz$A8f z#|PLI6Liah*_~cRkWo)u7xGC$Pt|Hjw!mhINj`7P1>#ap8_xWW#WD8#2PYvz$IWxC zvj-mde=L0Kf!J0GPjg-~nZ^xJ+~HShHsc))$4IuqakgyXjk$9!0L@as&y-ty?2-Jr zSEO~G7V+niaFs2tw?dpT4em7Q{=K+KcaR?(o?b0&YmZ9kzvM^}(NhB4{@;r;EpGi} ztk%1AIsug0Vmen4%kw|})Y(%*L=0H;WG5^02XpuqjbjyGRxRA+b*+~dhlhT}-N<%X z{2s3nv~|+D_1j>w$kkE{43(PO!JJ_@(j%7K=!!llUjyoJ#vb=6j}7kNmMkQF?|{YX#`ko!ekFSdF;^{+jt;A8b$w literal 0 HcmV?d00001 diff --git a/tts/VALL-E-X/launch-ui.py b/tts/VALL-E-X/launch-ui.py new file mode 100644 index 0000000..b8516d6 --- /dev/null +++ b/tts/VALL-E-X/launch-ui.py @@ -0,0 +1,628 @@ +# coding: utf-8 +import argparse +import logging +import os +import pathlib +import time +import tempfile +import platform +import webbrowser +import sys +print(f"default encoding is {sys.getdefaultencoding()},file system encoding is {sys.getfilesystemencoding()}") +print(f"You are using Python version {platform.python_version()}") +if(sys.version_info[0]<3 or sys.version_info[1]<7): + print("The Python version is too low and may cause problems") + +if platform.system().lower() == 'windows': + temp = pathlib.PosixPath + pathlib.PosixPath = pathlib.WindowsPath +else: + temp = pathlib.WindowsPath + pathlib.WindowsPath = pathlib.PosixPath +os.environ["PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION"] = "python" + +import langid +langid.set_languages(['en', 'zh', 'ja']) + +import nltk +nltk.data.path = nltk.data.path + [os.path.join(os.getcwd(), "nltk_data")] + +import torch +import torchaudio +import random + +import numpy as np + +from data.tokenizer import ( + AudioTokenizer, + tokenize_audio, +) +from data.collation import get_text_token_collater +from models.vallex import VALLE +from utils.g2p import PhonemeBpeTokenizer +from descriptions import * +from macros import * +from examples import * + +import gradio as gr +import whisper +from vocos import Vocos +import multiprocessing + +thread_count = multiprocessing.cpu_count() + +print("Use",thread_count,"cpu cores for computing") + +torch.set_num_threads(thread_count) +torch.set_num_interop_threads(thread_count) +torch._C._jit_set_profiling_executor(False) +torch._C._jit_set_profiling_mode(False) +torch._C._set_graph_executor_optimize(False) + +text_tokenizer = PhonemeBpeTokenizer(tokenizer_path="./utils/g2p/bpe_69.json") +text_collater = get_text_token_collater() + +device = torch.device("cpu") +if torch.cuda.is_available(): + device = torch.device("cuda", 0) + +# VALL-E-X model +if not os.path.exists("./checkpoints/"): os.mkdir("./checkpoints/") +if not os.path.exists(os.path.join("./checkpoints/", "vallex-checkpoint.pt")): + import wget + try: + logging.info("Downloading model from https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt ...") + # download from https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt to ./checkpoints/vallex-checkpoint.pt + wget.download("https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt", + out="./checkpoints/vallex-checkpoint.pt", bar=wget.bar_adaptive) + except Exception as e: + logging.info(e) + raise Exception( + "\n Model weights download failed, please go to 'https://huggingface.co/Plachta/VALL-E-X/resolve/main/vallex-checkpoint.pt'" + "\n manually download model weights and put it to {} .".format(os.getcwd() + "\checkpoints")) + +model = VALLE( + N_DIM, + NUM_HEAD, + NUM_LAYERS, + norm_first=True, + add_prenet=False, + prefix_mode=PREFIX_MODE, + share_embedding=True, + nar_scale_factor=1.0, + prepend_bos=True, + num_quantizers=NUM_QUANTIZERS, + ) +checkpoint = torch.load("./checkpoints/vallex-checkpoint.pt", map_location='cpu') +missing_keys, unexpected_keys = model.load_state_dict( + checkpoint["model"], strict=True +) +assert not missing_keys +model.eval() + +# Encodec model +audio_tokenizer = AudioTokenizer(device) + +# Vocos decoder +vocos = Vocos.from_pretrained('charactr/vocos-encodec-24khz').to(device) + +# ASR +if not os.path.exists("./whisper/"): os.mkdir("./whisper/") +try: + whisper_model = whisper.load_model("medium",download_root=os.path.join(os.getcwd(), "whisper")).cpu() +except Exception as e: + logging.info(e) + raise Exception( + "\n Whisper download failed or damaged, please go to " + "'https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt'" + "\n manually download model and put it to {} .".format(os.getcwd() + "\whisper")) + +# Voice Presets +preset_list = os.walk("./presets/").__next__()[2] +preset_list = [preset[:-4] for preset in preset_list if preset.endswith(".npz")] + +def clear_prompts(): + try: + path = tempfile.gettempdir() + for eachfile in os.listdir(path): + filename = os.path.join(path, eachfile) + if os.path.isfile(filename) and filename.endswith(".npz"): + lastmodifytime = os.stat(filename).st_mtime + endfiletime = time.time() - 60 + if endfiletime > lastmodifytime: + os.remove(filename) + except: + return + +def transcribe_one(model, audio_path): + # load audio and pad/trim it to fit 30 seconds + audio = whisper.load_audio(audio_path) + audio = whisper.pad_or_trim(audio) + + # make log-Mel spectrogram and move to the same device as the model + mel = whisper.log_mel_spectrogram(audio).to(model.device) + + # detect the spoken language + _, probs = model.detect_language(mel) + print(f"Detected language: {max(probs, key=probs.get)}") + lang = max(probs, key=probs.get) + # decode the audio + options = whisper.DecodingOptions(temperature=1.0, best_of=5, fp16=False if device == torch.device("cpu") else True, sample_len=150) + result = whisper.decode(model, mel, options) + + # print the recognized text + print(result.text) + + text_pr = result.text + if text_pr.strip(" ")[-1] not in "?!.,。，？！。、": + text_pr += "." + return lang, text_pr + +def make_npz_prompt(name, uploaded_audio, recorded_audio, transcript_content): + global model, text_collater, text_tokenizer, audio_tokenizer + clear_prompts() + audio_prompt = uploaded_audio if uploaded_audio is not None else recorded_audio + sr, wav_pr = audio_prompt + if not isinstance(wav_pr, torch.FloatTensor): + wav_pr = torch.FloatTensor(wav_pr) + if wav_pr.abs().max() > 1: + wav_pr /= wav_pr.abs().max() + if wav_pr.size(-1) == 2: + wav_pr = wav_pr[:, 0] + if wav_pr.ndim == 1: + wav_pr = wav_pr.unsqueeze(0) + assert wav_pr.ndim and wav_pr.size(0) == 1 + + if transcript_content == "": + text_pr, lang_pr = make_prompt(name, wav_pr, sr, save=False) + else: + lang_pr = langid.classify(str(transcript_content))[0] + lang_token = lang2token[lang_pr] + text_pr = f"{lang_token}{str(transcript_content)}{lang_token}" + # tokenize audio + encoded_frames = tokenize_audio(audio_tokenizer, (wav_pr, sr)) + audio_tokens = encoded_frames[0][0].transpose(2, 1).cpu().numpy() + + # tokenize text + phonemes, _ = text_tokenizer.tokenize(text=f"{text_pr}".strip()) + text_tokens, enroll_x_lens = text_collater( + [ + phonemes + ] + ) + + message = f"Detected language: {lang_pr}\n Detected text {text_pr}\n" + + # save as npz file + np.savez(os.path.join(tempfile.gettempdir(), f"{name}.npz"), + audio_tokens=audio_tokens, text_tokens=text_tokens, lang_code=lang2code[lang_pr]) + return message, os.path.join(tempfile.gettempdir(), f"{name}.npz") + + +def make_prompt(name, wav, sr, save=True): + global whisper_model + whisper_model.to(device) + if not isinstance(wav, torch.FloatTensor): + wav = torch.tensor(wav) + if wav.abs().max() > 1: + wav /= wav.abs().max() + if wav.size(-1) == 2: + wav = wav.mean(-1, keepdim=False) + if wav.ndim == 1: + wav = wav.unsqueeze(0) + assert wav.ndim and wav.size(0) == 1 + torchaudio.save(f"./prompts/{name}.wav", wav, sr) + lang, text = transcribe_one(whisper_model, f"./prompts/{name}.wav") + lang_token = lang2token[lang] + text = lang_token + text + lang_token + with open(f"./prompts/{name}.txt", 'w', encoding='utf-8') as f: + f.write(text) + if not save: + os.remove(f"./prompts/{name}.wav") + os.remove(f"./prompts/{name}.txt") + + whisper_model.cpu() + torch.cuda.empty_cache() + return text, lang + +@torch.no_grad() +def infer_from_audio(text, language, accent, audio_prompt, record_audio_prompt, transcript_content): + global model, text_collater, text_tokenizer, audio_tokenizer + audio_prompt = audio_prompt if audio_prompt is not None else record_audio_prompt + sr, wav_pr = audio_prompt + if not isinstance(wav_pr, torch.FloatTensor): + wav_pr = torch.FloatTensor(wav_pr) + if wav_pr.abs().max() > 1: + wav_pr /= wav_pr.abs().max() + if wav_pr.size(-1) == 2: + wav_pr = wav_pr[:, 0] + if wav_pr.ndim == 1: + wav_pr = wav_pr.unsqueeze(0) + assert wav_pr.ndim and wav_pr.size(0) == 1 + + if transcript_content == "": + text_pr, lang_pr = make_prompt('dummy', wav_pr, sr, save=False) + else: + lang_pr = langid.classify(str(transcript_content))[0] + lang_token = lang2token[lang_pr] + text_pr = f"{lang_token}{str(transcript_content)}{lang_token}" + + if language == 'auto-detect': + lang_token = lang2token[langid.classify(text)[0]] + else: + lang_token = langdropdown2token[language] + lang = token2lang[lang_token] + text = lang_token + text + lang_token + + # onload model + model.to(device) + + # tokenize audio + encoded_frames = tokenize_audio(audio_tokenizer, (wav_pr, sr)) + audio_prompts = encoded_frames[0][0].transpose(2, 1).to(device) + + # tokenize text + logging.info(f"synthesize text: {text}") + phone_tokens, langs = text_tokenizer.tokenize(text=f"_{text}".strip()) + text_tokens, text_tokens_lens = text_collater( + [ + phone_tokens + ] + ) + + enroll_x_lens = None + if text_pr: + text_prompts, _ = text_tokenizer.tokenize(text=f"{text_pr}".strip()) + text_prompts, enroll_x_lens = text_collater( + [ + text_prompts + ] + ) + text_tokens = torch.cat([text_prompts, text_tokens], dim=-1) + text_tokens_lens += enroll_x_lens + lang = lang if accent == "no-accent" else token2lang[langdropdown2token[accent]] + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=-100, + temperature=1, + prompt_language=lang_pr, + text_language=langs if accent == "no-accent" else lang, + best_of=5, + ) + # Decode with Vocos + frames = encoded_frames.permute(2,0,1) + features = vocos.codes_to_features(frames) + samples = vocos.decode(features, bandwidth_id=torch.tensor([2], device=device)) + + # offload model + model.to('cpu') + torch.cuda.empty_cache() + + message = f"text prompt: {text_pr}\nsythesized text: {text}" + return message, (24000, samples.squeeze(0).cpu().numpy()) + +@torch.no_grad() +def infer_from_prompt(text, language, accent, preset_prompt, prompt_file): + clear_prompts() + model.to(device) + # text to synthesize + if language == 'auto-detect': + lang_token = lang2token[langid.classify(text)[0]] + else: + lang_token = langdropdown2token[language] + lang = token2lang[lang_token] + text = lang_token + text + lang_token + + # load prompt + if prompt_file is not None: + prompt_data = np.load(prompt_file.name) + else: + prompt_data = np.load(os.path.join("./presets/", f"{preset_prompt}.npz")) + audio_prompts = prompt_data['audio_tokens'] + text_prompts = prompt_data['text_tokens'] + lang_pr = prompt_data['lang_code'] + lang_pr = code2lang[int(lang_pr)] + + # numpy to tensor + audio_prompts = torch.tensor(audio_prompts).type(torch.int32).to(device) + text_prompts = torch.tensor(text_prompts).type(torch.int32) + + enroll_x_lens = text_prompts.shape[-1] + logging.info(f"synthesize text: {text}") + phone_tokens, langs = text_tokenizer.tokenize(text=f"_{text}".strip()) + text_tokens, text_tokens_lens = text_collater( + [ + phone_tokens + ] + ) + text_tokens = torch.cat([text_prompts, text_tokens], dim=-1) + text_tokens_lens += enroll_x_lens + # accent control + lang = lang if accent == "no-accent" else token2lang[langdropdown2token[accent]] + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=-100, + temperature=1, + prompt_language=lang_pr, + text_language=langs if accent == "no-accent" else lang, + best_of=5, + ) + # Decode with Vocos + frames = encoded_frames.permute(2,0,1) + features = vocos.codes_to_features(frames) + samples = vocos.decode(features, bandwidth_id=torch.tensor([2], device=device)) + + model.to('cpu') + torch.cuda.empty_cache() + + message = f"sythesized text: {text}" + return message, (24000, samples.squeeze(0).cpu().numpy()) + + +from utils.sentence_cutter import split_text_into_sentences +@torch.no_grad() +def infer_long_text(text, preset_prompt, prompt=None, language='auto', accent='no-accent'): + """ + For long audio generation, two modes are available. + fixed-prompt: This mode will keep using the same prompt the user has provided, and generate audio sentence by sentence. + sliding-window: This mode will use the last sentence as the prompt for the next sentence, but has some concern on speaker maintenance. + """ + mode = 'fixed-prompt' + global model, audio_tokenizer, text_tokenizer, text_collater + model.to(device) + if (prompt is None or prompt == "") and preset_prompt == "": + mode = 'sliding-window' # If no prompt is given, use sliding-window mode + sentences = split_text_into_sentences(text) + # detect language + if language == "auto-detect": + language = langid.classify(text)[0] + else: + language = token2lang[langdropdown2token[language]] + + # if initial prompt is given, encode it + if prompt is not None and prompt != "": + # load prompt + prompt_data = np.load(prompt.name) + audio_prompts = prompt_data['audio_tokens'] + text_prompts = prompt_data['text_tokens'] + lang_pr = prompt_data['lang_code'] + lang_pr = code2lang[int(lang_pr)] + + # numpy to tensor + audio_prompts = torch.tensor(audio_prompts).type(torch.int32).to(device) + text_prompts = torch.tensor(text_prompts).type(torch.int32) + elif preset_prompt is not None and preset_prompt != "": + prompt_data = np.load(os.path.join("./presets/", f"{preset_prompt}.npz")) + audio_prompts = prompt_data['audio_tokens'] + text_prompts = prompt_data['text_tokens'] + lang_pr = prompt_data['lang_code'] + lang_pr = code2lang[int(lang_pr)] + + # numpy to tensor + audio_prompts = torch.tensor(audio_prompts).type(torch.int32).to(device) + text_prompts = torch.tensor(text_prompts).type(torch.int32) + else: + audio_prompts = torch.zeros([1, 0, NUM_QUANTIZERS]).type(torch.int32).to(device) + text_prompts = torch.zeros([1, 0]).type(torch.int32) + lang_pr = language if language != 'mix' else 'en' + if mode == 'fixed-prompt': + complete_tokens = torch.zeros([1, NUM_QUANTIZERS, 0]).type(torch.LongTensor).to(device) + for text in sentences: + text = text.replace("\n", "").strip(" ") + if text == "": + continue + lang_token = lang2token[language] + lang = token2lang[lang_token] + text = lang_token + text + lang_token + + enroll_x_lens = text_prompts.shape[-1] + logging.info(f"synthesize text: {text}") + phone_tokens, langs = text_tokenizer.tokenize(text=f"_{text}".strip()) + text_tokens, text_tokens_lens = text_collater( + [ + phone_tokens + ] + ) + text_tokens = torch.cat([text_prompts, text_tokens], dim=-1) + text_tokens_lens += enroll_x_lens + # accent control + lang = lang if accent == "no-accent" else token2lang[langdropdown2token[accent]] + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=-100, + temperature=1, + prompt_language=lang_pr, + text_language=langs if accent == "no-accent" else lang, + best_of=5, + ) + complete_tokens = torch.cat([complete_tokens, encoded_frames.transpose(2, 1)], dim=-1) + # Decode with Vocos + frames = complete_tokens.permute(1, 0, 2) + features = vocos.codes_to_features(frames) + samples = vocos.decode(features, bandwidth_id=torch.tensor([2], device=device)) + + model.to('cpu') + message = f"Cut into {len(sentences)} sentences" + return message, (24000, samples.squeeze(0).cpu().numpy()) + elif mode == "sliding-window": + complete_tokens = torch.zeros([1, NUM_QUANTIZERS, 0]).type(torch.LongTensor).to(device) + original_audio_prompts = audio_prompts + original_text_prompts = text_prompts + for text in sentences: + text = text.replace("\n", "").strip(" ") + if text == "": + continue + lang_token = lang2token[language] + lang = token2lang[lang_token] + text = lang_token + text + lang_token + + enroll_x_lens = text_prompts.shape[-1] + logging.info(f"synthesize text: {text}") + phone_tokens, langs = text_tokenizer.tokenize(text=f"_{text}".strip()) + text_tokens, text_tokens_lens = text_collater( + [ + phone_tokens + ] + ) + text_tokens = torch.cat([text_prompts, text_tokens], dim=-1) + text_tokens_lens += enroll_x_lens + # accent control + lang = lang if accent == "no-accent" else token2lang[langdropdown2token[accent]] + encoded_frames = model.inference( + text_tokens.to(device), + text_tokens_lens.to(device), + audio_prompts, + enroll_x_lens=enroll_x_lens, + top_k=-100, + temperature=1, + prompt_language=lang_pr, + text_language=langs if accent == "no-accent" else lang, + best_of=5, + ) + complete_tokens = torch.cat([complete_tokens, encoded_frames.transpose(2, 1)], dim=-1) + if torch.rand(1) < 1.0: + audio_prompts = encoded_frames[:, :, -NUM_QUANTIZERS:] + text_prompts = text_tokens[:, enroll_x_lens:] + else: + audio_prompts = original_audio_prompts + text_prompts = original_text_prompts + # Decode with Vocos + frames = complete_tokens.permute(1, 0, 2) + features = vocos.codes_to_features(frames) + samples = vocos.decode(features, bandwidth_id=torch.tensor([2], device=device)) + + model.to('cpu') + message = f"Cut into {len(sentences)} sentences" + return message, (24000, samples.squeeze(0).cpu().numpy()) + else: + raise ValueError(f"No such mode {mode}") + + +def main(): + app = gr.Blocks(title="VALL-E X") + with app: + gr.Markdown(top_md) + with gr.Tab("Infer from audio"): + gr.Markdown(infer_from_audio_md) + with gr.Row(): + with gr.Column(): + + textbox = gr.TextArea(label="Text", + placeholder="Type your sentence here", + value="Welcome back, Master. What can I do for you today?", elem_id=f"tts-input") + language_dropdown = gr.Dropdown(choices=['auto-detect', 'English', '中文', '日本語'], value='auto-detect', label='language') + accent_dropdown = gr.Dropdown(choices=['no-accent', 'English', '中文', '日本語'], value='no-accent', label='accent') + textbox_transcript = gr.TextArea(label="Transcript", + placeholder="Write transcript here. (leave empty to use whisper)", + value="", elem_id=f"prompt-name") + upload_audio_prompt = gr.Audio(label='uploaded audio prompt', source='upload', interactive=True) + record_audio_prompt = gr.Audio(label='recorded audio prompt', source='microphone', interactive=True) + with gr.Column(): + text_output = gr.Textbox(label="Message") + audio_output = gr.Audio(label="Output Audio", elem_id="tts-audio") + btn = gr.Button("Generate!") + btn.click(infer_from_audio, + inputs=[textbox, language_dropdown, accent_dropdown, upload_audio_prompt, record_audio_prompt, textbox_transcript], + outputs=[text_output, audio_output]) + textbox_mp = gr.TextArea(label="Prompt name", + placeholder="Name your prompt here", + value="prompt_1", elem_id=f"prompt-name") + btn_mp = gr.Button("Make prompt!") + prompt_output = gr.File(interactive=False) + btn_mp.click(make_npz_prompt, + inputs=[textbox_mp, upload_audio_prompt, record_audio_prompt, textbox_transcript], + outputs=[text_output, prompt_output]) + gr.Examples(examples=infer_from_audio_examples, + inputs=[textbox, language_dropdown, accent_dropdown, upload_audio_prompt, record_audio_prompt, textbox_transcript], + outputs=[text_output, audio_output], + fn=infer_from_audio, + cache_examples=False,) + with gr.Tab("Make prompt"): + gr.Markdown(make_prompt_md) + with gr.Row(): + with gr.Column(): + textbox2 = gr.TextArea(label="Prompt name", + placeholder="Name your prompt here", + value="prompt_1", elem_id=f"prompt-name") + # 添加选择语言和输入台本的地方 + textbox_transcript2 = gr.TextArea(label="Transcript", + placeholder="Write transcript here. (leave empty to use whisper)", + value="", elem_id=f"prompt-name") + upload_audio_prompt_2 = gr.Audio(label='uploaded audio prompt', source='upload', interactive=True) + record_audio_prompt_2 = gr.Audio(label='recorded audio prompt', source='microphone', interactive=True) + with gr.Column(): + text_output_2 = gr.Textbox(label="Message") + prompt_output_2 = gr.File(interactive=False) + btn_2 = gr.Button("Make!") + btn_2.click(make_npz_prompt, + inputs=[textbox2, upload_audio_prompt_2, record_audio_prompt_2, textbox_transcript2], + outputs=[text_output_2, prompt_output_2]) + gr.Examples(examples=make_npz_prompt_examples, + inputs=[textbox2, upload_audio_prompt_2, record_audio_prompt_2, textbox_transcript2], + outputs=[text_output_2, prompt_output_2], + fn=make_npz_prompt, + cache_examples=False,) + with gr.Tab("Infer from prompt"): + gr.Markdown(infer_from_prompt_md) + with gr.Row(): + with gr.Column(): + textbox_3 = gr.TextArea(label="Text", + placeholder="Type your sentence here", + value="Welcome back, Master. What can I do for you today?", elem_id=f"tts-input") + language_dropdown_3 = gr.Dropdown(choices=['auto-detect', 'English', '中文', '日本語', 'Mix'], value='auto-detect', + label='language') + accent_dropdown_3 = gr.Dropdown(choices=['no-accent', 'English', '中文', '日本語'], value='no-accent', + label='accent') + preset_dropdown_3 = gr.Dropdown(choices=preset_list, value=None, label='Voice preset') + prompt_file = gr.File(file_count='single', file_types=['.npz'], interactive=True) + with gr.Column(): + text_output_3 = gr.Textbox(label="Message") + audio_output_3 = gr.Audio(label="Output Audio", elem_id="tts-audio") + btn_3 = gr.Button("Generate!") + btn_3.click(infer_from_prompt, + inputs=[textbox_3, language_dropdown_3, accent_dropdown_3, preset_dropdown_3, prompt_file], + outputs=[text_output_3, audio_output_3]) + gr.Examples(examples=infer_from_prompt_examples, + inputs=[textbox_3, language_dropdown_3, accent_dropdown_3, preset_dropdown_3, prompt_file], + outputs=[text_output_3, audio_output_3], + fn=infer_from_prompt, + cache_examples=False,) + with gr.Tab("Infer long text"): + gr.Markdown("This is a long text generation demo. You can use this to generate long audio. ") + with gr.Row(): + with gr.Column(): + textbox_4 = gr.TextArea(label="Text", + placeholder="Type your sentence here", + value=long_text_example, elem_id=f"tts-input") + language_dropdown_4 = gr.Dropdown(choices=['auto-detect', 'English', '中文', '日本語'], value='auto-detect', + label='language') + accent_dropdown_4 = gr.Dropdown(choices=['no-accent', 'English', '中文', '日本語'], value='no-accent', + label='accent') + preset_dropdown_4 = gr.Dropdown(choices=preset_list, value=None, label='Voice preset') + prompt_file_4 = gr.File(file_count='single', file_types=['.npz'], interactive=True) + with gr.Column(): + text_output_4 = gr.TextArea(label="Message") + audio_output_4 = gr.Audio(label="Output Audio", elem_id="tts-audio") + btn_4 = gr.Button("Generate!") + btn_4.click(infer_long_text, + inputs=[textbox_4, preset_dropdown_4, prompt_file_4, language_dropdown_4, accent_dropdown_4], + outputs=[text_output_4, audio_output_4]) + + webbrowser.open("http://127.0.0.1:7860") + app.launch() + +if __name__ == "__main__": + formatter = ( + "%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s" + ) + logging.basicConfig(format=formatter, level=logging.INFO) + main() diff --git a/tts/VALL-E-X/macros.py b/tts/VALL-E-X/macros.py new file mode 100644 index 0000000..b192fcc --- /dev/null +++ b/tts/VALL-E-X/macros.py @@ -0,0 +1,39 @@ +NUM_LAYERS = 12 +NUM_HEAD = 16 +N_DIM = 1024 +PREFIX_MODE = 1 +NUM_QUANTIZERS = 8 +SAMPLE_RATE = 24000 + +lang2token = { + 'zh': "[ZH]", + 'ja': "[JA]", + "en": "[EN]", + 'mix': "", +} + +lang2code = { + 'zh': 0, + 'ja': 1, + "en": 2, +} + +token2lang = { + '[ZH]': "zh", + '[JA]': "ja", + "[EN]": "en", + "": "mix" +} + +code2lang = { + 0: 'zh', + 1: 'ja', + 2: "en", +} + +langdropdown2token = { + 'English': "[EN]", + '中文': "[ZH]", + '日本語': "[JA]", + 'Mix': "", +} \ No newline at end of file diff --git a/tts/VALL-E-X/main.py b/tts/VALL-E-X/main.py new file mode 100644 index 0000000..59d9100 --- /dev/null +++ b/tts/VALL-E-X/main.py @@ -0,0 +1,216 @@ +import argparse + +# python3 main.py -create_prompt_with_list 20.wav 21.wav -create_prompt_name alice +# python3 main.py -use_prompt_name alice -output_file_name alice_says.wav -text "This is the text to be converted to speech." + +# python3 main.py -create_prompt_with_list 20.wav 21.wav 20.wav 21.wav -create_prompt_name alice -use_prompt_name alice -output_file_name alice_says.wav -text "This is my voice." + +# python3 main.py -create_prompt_with_list goku.wav -create_prompt_name goku -use_prompt_name goku -output_file_name goku_test.wav -text "oh my god krillin! where did you put my sensu beans!?" + +# python3 main.py -create_prompt_with_list + +from typing import List +import pathlib +import whisperx + +# https://discuss.pytorch.org/t/how-to-calculate-the-gpu-memory-that-a-model-uses/157486/6 <-- for fine tuning memory usage later. + +from utils.prompt_making import make_prompt +import logging + +from utils.generation import SAMPLE_RATE, generate_audio, preload_models,load_models_from_mount +import soundfile as sf +import numpy as np +import torchaudio +from typing import List +import uuid +import torchaudio.transforms as T + +class AudioOps(): + + @staticmethod + def stitch_wav_tensors_with_crossfade(wav_tensor1:np.array,wav_tensor2:np.array,sample_rate:int): + # Load two audio signals as NumPy tensors + # Compute the length of the cross-fade window in samples + fade_len = int(0.01 * sample_rate) + + # Create a cross-fade window + fade_window = np.hanning(2*fade_len) + + # Concatenate the signals with cross-fade + audio_out = np.concatenate((wav_tensor1[:-fade_len], + wav_tensor1[-fade_len:] * fade_window[:fade_len] + wav_tensor2[:fade_len] * fade_window[fade_len:], + wav_tensor2[fade_len:])) + return audio_out + + @staticmethod + def cut_audio_range(file_path, start_seconds, end_seconds, output_path="cut_audio.wav"): + """ + Cuts out a specific segment from an audio file and saves the result. + + Args: + - file_path (str): Path to the audio file from which the segment will be cut. + - start_seconds (float): Start time of the segment to be cut out in seconds. + - end_seconds (float): End time of the segment to be cut out in seconds. + - output_path (str): Path to save the resulting audio. Defaults to "cut_audio.wav". + + Returns: + - None + """ + # Load the audio file + waveform, sr = torchaudio.load(file_path) + # Convert the range to sample indices + start_sample = int(start_seconds * sr) + end_sample = int(end_seconds * sr) + cut_tensor = waveform[:,start_sample:end_sample] + torchaudio.save(output_path, cut_tensor, sr) + + @staticmethod + def stitch_wav_files_resample(root_path:pathlib.Path, files:List[str],target_sample_rate:int=SAMPLE_RATE): + resampler = None + result = None + for file in files: + if type(result) == type(None): + tensor, sr = torchaudio.load(root_path / pathlib.Path(file)) + resampler = T.Resample(orig_freq=sr,new_freq=target_sample_rate) + resampled_tensor = resampler(tensor) + result = resampled_tensor.squeeze() + result = np.array(result) + else: + tensor, _ = torchaudio.load(root_path / pathlib.Path(file)) + resampled_tensor = resampler(tensor) + resampled_tensor = resampled_tensor.squeeze() + resampled_tensor = np.array(resampled_tensor) + result = AudioOps.stitch_wav_tensors_with_crossfade(result,resampled_tensor,sample_rate=target_sample_rate) + + return result + +class VoiceDesigner(): + def __init__(self) -> None: + # load models from host system. + self.logger = self.setup_logger() + + self.mount_folder = pathlib.Path.cwd() / pathlib.Path("..") / pathlib.Path("Vall-E-mount") + # Path for the inputs bunch of audio files + self.input_path = self.mount_folder / pathlib.Path("input") + # Path for the models to be used + self.models_path = self.mount_folder / pathlib.Path("models") + # Path for the existing presets + self.presets_path = self.mount_folder / pathlib.Path("presets") + # Path for the new prompts created by the user + self.prompts_path = self.mount_folder / pathlib.Path("prompts") + # Path for the temporary files + self.temp_path = self.mount_folder / pathlib.Path("temp") + # Path for output + self.output_path = self.mount_folder / pathlib.Path("output") + + load_models_from_mount(model_path=self.models_path, + vocos_path=pathlib.Path("vocos-encodec-24khz"), + vall_e_x=pathlib.Path("vallex-checkpoint.pt")) + + def tts_with_prompt(self,prompt:pathlib.Path,text:str,output_file_name:str): + # TODO Filtering bad words + # TODO check if the files are actually wav ... + audio_array = generate_audio(text,prompt=prompt) + # save audio to disk + sf.write(self.output_path / pathlib.Path(output_file_name), audio_array, SAMPLE_RATE) + + def create_prompt(self,audio_file_paths:List[pathlib.Path],prompt_file_name:str): + # multiple audio files concat + # concat audio and trim the audio if it is longer than 14-15 seconds# if prompt is too long it will error out anyways + if len(audio_file_paths) > 0: + # TODO: going to assume brandon renames the files as they are uploaded + try:# ./customs/ <-- folder it adds by default until we modified. + # load the audio files and then concat them. + tensor_result = AudioOps.stitch_wav_files_resample(root_path=self.input_path,files=audio_file_paths) + # save to disk + generated_name = f"{uuid.uuid4()}" + + concat_audio_file_path = self.temp_path / pathlib.Path(f"audio_concat_{generated_name}.wav") + sf.write(concat_audio_file_path, tensor_result, SAMPLE_RATE) + # save result into processing... + # trim to 15 seconds. + audio_cut_path = self.temp_path / pathlib.Path(f"audio_cut_{generated_name}.wav") + AudioOps.cut_audio_range(file_path=concat_audio_file_path,start_seconds=0,end_seconds=15,output_path=audio_cut_path) + # TODO get whisper x to work in the container and generate transcription that way or use a different container to do that work. + make_prompt(name=prompt_file_name,audio_prompt_output_path=self.prompts_path,audio_prompt_path=audio_cut_path,transcript=None) + except Exception as e: + self.logger.debug(f"Exception: {e.with_traceback()}") + raise + else: + self.logger.debug(f"No audio files to load") + + def setup_logger(self): + # Create a logger object + logger = logging.getLogger(__name__) + + # Set the log level + logger.setLevel(logging.DEBUG) + + # Create a file handler to log messages to a file + file_handler = logging.FileHandler('docker_valle-x.log') + file_handler.setLevel(logging.DEBUG) + + # Create a console handler to log messages to the console + console_handler = logging.StreamHandler() + console_handler.setLevel(logging.DEBUG) + + # Create a formatter + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # Set the formatter for the handlers + file_handler.setFormatter(formatter) + console_handler.setFormatter(formatter) + + # Add the handlers to the logger + logger.addHandler(file_handler) + logger.addHandler(console_handler) + + return logger + + +global voice_designer +voice_designer = VoiceDesigner() + +def main(args): + print(args.create_prompt_with_list) + print(args.create_prompt_name) + + print(args.text) + print(args.use_prompt_name) + print(args.output_file_name) + + if args.create_prompt_name is not None and args.create_prompt_name is not None: + voice_designer.create_prompt(audio_file_paths=args.create_prompt_with_list, + prompt_file_name=args.create_prompt_name) + + if args.text is not None and args.use_prompt_name is not None: + voice_designer.tts_with_prompt(prompt=args.use_prompt_name, + text=args.text, + output_file_name=args.output_file_name) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser(description="Process a list of speaker samples for zero shot - .wav file pathes, convert text to speech, and specify language.") + + parser.add_argument('-create_prompt_with_list', metavar='prompt wav files', nargs='+', + help='Takes in a list of .wav files for an audio prompt provide the absolute path. Must be less than 15 seconds otherwise will error out.') + + parser.add_argument('-create_prompt_name',metavar="prompt name to create", + help='required with -create-prompt prompt output name this will be written to the mount. In Vall-E-mount/prompts') + + parser.add_argument('-text', metavar='text to synthesize', type=str, + help='Text to be converted to speech.') + + parser.add_argument("-use_prompt_name",metavar="prompt you want to use",type=str) + + parser.add_argument("-output_file_name",metavar="file name",type=str) + + args = parser.parse_args() + + main(args) + + + + + \ No newline at end of file diff --git a/tts/VALL-E-X/model-card.md b/tts/VALL-E-X/model-card.md new file mode 100644 index 0000000..769b7b6 --- /dev/null +++ b/tts/VALL-E-X/model-card.md @@ -0,0 +1,33 @@ +# Model Card: VALL-E X + +**Author**: [Songting](https://github.com/Plachtaa).
+
+This is the official codebase for running open-sourced VALL-E X. + +The following is additional information about the models released here. + +## Model Details + +VALL-E X is a series of two transformer models that turn text into audio. + +### Phoneme to acoustic tokens + - Input: IPAs converted from input text by a rule-based G2P tool. + - Output: tokens from the first codebook of the [EnCodec Codec](https://github.com/facebookresearch/encodec) from facebook + +### Coarse to fine tokens + - Input: IPAs converted from input text by a rule-based G2P tool & the first codebook from EnCodec + - Output: 8 codebooks from EnCodec + +### Architecture +| Model | Parameters | Attention | Output Vocab size | +|:------------------------:|:----------:|------------|:-----------------:| +| G2P tool | - | - | 69 | +| Phoneme to coarse tokens | 150 M | Causal | 1x 1,024 | +| Coarse to fine tokens | 150 M | Non-causal | 7x 1,024 | + +### Release date +August 2023 + +## Broader Implications +We anticipate that this model's text to audio capabilities can be used to improve accessbility tools in a variety of languages. +Straightforward improvements will allow models to run faster than realtime, rendering them useful for applications such as virtual assistants. \ No newline at end of file diff --git a/tts/VALL-E-X/models/__init__.py b/tts/VALL-E-X/models/__init__.py new file mode 100644 index 0000000..f64eed0 --- /dev/null +++ b/tts/VALL-E-X/models/__init__.py @@ -0,0 +1,136 @@ +import argparse + +import torch.nn as nn +# from icefall.utils import AttributeDict, str2bool + +from .macros import ( + NUM_AUDIO_TOKENS, + NUM_MEL_BINS, + NUM_SPEAKER_CLASSES, + NUM_TEXT_TOKENS, + SPEAKER_EMBEDDING_DIM, +) +from .transformer import Transformer +from .vallex import VALLE, VALLF +from .visualizer import visualize + + +def add_model_arguments(parser: argparse.ArgumentParser): + parser.add_argument( + "--model-name", + type=str, + default="VALL-E", + help="VALL-E, VALL-F, Transformer.", + ) + parser.add_argument( + "--decoder-dim", + type=int, + default=1024, + help="Embedding dimension in the decoder model.", + ) + parser.add_argument( + "--nhead", + type=int, + default=16, + help="Number of attention heads in the Decoder layers.", + ) + parser.add_argument( + "--num-decoder-layers", + type=int, + default=12, + help="Number of Decoder layers.", + ) + parser.add_argument( + "--scale-factor", + type=float, + default=1.0, + help="Model scale factor which will be assigned different meanings in different models.", + ) + parser.add_argument( + "--norm-first", + type=bool, + default=True, + help="Pre or Post Normalization.", + ) + parser.add_argument( + "--add-prenet", + type=bool, + default=False, + help="Whether add PreNet after Inputs.", + ) + + # VALL-E & F + parser.add_argument( + "--prefix-mode", + type=int, + default=1, + help="The mode for how to prefix VALL-E NAR Decoder, " + "0: no prefix, 1: 0 to random, 2: random to random, 4: chunk of pre or post utterance.", + ) + parser.add_argument( + "--share-embedding", + type=bool, + default=True, + help="Share the parameters of the output projection layer with the parameters of the acoustic embedding.", + ) + parser.add_argument( + "--prepend-bos", + type=bool, + default=False, + help="Whether prepend to the acoustic tokens -> AR Decoder inputs.", + ) + parser.add_argument( + "--num-quantizers", + type=int, + default=8, + help="Number of Audio/Semantic quantization layers.", + ) + + # Transformer + parser.add_argument( + "--scaling-xformers", + type=bool, + default=False, + help="Apply Reworked Conformer scaling on Transformers.", + ) + + +def get_model(params) -> nn.Module: + if params.model_name.lower() in ["vall-f", "vallf"]: + model = VALLF( + params.decoder_dim, + params.nhead, + params.num_decoder_layers, + norm_first=params.norm_first, + add_prenet=params.add_prenet, + prefix_mode=params.prefix_mode, + share_embedding=params.share_embedding, + nar_scale_factor=params.scale_factor, + prepend_bos=params.prepend_bos, + num_quantizers=params.num_quantizers, + ) + elif params.model_name.lower() in ["vall-e", "valle"]: + model = VALLE( + params.decoder_dim, + params.nhead, + params.num_decoder_layers, + norm_first=params.norm_first, + add_prenet=params.add_prenet, + prefix_mode=params.prefix_mode, + share_embedding=params.share_embedding, + nar_scale_factor=params.scale_factor, + prepend_bos=params.prepend_bos, + num_quantizers=params.num_quantizers, + ) + else: + assert params.model_name in ["Transformer"] + model = Transformer( + params.decoder_dim, + params.nhead, + params.num_decoder_layers, + norm_first=params.norm_first, + add_prenet=params.add_prenet, + scaling_xformers=params.scaling_xformers, + ) + + return model diff --git a/tts/VALL-E-X/models/macros.py b/tts/VALL-E-X/models/macros.py new file mode 100644 index 0000000..cbc5496 --- /dev/null +++ b/tts/VALL-E-X/models/macros.py @@ -0,0 +1,11 @@ +# Text +NUM_TEXT_TOKENS = 2048 + +# Audio +NUM_AUDIO_TOKENS = 1024 # EnCodec RVQ bins +NUM_MEL_BINS = 100 # BigVGAN bigvgan_24khz_100band + + +# Speaker +NUM_SPEAKER_CLASSES = 4096 +SPEAKER_EMBEDDING_DIM = 64 diff --git a/tts/VALL-E-X/models/transformer.py b/tts/VALL-E-X/models/transformer.py new file mode 100644 index 0000000..8dca92d --- /dev/null +++ b/tts/VALL-E-X/models/transformer.py @@ -0,0 +1,394 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial +from typing import Any, Dict, List, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +# from icefall.utils import make_pad_mask +# from torchmetrics.classification import BinaryAccuracy + +from models.vallex import Transpose +from modules.embedding import SinePositionalEmbedding, TokenEmbedding +from modules.scaling import BalancedDoubleSwish, ScaledLinear +from modules.transformer import ( + BalancedBasicNorm, + IdentityNorm, + TransformerDecoderLayer, + TransformerEncoder, + TransformerEncoderLayer, +) + +from .macros import NUM_MEL_BINS, NUM_TEXT_TOKENS +from .visualizer import visualize + +IdentityNorm = IdentityNorm + + +class Transformer(nn.Module): + """It implements seq2seq Transformer TTS for debug(No StopPredictor and SpeakerEmbeding) + Neural Speech Synthesis with Transformer Network + https://arxiv.org/abs/1809.08895 + """ + + def __init__( + self, + d_model: int, + nhead: int, + num_layers: int, + norm_first: bool = True, + add_prenet: bool = False, + scaling_xformers: bool = False, + ): + """ + Args: + d_model: + The number of expected features in the input (required). + nhead: + The number of heads in the multiheadattention models (required). + num_layers: + The number of sub-decoder-layers in the decoder (required). + """ + super().__init__() + self.text_embedding = TokenEmbedding(d_model, NUM_TEXT_TOKENS) # W_x + + if add_prenet: + self.encoder_prenet = nn.Sequential( + Transpose(), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + Transpose(), + nn.Linear(d_model, d_model), + ) + + self.decoder_prenet = nn.Sequential( + nn.Linear(NUM_MEL_BINS, 256), + nn.ReLU(), + nn.Dropout(0.5), + nn.Linear(256, 256), + nn.ReLU(), + nn.Dropout(0.5), + nn.Linear(256, d_model), + ) + + assert scaling_xformers is False # TODO: update this block + else: + self.encoder_prenet = nn.Identity() + if scaling_xformers: + self.decoder_prenet = ScaledLinear(NUM_MEL_BINS, d_model) + else: + self.decoder_prenet = nn.Linear(NUM_MEL_BINS, d_model) + + self.encoder_position = SinePositionalEmbedding( + d_model, + dropout=0.1, + scale=False, + ) + self.decoder_position = SinePositionalEmbedding( + d_model, dropout=0.1, scale=False + ) + + if scaling_xformers: + self.encoder = TransformerEncoder( + TransformerEncoderLayer( + d_model, + nhead, + dim_feedforward=d_model * 4, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + linear1_self_attention_cls=ScaledLinear, + linear2_self_attention_cls=partial( + ScaledLinear, initial_scale=0.01 + ), + linear1_feedforward_cls=ScaledLinear, + linear2_feedforward_cls=partial( + ScaledLinear, initial_scale=0.01 + ), + activation=partial( + BalancedDoubleSwish, + channel_dim=-1, + max_abs=10.0, + min_prob=0.25, + ), + layer_norm_cls=IdentityNorm, + ), + num_layers=num_layers, + norm=BalancedBasicNorm(d_model) if norm_first else None, + ) + + self.decoder = nn.TransformerDecoder( + TransformerDecoderLayer( + d_model, + nhead, + dim_feedforward=d_model * 4, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + linear1_self_attention_cls=ScaledLinear, + linear2_self_attention_cls=partial( + ScaledLinear, initial_scale=0.01 + ), + linear1_feedforward_cls=ScaledLinear, + linear2_feedforward_cls=partial( + ScaledLinear, initial_scale=0.01 + ), + activation=partial( + BalancedDoubleSwish, + channel_dim=-1, + max_abs=10.0, + min_prob=0.25, + ), + layer_norm_cls=IdentityNorm, + ), + num_layers=num_layers, + norm=BalancedBasicNorm(d_model) if norm_first else None, + ) + + self.predict_layer = ScaledLinear(d_model, NUM_MEL_BINS) + self.stop_layer = nn.Linear(d_model, 1) + else: + self.encoder = nn.TransformerEncoder( + nn.TransformerEncoderLayer( + d_model, + nhead, + dim_feedforward=d_model * 4, + activation=F.relu, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + ), + num_layers=num_layers, + norm=nn.LayerNorm(d_model) if norm_first else None, + ) + + self.decoder = nn.TransformerDecoder( + nn.TransformerDecoderLayer( + d_model, + nhead, + dim_feedforward=d_model * 4, + activation=F.relu, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + ), + num_layers=num_layers, + norm=nn.LayerNorm(d_model) if norm_first else None, + ) + + self.predict_layer = nn.Linear(d_model, NUM_MEL_BINS) + self.stop_layer = nn.Linear(d_model, 1) + + self.stop_accuracy_metric = BinaryAccuracy( + threshold=0.5, multidim_average="global" + ) + + # self.apply(self._init_weights) + + # def _init_weights(self, module): + # if isinstance(module, (nn.Linear)): + # module.weight.data.normal_(mean=0.0, std=0.02) + # if isinstance(module, nn.Linear) and module.bias is not None: + # module.bias.data.zero_() + # elif isinstance(module, nn.LayerNorm): + # module.bias.data.zero_() + # module.weight.data.fill_(1.0) + # elif isinstance(module, nn.Embedding): + # module.weight.data.normal_(mean=0.0, std=0.02) + + def forward( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: torch.Tensor, + y_lens: torch.Tensor, + reduction: str = "sum", + train_stage: int = 0, + **kwargs, + ) -> Tuple[torch.Tensor, Union[torch.Tensor, None]]: + """ + Args: + x: + A 2-D tensor of shape (N, S). + x_lens: + A 1-D tensor of shape (N,). It contains the number of tokens in `x` + before padding. + y: + A 3-D tensor of shape (N, T, 8). + y_lens: + A 1-D tensor of shape (N,). It contains the number of tokens in `x` + before padding. + train_stage: + Not used in this model. + Returns: + Return the predicted audio code matrix, cross-entropy loss and Top-10 accuracy. + """ + del train_stage + + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + assert y.ndim == 3, y.shape + assert y_lens.ndim == 1, y_lens.shape + + assert torch.all(x_lens > 0) + + # NOTE: x has been padded in TextTokenCollater + x_mask = make_pad_mask(x_lens).to(x.device) + + x = self.text_embedding(x) + x = self.encoder_prenet(x) + x = self.encoder_position(x) + x = self.encoder(x, src_key_padding_mask=x_mask) + + total_loss, metrics = 0.0, {} + + y_mask = make_pad_mask(y_lens).to(y.device) + y_mask_float = y_mask.type(torch.float32) + data_mask = 1.0 - y_mask_float.unsqueeze(-1) + + # Training + # AR Decoder + def pad_y(y): + y = F.pad(y, (0, 0, 1, 0, 0, 0), value=0).detach() + # inputs, targets + return y[:, :-1], y[:, 1:] + + y, targets = pad_y(y * data_mask) # mask padding as zeros + + y_emb = self.decoder_prenet(y) + y_pos = self.decoder_position(y_emb) + + y_len = y_lens.max() + tgt_mask = torch.triu( + torch.ones(y_len, y_len, device=y.device, dtype=torch.bool), + diagonal=1, + ) + y_dec = self.decoder( + y_pos, + x, + tgt_mask=tgt_mask, + memory_key_padding_mask=x_mask, + ) + + predict = self.predict_layer(y_dec) + # loss + total_loss = F.mse_loss(predict, targets, reduction=reduction) + + logits = self.stop_layer(y_dec).squeeze(-1) + stop_loss = F.binary_cross_entropy_with_logits( + logits, + y_mask_float.detach(), + weight=1.0 + y_mask_float.detach() * 4.0, + reduction=reduction, + ) + metrics["stop_loss"] = stop_loss.detach() + + stop_accuracy = self.stop_accuracy_metric( + (torch.sigmoid(logits) >= 0.5).type(torch.int64), + y_mask.type(torch.int64), + ) + # icefall MetricsTracker.norm_items() + metrics["stop_accuracy"] = stop_accuracy.item() * y_lens.sum().type( + torch.float32 + ) + + return ((x, predict), total_loss + 100.0 * stop_loss, metrics) + + def inference( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: Any = None, + **kwargs, + ) -> torch.Tensor: + """ + Args: + x: + A 2-D tensor of shape (1, S). + x_lens: + A 1-D tensor of shape (1,). It contains the number of tokens in `x` + before padding. + Returns: + Return the predicted audio code matrix and cross-entropy loss. + """ + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + + assert torch.all(x_lens > 0) + + x_mask = make_pad_mask(x_lens).to(x.device) + + x = self.text_embedding(x) + x = self.encoder_prenet(x) + x = self.encoder_position(x) + x = self.encoder(x, src_key_padding_mask=x_mask) + + x_mask = make_pad_mask(x_lens).to(x.device) + + # AR Decoder + # TODO: Managing decoder steps avoid repetitive computation + y = torch.zeros( + [x.shape[0], 1, NUM_MEL_BINS], dtype=torch.float32, device=x.device + ) + while True: + y_emb = self.decoder_prenet(y) + y_pos = self.decoder_position(y_emb) + + tgt_mask = torch.triu( + torch.ones( + y.shape[1], y.shape[1], device=y.device, dtype=torch.bool + ), + diagonal=1, + ) + + y_dec = self.decoder( + y_pos, + x, + tgt_mask=tgt_mask, + memory_mask=None, + memory_key_padding_mask=x_mask, + ) + predict = self.predict_layer(y_dec[:, -1:]) + + logits = self.stop_layer(y_dec[:, -1:]) > 0 # sigmoid(0.0) = 0.5 + if y.shape[1] > x_lens.max() * 10 or all(logits.cpu().numpy()): + print( + f"TransformerTTS EOS [Text {x_lens[0]} -> Audio {y.shape[1]}]" + ) + break + + y = torch.concat([y, predict], dim=1) + + return y[:, 1:] + + def visualize( + self, + predicts: Tuple[torch.Tensor], + batch: Dict[str, Union[List, torch.Tensor]], + output_dir: str, + limit: int = 4, + ) -> None: + visualize(predicts, batch, output_dir, limit=limit) diff --git a/tts/VALL-E-X/models/vallex.py b/tts/VALL-E-X/models/vallex.py new file mode 100644 index 0000000..2f35472 --- /dev/null +++ b/tts/VALL-E-X/models/vallex.py @@ -0,0 +1,853 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +from typing import Dict, Iterator, List, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +# from icefall.utils import make_pad_mask +# from torchmetrics.classification import MulticlassAccuracy + +from data.input_strategies import PromptedFeatures +from modules.embedding import SinePositionalEmbedding, TokenEmbedding +from modules.transformer import ( + AdaptiveLayerNorm, + LayerNorm, + TransformerDecoderLayer, + TransformerEncoder, + TransformerEncoderLayer, +) + +from .macros import NUM_AUDIO_TOKENS, NUM_TEXT_TOKENS +from .visualizer import visualize + + +class Transpose(nn.Identity): + """(N, T, D) -> (N, D, T)""" + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return input.transpose(1, 2) + + +# NOTE: There are two ways to implement the model +# 1) [VALL-F] standard TransformerDecoder, use x as memory +# 2) [VALL-E] modified TransformerDecoder like GPT-x(e.g. causal TransformerEncoder), +# use x as the prefix of decoder inputs +class VALLF(nn.Module): + """It implements https://arxiv.org/abs/2301.02111 + "Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers" + """ + + def __init__( + self, + d_model: int, + nhead: int, + num_layers: int, + norm_first: bool = True, + add_prenet: bool = False, + decoder_cls: Union[ + nn.TransformerDecoder, nn.TransformerEncoder + ] = nn.TransformerDecoder, + decoder_layer_cls: Union[ + TransformerDecoderLayer, TransformerEncoderLayer + ] = TransformerDecoderLayer, + prefix_mode: int = 0, + share_embedding: bool = True, + nar_scale_factor: float = 1.0, + prepend_bos: bool = True, + num_quantizers: int = 8, + ): + """ + Args: + d_model: + The number of expected features in the input (required). + nhead: + The number of heads in the multiheadattention models (required). + num_layers: + The number of sub-decoder-layers in the decoder (required). + """ + super().__init__() + nar_d_model = int(d_model * nar_scale_factor) + + self.ar_text_embedding = TokenEmbedding(d_model, NUM_TEXT_TOKENS) # W_x + self.nar_text_embedding = TokenEmbedding(nar_d_model, NUM_TEXT_TOKENS) + + # ID NUM_AUDIO_TOKENS -> PAD + # ID NUM_AUDIO_TOKENS + 1 -> BOS + self.ar_audio_prepend_bos = prepend_bos + self.ar_audio_embedding = TokenEmbedding( + d_model, NUM_AUDIO_TOKENS + 1 + int(prepend_bos) + ) + + # PreNet + if add_prenet: + self.ar_text_prenet = nn.Sequential( + Transpose(), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d(d_model, d_model, kernel_size=5, padding="same"), + nn.BatchNorm1d(d_model), + nn.ReLU(), + nn.Dropout(0.5), + Transpose(), + nn.Linear(d_model, d_model), + ) + + self.ar_audio_prenet = nn.Sequential( + nn.Linear(d_model, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, d_model), + ) + else: + self.ar_text_prenet = nn.Identity() + self.ar_audio_prenet = nn.Identity() + + self.ar_text_position = SinePositionalEmbedding( + d_model, + dropout=0.1, + scale=False, + alpha=True, + ) + self.ar_audio_position = SinePositionalEmbedding( + d_model, + dropout=0.1, + scale=False, + alpha=True, + ) + + self.ar_decoder = decoder_cls( + decoder_layer_cls( + d_model, + nhead, + dim_feedforward=d_model * 4, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + ), + num_layers=num_layers, + norm=LayerNorm(d_model) if norm_first else None, + ) + self.ar_predict_layer = nn.Linear( + d_model, NUM_AUDIO_TOKENS + 1, bias=False + ) + + self.rng = random.Random(0) + self.num_heads = nhead + self.prefix_mode = prefix_mode + self.num_quantizers = num_quantizers + + assert num_quantizers >= 1 + if num_quantizers > 1: + self.nar_audio_embeddings = nn.ModuleList( + [TokenEmbedding(nar_d_model, NUM_AUDIO_TOKENS + 1)] + + [ + TokenEmbedding(nar_d_model, NUM_AUDIO_TOKENS) + for i in range(num_quantizers - 1) + ] + ) # W_a + + # PreNet + if add_prenet: + self.nar_text_prenet = nn.Sequential( + Transpose(), + nn.Conv1d( + nar_d_model, nar_d_model, kernel_size=5, padding="same" + ), + nn.BatchNorm1d(nar_d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d( + nar_d_model, nar_d_model, kernel_size=5, padding="same" + ), + nn.BatchNorm1d(nar_d_model), + nn.ReLU(), + nn.Dropout(0.5), + nn.Conv1d( + nar_d_model, nar_d_model, kernel_size=5, padding="same" + ), + nn.BatchNorm1d(nar_d_model), + nn.ReLU(), + nn.Dropout(0.5), + Transpose(), + nn.Linear(nar_d_model, nar_d_model), + ) + self.nar_audio_prenet = nn.Sequential( + nn.Linear(nar_d_model, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, 256), + nn.ReLU(), + nn.Dropout(0.25), + nn.Linear(256, nar_d_model), + ) + else: + self.nar_text_prenet = nn.Identity() + self.nar_audio_prenet = nn.Identity() + + self.nar_text_position = SinePositionalEmbedding( + nar_d_model, + dropout=0.0, + scale=False, + alpha=False, + ) + self.nar_audio_position = SinePositionalEmbedding( + nar_d_model, + dropout=0.1, + scale=False, + alpha=False, + ) + + self.nar_decoder = decoder_cls( + decoder_layer_cls( + nar_d_model, + int(nhead * nar_scale_factor), + dim_feedforward=nar_d_model * 4, + dropout=0.1, + batch_first=True, + norm_first=norm_first, + adaptive_layer_norm=True, + ), + num_layers=int(num_layers * nar_scale_factor), + norm=AdaptiveLayerNorm( + nar_d_model, norm=nn.LayerNorm(nar_d_model) + ) + if norm_first + else None, + ) + self.nar_predict_layers = nn.ModuleList( + [ + nn.Linear(nar_d_model, NUM_AUDIO_TOKENS, bias=False) + for i in range(num_quantizers - 1) + ] + ) + self.nar_stage_embeddings = nn.ModuleList( + [ + TokenEmbedding(nar_d_model, 1) + for i in range(num_quantizers - 1) + ] + ) + + if share_embedding: + # We share the parameters of the output projection layer with the parameters of the acoustic embedding Wa + # NOTE(Feiteng): In the experiment, this undermines accuracy + # self.ar_predict_layer.weight = self.ar_audio_embedding.weight + + # We also share the parameters of the acoustic embedding layer and the output prediction layer, + # which means the weights of the j-th prediction layer are the same as the (j + 1)-th acoustic embedding layer. + for j in range(0, num_quantizers - 2): + self.nar_predict_layers[ + j + ].weight = self.nar_audio_embeddings[j + 2].weight + + def stage_parameters(self, stage: int = 1) -> Iterator[nn.Parameter]: + assert stage > 0 + if stage == 1: + for name, param in self.named_parameters(): + if name.startswith("ar_"): + print(f" AR parameter: {name}") + yield param + + if stage == 2: + for name, param in self.named_parameters(): + if name.startswith("nar_"): + print(f"NAR parameter: {name}") + yield param + + def stage_named_parameters( + self, stage: int = 1 + ) -> Iterator[Tuple[str, nn.Parameter]]: + assert stage > 0 + if stage == 1: + for pair in self.named_parameters(): + if pair[0].startswith("ar_"): + yield pair + + if stage == 2: + for pair in self.named_parameters(): + if pair[0].startswith("nar_"): + yield pair + + def pad_y_eos(self, y, y_mask_int, eos_id): + targets = F.pad(y, (0, 1), value=0) + eos_id * F.pad( + y_mask_int, (0, 1), value=1 + ) + # inputs, targets + if self.ar_audio_prepend_bos: + return ( + F.pad(targets[:, :-1], (1, 0), value=NUM_AUDIO_TOKENS + 1), + targets, + ) + + return targets[:, :-1], targets[:, 1:] + + def _prepare_prompts(self, y, y_lens, codes, nar_stage, y_prompts_codes, prefix_mode): + # 5.1 For the NAR acoustic prompt tokens, we select a random segment waveform of 3 seconds + # from the same utterance. + # We implement this differently. + if prefix_mode == 0: + # no prefix + prefix_len = 0 + y_emb = self.nar_audio_embeddings[0](y) + for j in range(1, nar_stage): + # Formula (4) (5) + y_emb = y_emb + self.nar_audio_embeddings[j](codes[..., j]) + elif prefix_mode == 1: + # prefix at begining + int_low = (0.25 * y_lens.min()).type(torch.int64).item() + prefix_len = torch.randint(0, int_low * 2, size=()).item() + prefix_len = min(prefix_len, 225) # 24000/320 * 3s = 225 frames + + y_prompts = self.nar_audio_embeddings[0](y[:, :prefix_len]) + y_emb = self.nar_audio_embeddings[0](y[:, prefix_len:]) + for j in range(1, self.num_quantizers): + y_prompts += self.nar_audio_embeddings[j]( + codes[:, :prefix_len, j] + ) + if j < nar_stage: + y_emb += self.nar_audio_embeddings[j]( + codes[:, prefix_len:, j] + ) + y_emb = torch.concat([y_prompts, y_emb], axis=1) + elif prefix_mode in [2, 4]: + if prefix_mode == 2: + # random prefix + prefix_len = min(225, int(0.25 * y_lens.min().item())) + + y_prompts_codes = [] + for b in range(codes.shape[0]): + start = self.rng.randint(0, y_lens[b].item() - prefix_len) + y_prompts_codes.append( + torch.clone(codes[b, start : start + prefix_len]) + ) + codes[ + b, start : start + prefix_len, nar_stage + ] = NUM_AUDIO_TOKENS + y_prompts_codes = torch.stack(y_prompts_codes, dim=0) + else: + prefix_len = y_prompts_codes.shape[1] + + y_prompts = self.nar_audio_embeddings[0](y_prompts_codes[..., 0]) + y_emb = self.nar_audio_embeddings[0](y) + for j in range(1, self.num_quantizers): + y_prompts += self.nar_audio_embeddings[j]( + y_prompts_codes[..., j] + ) + if j < nar_stage: + y_emb += self.nar_audio_embeddings[j](codes[..., j]) + y_emb = torch.concat([y_prompts, y_emb], axis=1) + else: + raise ValueError + + return y_emb, prefix_len + + def forward( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: Union[torch.Tensor, PromptedFeatures], + y_lens: Union[torch.Tensor, PromptedFeatures], + reduction: str = "sum", + train_stage: int = 0, + **kwargs, + ) -> Tuple[torch.Tensor, Union[torch.Tensor, None]]: + raise NotImplementedError + + def inference( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: torch.Tensor, + enroll_x_lens: Union[torch.Tensor, None] = None, + top_k: int = -100, + temperature: float = 1.0, + ) -> torch.Tensor: + raise NotImplementedError + + def visualize( + self, + predicts: Tuple[torch.Tensor], + batch: Dict[str, Union[List, torch.Tensor]], + output_dir: str, + limit: int = 4, + ) -> None: + raise NotImplementedError + + +class VALLE(VALLF): + """It implements https://arxiv.org/abs/2301.02111 + "Neural Codec Language Models are Zero-Shot Text to Speech Synthesizers" + """ + + def __init__( + self, + d_model: int, + nhead: int, + num_layers: int, + norm_first: bool = True, + add_prenet: bool = False, + prefix_mode: int = 0, + share_embedding: bool = True, + nar_scale_factor: float = 1.0, + **kwargs, + ): + """ + Args: + d_model: + The number of expected features in the input (required). + nhead: + The number of heads in the multiheadattention models (required). + num_layers: + The number of sub-decoder-layers in the decoder (required). + """ + super(VALLE, self).__init__( + d_model, + nhead, + num_layers, + norm_first=norm_first, + add_prenet=add_prenet, + decoder_cls=TransformerEncoder, + decoder_layer_cls=TransformerEncoderLayer, + prefix_mode=prefix_mode, + share_embedding=share_embedding, + nar_scale_factor=nar_scale_factor, + **kwargs, + ) + self.language_ID = { + 'en': 0, + 'zh': 1, + 'ja': 2, + } + self.ar_language_embedding = TokenEmbedding(d_model, len(self.language_ID)) + self.nar_language_embedding = TokenEmbedding(d_model, len(self.language_ID)) + + def forward( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: Union[torch.Tensor, PromptedFeatures], + y_lens: Union[torch.Tensor, PromptedFeatures], + reduction: str = "sum", + train_stage: int = 0, + **kwargs, + ): + raise NotImplementedError + def inference( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: torch.Tensor, + enroll_x_lens: torch.Tensor, + top_k: int = -100, + temperature: float = 1.0, + prompt_language: str = None, + text_language: str = None, + best_of: int = 1, + length_penalty: float = 1.0, + return_worst: bool = False, + ) -> torch.Tensor: + """ + Args: + x: + A 2-D tensor of shape (1, S). + x_lens: + A 1-D tensor of shape (1,). It contains the number of tokens in `x` + before padding. + y: + A 3-D tensor of shape (1, T, 8). + top_k: (`optional`) int + The number of highest probability tokens to keep for top-k-filtering. Default to -100. + temperature: (`optional`) float + The value used to module the next token probabilities. Must be strictly positive. Default to 1.0. + Returns: + Return the predicted audio code matrix. + """ + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + assert y.ndim == 3, y.shape + assert y.shape[0] == 1, y.shape + + assert torch.all(x_lens > 0) + + # NOTE: x has been padded in TextTokenCollater + text = x + x = self.ar_text_embedding(text) + # Add language embedding + prompt_language_id = torch.LongTensor(np.array([self.language_ID[prompt_language]])).to(x.device) + if isinstance(text_language, str): + text_language_id = torch.LongTensor(np.array([self.language_ID[text_language]])).to(x.device) + elif isinstance(text_language, List): + text_language_id = torch.LongTensor(np.array([self.language_ID[tl] for tl in text_language])).to(x.device) + x[:, :enroll_x_lens, :] += self.ar_language_embedding(prompt_language_id) + x[:, enroll_x_lens:, :] += self.ar_language_embedding(text_language_id) + x = self.ar_text_prenet(x) + x = self.ar_text_position(x) + + text_len = x_lens.max() + prompts = y + prefix_len = y.shape[1] + + # AR Decoder + # TODO: Managing decoder steps avoid repetitive computation + y = prompts[..., 0] + if self.ar_audio_prepend_bos: + y = F.pad(y, (1, 0), value=NUM_AUDIO_TOKENS + 1) + + x_len = x_lens.max() + x_attn_mask = torch.zeros((x_len, x_len), dtype=torch.bool) + + kv_cache = None + use_kv_caching = True + + sum_logprobs = torch.zeros(best_of, device=y.device) # implement batch decoding here + x = x.repeat(best_of, 1, 1) + y = y.repeat(best_of, 1) + while True: + y_emb = self.ar_audio_embedding(y) + y_emb = self.ar_audio_prenet(y_emb) + y_pos = self.ar_audio_position(y_emb) + xy_pos = torch.concat([x, y_pos], dim=1) + + y_len = y.shape[1] + x_attn_mask_pad = F.pad( + x_attn_mask, + (0, y_len), + value=True, + ) + y_attn_mask = F.pad( + torch.triu( + torch.ones(y_len, y_len, dtype=torch.bool), diagonal=1 + ), + (x_len, 0), + value=False, + ) + xy_attn_mask = torch.concat( + [x_attn_mask_pad, y_attn_mask], dim=0 + ).to(y.device) + + + if use_kv_caching and kv_cache is not None: + xy_pos = xy_pos[:, [-1]] + else: + pass + + xy_dec, kv_cache = self.ar_decoder.infer( + xy_pos, + mask=xy_attn_mask, + past_kv=kv_cache, + use_cache=use_kv_caching, + ) + # xy_dec, _ = self.ar_decoder( + # (xy_pos, None), + # mask=xy_attn_mask, + # ) + + logits = self.ar_predict_layer(xy_dec[:, -1]) + samples, current_logprobs = topk_sampling( + logits, top_k=top_k, top_p=1, temperature=temperature + ) + sum_logprobs += current_logprobs * (y[:, -1] != NUM_AUDIO_TOKENS) + samples[y[:, -1] == NUM_AUDIO_TOKENS] = NUM_AUDIO_TOKENS + completed = (samples[:, -1] == NUM_AUDIO_TOKENS).all() + if ( + completed + or (y.shape[1] - prompts.shape[1]) > x_lens.max() * 16 + ): + if prompts.shape[1] == y.shape[1]: + raise SyntaxError( + "well trained model shouldn't reach here." + ) + lengths = torch.sum(y != NUM_AUDIO_TOKENS, dim=1) + avg_logprobs = sum_logprobs / lengths ** length_penalty + # choose the best beam according to sum_logprobs + best_beam = y[torch.argmax(avg_logprobs), :] + worst_beam = y[torch.argmin(avg_logprobs), :] + # strip all eos tokens + best_beam = best_beam[best_beam != NUM_AUDIO_TOKENS] + worst_beam = worst_beam[worst_beam != NUM_AUDIO_TOKENS] + if return_worst: + y = worst_beam.unsqueeze(0) + else: + y = best_beam.unsqueeze(0) + print(f"VALL-E EOS [{prompts.shape[1]} -> {y.shape[1]}]") + break + + y = torch.concat([y, samples], dim=1) + + codes = [y[:, prefix_len + int(self.ar_audio_prepend_bos) :]] + if self.num_quantizers == 1: + return torch.stack(codes, dim=-1) + + # Non-AR Decoders + y_emb = self.nar_audio_embeddings[0]( + y[:, int(self.ar_audio_prepend_bos) :] + ) + + if self.prefix_mode in [2, 4]: # Exclude enrolled_phonemes + enrolled_len = enroll_x_lens.max().item() + # SOS + Synthesis Text + EOS + text = torch.concat( + [ + text[:, :1], + text[:, enrolled_len - 1 :], + ], + dim=1, + ) + text_len = text_len - (enrolled_len - 2) + assert text.shape[0] == 1 + + x = self.nar_text_embedding(text) + # Add language embedding + prompt_language_id = torch.LongTensor(np.array([self.language_ID[prompt_language]])).to(x.device) + if isinstance(text_language, str): + text_language_id = torch.LongTensor(np.array([self.language_ID[text_language]])).to(x.device) + elif isinstance(text_language, List): + text_language_id = torch.LongTensor(np.array([self.language_ID[tl] for tl in text_language])).to(x.device) + x[:, :enroll_x_lens, :] += self.nar_language_embedding(prompt_language_id) + x[:, enroll_x_lens:, :] += self.nar_language_embedding(text_language_id) + x = self.nar_text_prenet(x) + x = self.nar_text_position(x) + + if self.prefix_mode == 0: + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < self.num_quantizers - 2: + y_emb[:, :prefix_len] += embedding_layer( + prompts[..., i + 1] + ) + y_emb[:, prefix_len:] += embedding_layer(samples) + else: + for j in range(1, self.num_quantizers): + y_emb[:, :prefix_len] += self.nar_audio_embeddings[j]( + prompts[..., j] + ) + + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < self.num_quantizers - 2: + y_emb[:, prefix_len:] += embedding_layer(samples) + + assert len(codes) == self.num_quantizers + return torch.stack(codes, dim=-1) + + def continual( + self, + x: torch.Tensor, + x_lens: torch.Tensor, + y: torch.Tensor, + ) -> torch.Tensor: + """ + Args: + x: + A 2-D tensor of shape (1, S). + x_lens: + A 1-D tensor of shape (1,). It contains the number of tokens in `x` + before padding. + y: + A 3-D tensor of shape (1, T, 8). + Returns: + Return the predicted audio code matrix. + """ + assert x.ndim == 2, x.shape + assert x_lens.ndim == 1, x_lens.shape + assert y.ndim == 3, y.shape + assert y.shape[0] == 1, y.shape + + assert torch.all(x_lens > 0) + assert self.num_quantizers == 8 + + # NOTE: x has been padded in TextTokenCollater + text = x + x = self.ar_text_embedding(text) + x = self.ar_text_prenet(x) + x = self.ar_text_position(x) + + text_len = x_lens.max() + + prefix_len = min(int(y.shape[1] * 0.5), 3 * 75) + + # AR Decoder + prompts = y[:, :prefix_len] + + codes = [y[:, prefix_len:, 0]] + # Non-AR Decoders + x = self.nar_text_embedding(text) + x = self.nar_text_prenet(x) + x = self.nar_text_position(x) + + y_emb = self.nar_audio_embeddings[0](y[..., 0]) + + if self.prefix_mode == 0: + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_position(y_emb) + y_pos = self.nar_audio_prenet(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < 6: + y_emb[:, :prefix_len] += embedding_layer( + prompts[..., i + 1] + ) + y_emb[:, prefix_len:] += embedding_layer(samples) + else: + for j in range(1, 8): + y_emb[:, :prefix_len] += self.nar_audio_embeddings[j]( + prompts[..., j] + ) + + for i, (predict_layer, embedding_layer) in enumerate( + zip( + self.nar_predict_layers, + self.nar_audio_embeddings[1:], + ) + ): + y_pos = self.nar_audio_prenet(y_emb) + y_pos = self.nar_audio_position(y_pos) + xy_pos = torch.concat([x, y_pos], dim=1) + + xy_dec, _ = self.nar_decoder( + (xy_pos, self.nar_stage_embeddings[i].weight) + ) + logits = predict_layer(xy_dec[:, text_len + prefix_len :]) + + samples = torch.argmax(logits, dim=-1) + codes.append(samples) + + if i < 6: + y_emb[:, prefix_len:] += embedding_layer(samples) + + assert len(codes) == 8 + return torch.stack(codes, dim=-1) + + +# https://github.com/microsoft/unilm/blob/master/xtune/src/transformers/modeling_utils.py +def top_k_top_p_filtering( + logits, top_k=0, top_p=1.0, filter_value=-float("Inf"), min_tokens_to_keep=1 +): + """Filter a distribution of logits using top-k and/or nucleus (top-p) filtering + Args: + logits: logits distribution shape (batch size, vocabulary size) + if top_k > 0: keep only top k tokens with highest probability (top-k filtering). + if top_p < 1.0: keep the top tokens with cumulative probability >= top_p (nucleus filtering). + Nucleus filtering is described in Holtzman et al. (http://arxiv.org/abs/1904.09751) + Make sure we keep at least min_tokens_to_keep per batch example in the output + From: https://gist.github.com/thomwolf/1a5a29f6962089e871b94cbd09daf317 + """ + if top_k > 0: + top_k = min( + max(top_k, min_tokens_to_keep), logits.size(-1) + ) # Safety check + # Remove all tokens with a probability less than the last token of the top-k + indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] + logits[indices_to_remove] = filter_value + + if top_p < 1.0: + sorted_logits, sorted_indices = torch.sort(logits, descending=True) + cumulative_probs = torch.cumsum( + F.softmax(sorted_logits, dim=-1), dim=-1 + ) + + # Remove tokens with cumulative probability above the threshold (token with 0 are kept) + sorted_indices_to_remove = cumulative_probs > top_p + if min_tokens_to_keep > 1: + # Keep at least min_tokens_to_keep (set to min_tokens_to_keep-1 because we add the first one below) + sorted_indices_to_remove[..., :min_tokens_to_keep] = 0 + # Shift the indices to the right to keep also the first token above the threshold + sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[ + ..., :-1 + ].clone() + sorted_indices_to_remove[..., 0] = 0 + + # scatter sorted tensors to original indexing + indices_to_remove = sorted_indices_to_remove.scatter( + 1, sorted_indices, sorted_indices_to_remove + ) + logits[indices_to_remove] = filter_value + return logits + + +def topk_sampling(logits, top_k=10, top_p=1.0, temperature=1.0): + # temperature: (`optional`) float + # The value used to module the next token probabilities. Must be strictly positive. Default to 1.0. + # top_k: (`optional`) int + # The number of highest probability vocabulary tokens to keep for top-k-filtering. Between 1 and infinity. Default to 50. + # top_p: (`optional`) float + # The cumulative probability of parameter highest probability vocabulary tokens to keep for nucleus sampling. Must be between 0 and 1. Default to 1. + + # Temperature (higher temperature => more likely to sample low probability tokens) + if temperature != 1.0: + logits = logits / temperature + # Top-p/top-k filtering + logits = top_k_top_p_filtering(logits, top_k=top_k, top_p=top_p) + # Sample + token = torch.multinomial(F.softmax(logits, dim=-1), num_samples=1) + logprobs = F.log_softmax(logits.float(), dim=-1) + current_logprobs = logprobs[torch.arange(logprobs.shape[0]), token.squeeze(1)] + return token, current_logprobs diff --git a/tts/VALL-E-X/models/visualizer.py b/tts/VALL-E-X/models/visualizer.py new file mode 100644 index 0000000..160390f --- /dev/null +++ b/tts/VALL-E-X/models/visualizer.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright 2023 (authors: Feiteng Li) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Dict, List, Tuple, Union + +import matplotlib.pyplot as plt +import numpy as np +import torch + + +def visualize( + predicts: Tuple[torch.Tensor], + batch: Dict[str, Union[List, torch.Tensor]], + output_dir: str, + limit: int = 4, +) -> None: + text_tokens = batch["text_tokens"].to("cpu").detach().numpy() + text_tokens_lens = batch["text_tokens_lens"].to("cpu").detach().numpy() + audio_features = batch["audio_features"].to("cpu").detach().numpy() + audio_features_lens = ( + batch["audio_features_lens"].to("cpu").detach().numpy() + ) + assert text_tokens.ndim == 2 + + utt_ids, texts = batch["utt_id"], batch["text"] + + encoder_outputs = predicts[0].to("cpu").type(torch.float32).detach().numpy() + decoder_outputs = predicts[1] + if isinstance(decoder_outputs, list): + decoder_outputs = decoder_outputs[-1] + decoder_outputs = ( + decoder_outputs.to("cpu").type(torch.float32).detach().numpy() + ) + + vmin, vmax = 0, 1024 # Encodec + if decoder_outputs.dtype == np.float32: + vmin, vmax = -6, 0 # Fbank + + num_figures = 3 + for b, (utt_id, text) in enumerate(zip(utt_ids[:limit], texts[:limit])): + _ = plt.figure(figsize=(14, 8 * num_figures)) + + S = text_tokens_lens[b] + T = audio_features_lens[b] + + # encoder + plt.subplot(num_figures, 1, 1) + plt.title(f"Text: {text}") + plt.imshow( + X=np.transpose(encoder_outputs[b]), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + ) + plt.gca().invert_yaxis() + plt.axvline(x=S - 0.4, linewidth=2, color="r") + plt.xlabel("Encoder Output") + plt.colorbar() + + # decoder + plt.subplot(num_figures, 1, 2) + plt.imshow( + X=np.transpose(decoder_outputs[b]), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + vmin=vmin, + vmax=vmax, + ) + plt.gca().invert_yaxis() + plt.axvline(x=T - 0.4, linewidth=2, color="r") + plt.xlabel("Decoder Output") + plt.colorbar() + + # target + plt.subplot(num_figures, 1, 3) + plt.imshow( + X=np.transpose(audio_features[b]), + cmap=plt.get_cmap("jet"), + aspect="auto", + interpolation="nearest", + vmin=vmin, + vmax=vmax, + ) + plt.gca().invert_yaxis() + plt.axvline(x=T - 0.4, linewidth=2, color="r") + plt.xlabel("Decoder Target") + plt.colorbar() + + plt.savefig(f"{output_dir}/{utt_id}.png") + plt.close() diff --git a/tts/VALL-E-X/modules/__init__.py b/tts/VALL-E-X/modules/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tts/VALL-E-X/modules/activation.py b/tts/VALL-E-X/modules/activation.py new file mode 100644 index 0000000..e51c8eb --- /dev/null +++ b/tts/VALL-E-X/modules/activation.py @@ -0,0 +1,612 @@ +from typing import Optional, Tuple, List +import math + +import torch +from torch import Tensor +from torch.nn import Linear, Module +from torch.nn import functional as F +from torch.nn.init import constant_, xavier_normal_, xavier_uniform_ +from torch.nn.modules.linear import NonDynamicallyQuantizableLinear +from torch.nn.parameter import Parameter + +def _in_projection_packed( + q: Tensor, + k: Tensor, + v: Tensor, + w: Tensor, + b: Optional[Tensor] = None, +) -> List[Tensor]: + r""" + Performs the in-projection step of the attention operation, using packed weights. + Output is a triple containing projection tensors for query, key and value. + + Args: + q, k, v: query, key and value tensors to be projected. For self-attention, + these are typically the same tensor; for encoder-decoder attention, + k and v are typically the same tensor. (We take advantage of these + identities for performance if they are present.) Regardless, q, k and v + must share a common embedding dimension; otherwise their shapes may vary. + w: projection weights for q, k and v, packed into a single tensor. Weights + are packed along dimension 0, in q, k, v order. + b: optional projection biases for q, k and v, packed into a single tensor + in q, k, v order. + + Shape: + Inputs: + - q: :math:`(..., E)` where E is the embedding dimension + - k: :math:`(..., E)` where E is the embedding dimension + - v: :math:`(..., E)` where E is the embedding dimension + - w: :math:`(E * 3, E)` where E is the embedding dimension + - b: :math:`E * 3` where E is the embedding dimension + + Output: + - in output list :math:`[q', k', v']`, each output tensor will have the + same shape as the corresponding input tensor. + """ + E = q.size(-1) + if k is v: + if q is k: + # self-attention + return F.linear(q, w, b).chunk(3, dim=-1) + else: + # encoder-decoder attention + w_q, w_kv = w.split([E, E * 2]) + if b is None: + b_q = b_kv = None + else: + b_q, b_kv = b.split([E, E * 2]) + return (F.linear(q, w_q, b_q),) + F.linear(k, w_kv, b_kv).chunk(2, dim=-1) + else: + w_q, w_k, w_v = w.chunk(3) + if b is None: + b_q = b_k = b_v = None + else: + b_q, b_k, b_v = b.chunk(3) + return F.linear(q, w_q, b_q), F.linear(k, w_k, b_k), F.linear(v, w_v, b_v) + +def _scaled_dot_product_attention( + q: Tensor, + k: Tensor, + v: Tensor, + attn_mask: Optional[Tensor] = None, + dropout_p: float = 0.0, +) -> Tuple[Tensor, Tensor]: + r""" + Computes scaled dot product attention on query, key and value tensors, using + an optional attention mask if passed, and applying dropout if a probability + greater than 0.0 is specified. + Returns a tensor pair containing attended values and attention weights. + + Args: + q, k, v: query, key and value tensors. See Shape section for shape details. + attn_mask: optional tensor containing mask values to be added to calculated + attention. May be 2D or 3D; see Shape section for details. + dropout_p: dropout probability. If greater than 0.0, dropout is applied. + + Shape: + - q: :math:`(B, Nt, E)` where B is batch size, Nt is the target sequence length, + and E is embedding dimension. + - key: :math:`(B, Ns, E)` where B is batch size, Ns is the source sequence length, + and E is embedding dimension. + - value: :math:`(B, Ns, E)` where B is batch size, Ns is the source sequence length, + and E is embedding dimension. + - attn_mask: either a 3D tensor of shape :math:`(B, Nt, Ns)` or a 2D tensor of + shape :math:`(Nt, Ns)`. + + - Output: attention values have shape :math:`(B, Nt, E)`; attention weights + have shape :math:`(B, Nt, Ns)` + """ + B, Nt, E = q.shape + q = q / math.sqrt(E) + # (B, Nt, E) x (B, E, Ns) -> (B, Nt, Ns) + if attn_mask is not None: + attn = torch.baddbmm(attn_mask, q, k.transpose(-2, -1)) + else: + attn = torch.bmm(q, k.transpose(-2, -1)) + + attn = F.softmax(attn, dim=-1) + if dropout_p > 0.0: + attn = F.dropout(attn, p=dropout_p) + # (B, Nt, Ns) x (B, Ns, E) -> (B, Nt, E) + output = torch.bmm(attn, v) + return output, attn + +def multi_head_attention_forward( + x, + ipw, + ipb, + opw, + opb, + n_head, + attn_mask, + past_kv=None, + use_cache=False, +): + # x = x.transpose(1, 0) + # tgt_len, bsz, embed_dim = x.shape + # head_dim = embed_dim // n_head + # q, k, v = _in_projection_packed(x, x, x, ipw, ipb) + # q = q.contiguous().view(tgt_len, bsz * n_head, head_dim).transpose(0, 1) + # k = k.contiguous().view(k.shape[0], bsz * n_head, head_dim).transpose(0, 1) + # v = v.contiguous().view(v.shape[0], bsz * n_head, head_dim).transpose(0, 1) + + # new_attn_mask = torch.zeros_like(attn_mask, dtype=q.dtype) + # new_attn_mask.masked_fill_(attn_mask, float("-inf")) + # attn_mask = new_attn_mask + # + # attn_output, attn_output_weights = _scaled_dot_product_attention(q, k, v, attn_mask, 0.0) + # attn_output = attn_output.transpose(0, 1).contiguous().view(tgt_len * bsz, embed_dim) + # attn_output = torch._C._nn.linear(attn_output, opw, opb) + # attn_output = attn_output.view(tgt_len, bsz, attn_output.size(1)) + + B, T, C = x.size() + + q, k, v = torch._C._nn.linear(x, ipw, ipb).chunk(3, dim=-1) + k = k.view(B, T, n_head, C // n_head).transpose(1, 2) # (B, nh, T, hs) + q = q.view(B, T, n_head, C // n_head).transpose(1, 2) # (B, nh, T, hs) + v = v.view(B, T, n_head, C // n_head).transpose(1, 2) # (B, nh, T, hs) + if past_kv is not None: + past_key = past_kv[0] + past_value = past_kv[1] + k = torch.cat((past_key, k), dim=-2) + v = torch.cat((past_value, v), dim=-2) + + FULL_T = k.shape[-2] + + if use_cache is True: + present = (k, v) + else: + present = None + + att = (q @ k.transpose(-2, -1)) * (1.0 / math.sqrt(k.size(-1))) + att = att.masked_fill(attn_mask[FULL_T - T:FULL_T, :FULL_T], float('-inf')) + att = F.softmax(att, dim=-1) + y = att @ v # (B, nh, T, T) x (B, nh, T, hs) -> (B, nh, T, hs) + y = y.transpose(1, 2).contiguous().view(B, T, C) # re-assemble all head outputs side by side + y = torch._C._nn.linear(y, opw, opb) + return (y, present) + + +class MultiheadAttention(Module): + r"""Allows the model to jointly attend to information + from different representation subspaces as described in the paper: + `Attention Is All You Need `_. + + Multi-Head Attention is defined as: + + .. math:: + \text{MultiHead}(Q, K, V) = \text{Concat}(head_1,\dots,head_h)W^O + + where :math:`head_i = \text{Attention}(QW_i^Q, KW_i^K, VW_i^V)`. + + ``forward()`` will use a special optimized implementation if all of the following + conditions are met: + + - self attention is being computed (i.e., ``query``, ``key``, and ``value`` are the same tensor. This + restriction will be loosened in the future.) + - Either autograd is disabled (using ``torch.inference_mode`` or ``torch.no_grad``) or no tensor argument ``requires_grad`` + - training is disabled (using ``.eval()``) + - dropout is 0 + - ``add_bias_kv`` is ``False`` + - ``add_zero_attn`` is ``False`` + - ``batch_first`` is ``True`` and the input is batched + - ``kdim`` and ``vdim`` are equal to ``embed_dim`` + - at most one of ``key_padding_mask`` or ``attn_mask`` is passed + - if a `NestedTensor `_ is passed, neither ``key_padding_mask`` + nor ``attn_mask`` is passed + + If the optimized implementation is in use, a + `NestedTensor `_ can be passed for + ``query``/``key``/``value`` to represent padding more efficiently than using a + padding mask. In this case, a `NestedTensor `_ + will be returned, and an additional speedup proportional to the fraction of the input + that is padding can be expected. + + Args: + embed_dim: Total dimension of the model. + num_heads: Number of parallel attention heads. Note that ``embed_dim`` will be split + across ``num_heads`` (i.e. each head will have dimension ``embed_dim // num_heads``). + dropout: Dropout probability on ``attn_output_weights``. Default: ``0.0`` (no dropout). + bias: If specified, adds bias to input / output projection layers. Default: ``True``. + add_bias_kv: If specified, adds bias to the key and value sequences at dim=0. Default: ``False``. + add_zero_attn: If specified, adds a new batch of zeros to the key and value sequences at dim=1. + Default: ``False``. + kdim: Total number of features for keys. Default: ``None`` (uses ``kdim=embed_dim``). + vdim: Total number of features for values. Default: ``None`` (uses ``vdim=embed_dim``). + batch_first: If ``True``, then the input and output tensors are provided + as (batch, seq, feature). Default: ``False`` (seq, batch, feature). + + Examples:: + + >>> # xdoctest: +SKIP + >>> multihead_attn = nn.MultiheadAttention(embed_dim, num_heads) + >>> attn_output, attn_output_weights = multihead_attn(query, key, value) + + """ + __constants__ = ["batch_first"] + bias_k: Optional[torch.Tensor] + bias_v: Optional[torch.Tensor] + + def __init__( + self, + embed_dim, + num_heads, + dropout=0.0, + bias=True, + add_bias_kv=False, + add_zero_attn=False, + kdim=None, + vdim=None, + batch_first=False, + linear1_cls=Linear, + linear2_cls=Linear, + device=None, + dtype=None, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(MultiheadAttention, self).__init__() + self.embed_dim = embed_dim + self.kdim = kdim if kdim is not None else embed_dim + self.vdim = vdim if vdim is not None else embed_dim + self._qkv_same_embed_dim = ( + self.kdim == embed_dim and self.vdim == embed_dim + ) + + self.num_heads = num_heads + self.dropout = dropout + self.batch_first = batch_first + self.head_dim = embed_dim // num_heads + assert ( + self.head_dim * num_heads == self.embed_dim + ), "embed_dim must be divisible by num_heads" + + if add_bias_kv: + self.bias_k = Parameter( + torch.empty((1, 1, embed_dim), **factory_kwargs) + ) + self.bias_v = Parameter( + torch.empty((1, 1, embed_dim), **factory_kwargs) + ) + else: + self.bias_k = self.bias_v = None + + if linear1_cls == Linear: + if not self._qkv_same_embed_dim: + self.q_proj_weight = Parameter( + torch.empty((embed_dim, embed_dim), **factory_kwargs) + ) + self.k_proj_weight = Parameter( + torch.empty((embed_dim, self.kdim), **factory_kwargs) + ) + self.v_proj_weight = Parameter( + torch.empty((embed_dim, self.vdim), **factory_kwargs) + ) + self.register_parameter("in_proj_weight", None) + else: + self.in_proj_weight = Parameter( + torch.empty((3 * embed_dim, embed_dim), **factory_kwargs) + ) + self.register_parameter("q_proj_weight", None) + self.register_parameter("k_proj_weight", None) + self.register_parameter("v_proj_weight", None) + + if bias: + self.in_proj_bias = Parameter( + torch.empty(3 * embed_dim, **factory_kwargs) + ) + else: + self.register_parameter("in_proj_bias", None) + self.out_proj = NonDynamicallyQuantizableLinear( + embed_dim, embed_dim, bias=bias, **factory_kwargs + ) + + self._reset_parameters() + else: + if not self._qkv_same_embed_dim: + raise NotImplementedError + else: + self.in_proj_linear = linear1_cls( + embed_dim, 3 * embed_dim, bias=bias, **factory_kwargs + ) + self.in_proj_weight = self.in_proj_linear.weight + + self.register_parameter("q_proj_weight", None) + self.register_parameter("k_proj_weight", None) + self.register_parameter("v_proj_weight", None) + + if bias: + self.in_proj_bias = self.in_proj_linear.bias + else: + self.register_parameter("in_proj_bias", None) + + self.out_proj = linear2_cls( + embed_dim, embed_dim, bias=bias, **factory_kwargs + ) + + if self.bias_k is not None: + xavier_normal_(self.bias_k) + if self.bias_v is not None: + xavier_normal_(self.bias_v) + + self.add_zero_attn = add_zero_attn + + def _reset_parameters(self): + if self._qkv_same_embed_dim: + xavier_uniform_(self.in_proj_weight) + else: + xavier_uniform_(self.q_proj_weight) + xavier_uniform_(self.k_proj_weight) + xavier_uniform_(self.v_proj_weight) + + if self.in_proj_bias is not None: + constant_(self.in_proj_bias, 0.0) + constant_(self.out_proj.bias, 0.0) + + if self.bias_k is not None: + xavier_normal_(self.bias_k) + if self.bias_v is not None: + xavier_normal_(self.bias_v) + + def __setstate__(self, state): + # Support loading old MultiheadAttention checkpoints generated by v1.1.0 + if "_qkv_same_embed_dim" not in state: + state["_qkv_same_embed_dim"] = True + + super(MultiheadAttention, self).__setstate__(state) + + def forward( + self, + query: Tensor, + key: Tensor, + value: Tensor, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + average_attn_weights: bool = True, + ) -> Tuple[Tensor, Optional[Tensor]]: + r""" + Args: + query: Query embeddings of shape :math:`(L, E_q)` for unbatched input, :math:`(L, N, E_q)` when ``batch_first=False`` + or :math:`(N, L, E_q)` when ``batch_first=True``, where :math:`L` is the target sequence length, + :math:`N` is the batch size, and :math:`E_q` is the query embedding dimension ``embed_dim``. + Queries are compared against key-value pairs to produce the output. + See "Attention Is All You Need" for more details. + key: Key embeddings of shape :math:`(S, E_k)` for unbatched input, :math:`(S, N, E_k)` when ``batch_first=False`` + or :math:`(N, S, E_k)` when ``batch_first=True``, where :math:`S` is the source sequence length, + :math:`N` is the batch size, and :math:`E_k` is the key embedding dimension ``kdim``. + See "Attention Is All You Need" for more details. + value: Value embeddings of shape :math:`(S, E_v)` for unbatched input, :math:`(S, N, E_v)` when + ``batch_first=False`` or :math:`(N, S, E_v)` when ``batch_first=True``, where :math:`S` is the source + sequence length, :math:`N` is the batch size, and :math:`E_v` is the value embedding dimension ``vdim``. + See "Attention Is All You Need" for more details. + key_padding_mask: If specified, a mask of shape :math:`(N, S)` indicating which elements within ``key`` + to ignore for the purpose of attention (i.e. treat as "padding"). For unbatched `query`, shape should be :math:`(S)`. + Binary and byte masks are supported. + For a binary mask, a ``True`` value indicates that the corresponding ``key`` value will be ignored for + the purpose of attention. For a float mask, it will be directly added to the corresponding ``key`` value. + need_weights: If specified, returns ``attn_output_weights`` in addition to ``attn_outputs``. + Default: ``True``. + attn_mask: If specified, a 2D or 3D mask preventing attention to certain positions. Must be of shape + :math:`(L, S)` or :math:`(N\cdot\text{num\_heads}, L, S)`, where :math:`N` is the batch size, + :math:`L` is the target sequence length, and :math:`S` is the source sequence length. A 2D mask will be + broadcasted across the batch while a 3D mask allows for a different mask for each entry in the batch. + Binary, byte, and float masks are supported. For a binary mask, a ``True`` value indicates that the + corresponding position is not allowed to attend. For a byte mask, a non-zero value indicates that the + corresponding position is not allowed to attend. For a float mask, the mask values will be added to + the attention weight. + average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across + heads. Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an + effect when ``need_weights=True``. Default: ``True`` (i.e. average weights across heads) + + Outputs: + - **attn_output** - Attention outputs of shape :math:`(L, E)` when input is unbatched, + :math:`(L, N, E)` when ``batch_first=False`` or :math:`(N, L, E)` when ``batch_first=True``, + where :math:`L` is the target sequence length, :math:`N` is the batch size, and :math:`E` is the + embedding dimension ``embed_dim``. + - **attn_output_weights** - Only returned when ``need_weights=True``. If ``average_attn_weights=True``, + returns attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or + :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and + :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per + head of shape :math:`(\text{num\_heads}, L, S)` when input is unbatched or :math:`(N, \text{num\_heads}, L, S)`. + + .. note:: + `batch_first` argument is ignored for unbatched inputs. + """ + is_batched = query.dim() == 3 + if key_padding_mask is not None: + _kpm_dtype = key_padding_mask.dtype + if _kpm_dtype != torch.bool and not torch.is_floating_point( + key_padding_mask + ): + raise AssertionError( + "only bool and floating types of key_padding_mask are supported" + ) + why_not_fast_path = "" + if not is_batched: + why_not_fast_path = f"input not batched; expected query.dim() of 3 but got {query.dim()}" + elif query is not key or key is not value: + # When lifting this restriction, don't forget to either + # enforce that the dtypes all match or test cases where + # they don't! + why_not_fast_path = "non-self attention was used (query, key, and value are not the same Tensor)" + elif ( + self.in_proj_bias is not None + and query.dtype != self.in_proj_bias.dtype + ): + why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_bias ({self.in_proj_bias.dtype}) don't match" + elif ( + self.in_proj_weight is not None + and query.dtype != self.in_proj_weight.dtype + ): + # this case will fail anyway, but at least they'll get a useful error message. + why_not_fast_path = f"dtypes of query ({query.dtype}) and self.in_proj_weight ({self.in_proj_weight.dtype}) don't match" + elif self.training: + why_not_fast_path = "training is enabled" + elif not self.batch_first: + why_not_fast_path = "batch_first was not True" + elif self.bias_k is not None: + why_not_fast_path = "self.bias_k was not None" + elif self.bias_v is not None: + why_not_fast_path = "self.bias_v was not None" + elif self.dropout: + why_not_fast_path = f"dropout was {self.dropout}, required zero" + elif self.add_zero_attn: + why_not_fast_path = "add_zero_attn was enabled" + elif not self._qkv_same_embed_dim: + why_not_fast_path = "_qkv_same_embed_dim was not True" + elif attn_mask is not None: + why_not_fast_path = "attn_mask was not None" + elif query.is_nested and key_padding_mask is not None: + why_not_fast_path = ( + "key_padding_mask is not supported with NestedTensor input" + ) + elif self.num_heads % 2 == 1: + why_not_fast_path = "num_heads is odd" + elif torch.is_autocast_enabled(): + why_not_fast_path = "autocast is enabled" + + if not why_not_fast_path: + tensor_args = ( + query, + key, + value, + self.in_proj_weight, + self.in_proj_bias, + self.out_proj.weight, + self.out_proj.bias, + ) + # We have to use list comprehensions below because TorchScript does not support + # generator expressions. + if torch.overrides.has_torch_function(tensor_args): + why_not_fast_path = "some Tensor argument has_torch_function" + elif not all( + [ + (x is None or x.is_cuda or "cpu" in str(x.device)) + for x in tensor_args + ] + ): + why_not_fast_path = ( + "some Tensor argument is neither CUDA nor CPU" + ) + elif torch.is_grad_enabled() and any( + [x is not None and x.requires_grad for x in tensor_args] + ): + why_not_fast_path = ( + "grad is enabled and at least one of query or the " + "input/output projection weights or biases requires_grad" + ) + if not why_not_fast_path: + return torch._native_multi_head_attention( + query, + key, + value, + self.embed_dim, + self.num_heads, + self.in_proj_weight, + self.in_proj_bias, + self.out_proj.weight, + self.out_proj.bias, + key_padding_mask + if key_padding_mask is not None + else attn_mask, + need_weights, + average_attn_weights, + 1 + if key_padding_mask is not None + else 0 + if attn_mask is not None + else None, + ) + + any_nested = query.is_nested or key.is_nested or value.is_nested + assert not any_nested, ( + "MultiheadAttention does not support NestedTensor outside of its fast path. " + + f"The fast path was not hit because {why_not_fast_path}" + ) + + if self.batch_first and is_batched: + # make sure that the transpose op does not affect the "is" property + if key is value: + if query is key: + query = key = value = query.transpose(1, 0) + else: + query, key = [x.transpose(1, 0) for x in (query, key)] + value = key + else: + query, key, value = [ + x.transpose(1, 0) for x in (query, key, value) + ] + + if not self._qkv_same_embed_dim: + attn_output, attn_output_weights = F.multi_head_attention_forward( + query, + key, + value, + self.embed_dim, + self.num_heads, + self.in_proj_weight, + self.in_proj_bias, + self.bias_k, + self.bias_v, + self.add_zero_attn, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + training=self.training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + use_separate_proj_weight=True, + q_proj_weight=self.q_proj_weight, + k_proj_weight=self.k_proj_weight, + v_proj_weight=self.v_proj_weight, + average_attn_weights=average_attn_weights, + ) + else: + attn_output, attn_output_weights = F.multi_head_attention_forward( + query, + key, + value, + self.embed_dim, + self.num_heads, + self.in_proj_weight, + self.in_proj_bias, + self.bias_k, + self.bias_v, + self.add_zero_attn, + self.dropout, + self.out_proj.weight, + self.out_proj.bias, + training=self.training, + key_padding_mask=key_padding_mask, + need_weights=need_weights, + attn_mask=attn_mask, + average_attn_weights=average_attn_weights, + ) + if self.batch_first and is_batched: + return attn_output.transpose(1, 0), attn_output_weights + else: + return attn_output, attn_output_weights + + def infer(self, + x: Tensor, + key_padding_mask: Optional[Tensor] = None, + need_weights: bool = True, + attn_mask: Optional[Tensor] = None, + average_attn_weights: bool = True, + past_kv = None, + use_cache = False + ): + # x = x.transpose(1, 0) + y, kv = multi_head_attention_forward( + x=x, + ipw=self.in_proj_weight, + ipb=self.in_proj_bias, + opw=self.out_proj.weight, + opb=self.out_proj.bias, + n_head=self.num_heads, + attn_mask=attn_mask, + past_kv=past_kv, + use_cache=use_cache, + ) + return (y, kv) diff --git a/tts/VALL-E-X/modules/embedding.py b/tts/VALL-E-X/modules/embedding.py new file mode 100644 index 0000000..17f6c31 --- /dev/null +++ b/tts/VALL-E-X/modules/embedding.py @@ -0,0 +1,97 @@ +# Copyright 2023 (authors: Feiteng Li) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import torch +import torch.nn as nn + + +class TokenEmbedding(nn.Module): + def __init__( + self, + dim_model: int, + vocab_size: int, + dropout: float = 0.0, + ): + super().__init__() + + self.vocab_size = vocab_size + self.dim_model = dim_model + + self.dropout = torch.nn.Dropout(p=dropout) + self.word_embeddings = nn.Embedding(self.vocab_size, self.dim_model) + + @property + def weight(self) -> torch.Tensor: + return self.word_embeddings.weight + + def embedding(self, index: int) -> torch.Tensor: + return self.word_embeddings.weight[index : index + 1] + + def forward(self, x: torch.Tensor): + X = self.word_embeddings(x) + X = self.dropout(X) + + return X + + +class SinePositionalEmbedding(nn.Module): + def __init__( + self, + dim_model: int, + dropout: float = 0.0, + scale: bool = False, + alpha: bool = False, + ): + super().__init__() + self.dim_model = dim_model + self.x_scale = math.sqrt(dim_model) if scale else 1.0 + self.alpha = nn.Parameter(torch.ones(1), requires_grad=alpha) + self.dropout = torch.nn.Dropout(p=dropout) + + self.reverse = False + self.pe = None + self.extend_pe(torch.tensor(0.0).expand(1, 4000)) + + def extend_pe(self, x): + """Reset the positional encodings.""" + if self.pe is not None: + if self.pe.size(1) >= x.size(1): + if self.pe.dtype != x.dtype or self.pe.device != x.device: + self.pe = self.pe.to(dtype=x.dtype, device=x.device) + return + pe = torch.zeros(x.size(1), self.dim_model) + if self.reverse: + position = torch.arange( + x.size(1) - 1, -1, -1.0, dtype=torch.float32 + ).unsqueeze(1) + else: + position = torch.arange( + 0, x.size(1), dtype=torch.float32 + ).unsqueeze(1) + div_term = torch.exp( + torch.arange(0, self.dim_model, 2, dtype=torch.float32) + * -(math.log(10000.0) / self.dim_model) + ) + pe[:, 0::2] = torch.sin(position * div_term) + pe[:, 1::2] = torch.cos(position * div_term) + pe = pe.unsqueeze(0) + self.pe = pe.to(device=x.device, dtype=x.dtype).detach() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + self.extend_pe(x) + output = x.unsqueeze(-1) if x.ndim == 2 else x + output = output * self.x_scale + self.alpha * self.pe[:, : x.size(1)] + return self.dropout(output) diff --git a/tts/VALL-E-X/modules/optim.py b/tts/VALL-E-X/modules/optim.py new file mode 100644 index 0000000..8e59083 --- /dev/null +++ b/tts/VALL-E-X/modules/optim.py @@ -0,0 +1,1105 @@ +# Copyright 2022 Xiaomi Corp. (authors: Daniel Povey) +# +# See ../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import logging +import random +from collections import defaultdict +from typing import List, Optional, Tuple, Union + +import torch +from lhotse.utils import fix_random_seed +from torch import Tensor +from torch.optim import Optimizer + + +class BatchedOptimizer(Optimizer): + """ + This class adds to class Optimizer the capability to optimize parameters in batches: + it will stack the parameters and their grads for you so the optimizer can work + on tensors with an extra leading dimension. This is intended for speed with GPUs, + as it reduces the number of kernels launched in the optimizer. + + Args: + params: + """ + + def __init__(self, params, defaults): + super(BatchedOptimizer, self).__init__(params, defaults) + + @contextlib.contextmanager + def batched_params(self, param_group, group_params_names): + """ + This function returns (technically, yields) a list of + of tuples (p, state), where + p is a `fake` parameter that is stacked (over axis 0) from real parameters + that share the same shape, and its gradient is also stacked; + `state` is the state corresponding to this batch of parameters + (it will be physically located in the "state" for one of the real + parameters, the last one that has any particular shape and dtype). + + This function is decorated as a context manager so that it can + write parameters back to their "real" locations. + + The idea is, instead of doing: + + for p in group["params"]: + state = self.state[p] + ... + + you can do: + + with self.batched_params(group["params"]) as batches: + for p, state, p_names in batches: + ... + + + Args: + group: a parameter group, which is a list of parameters; should be + one of self.param_groups. + group_params_names: name for each parameter in group, + which is List[str]. + """ + batches = defaultdict( + list + ) # `batches` maps from tuple (dtype_as_str,*shape) to list of nn.Parameter + batches_names = defaultdict( + list + ) # `batches` maps from tuple (dtype_as_str,*shape) to list of str + + assert len(param_group) == len(group_params_names) + for p, named_p in zip(param_group, group_params_names): + key = (str(p.dtype), *p.shape) + batches[key].append(p) + batches_names[key].append(named_p) + + batches_names_keys = list(batches_names.keys()) + sorted_idx = sorted( + range(len(batches_names)), key=lambda i: batches_names_keys[i] + ) + batches_names = [ + batches_names[batches_names_keys[idx]] for idx in sorted_idx + ] + batches = [batches[batches_names_keys[idx]] for idx in sorted_idx] + + stacked_params_dict = dict() + + # turn batches into a list, in deterministic order. + # tuples will contain tuples of (stacked_param, state, stacked_params_names), + # one for each batch in `batches`. + tuples = [] + + for batch, batch_names in zip(batches, batches_names): + p = batch[0] + # we arbitrarily store the state in the + # state corresponding to the 1st parameter in the + # group. class Optimizer will take care of saving/loading state. + state = self.state[p] + p_stacked = torch.stack(batch) + grad = torch.stack( + [ + torch.zeros_like(p) if p.grad is None else p.grad + for p in batch + ] + ) + p_stacked.grad = grad + stacked_params_dict[key] = p_stacked + tuples.append((p_stacked, state, batch_names)) + + yield tuples # <-- calling code will do the actual optimization here! + + for ((stacked_params, _state, _names), batch) in zip(tuples, batches): + for i, p in enumerate(batch): # batch is list of Parameter + p.copy_(stacked_params[i]) + + +class ScaledAdam(BatchedOptimizer): + """ + Implements 'Scaled Adam', a variant of Adam where we scale each parameter's update + proportional to the norm of that parameter; and also learn the scale of the parameter, + in log space, subject to upper and lower limits (as if we had factored each parameter as + param = underlying_param * log_scale.exp()) + + + Args: + params: The parameters or param_groups to optimize (like other Optimizer subclasses) + lr: The learning rate. We will typically use a learning rate schedule that starts + at 0.03 and decreases over time, i.e. much higher than other common + optimizers. + clipping_scale: (e.g. 2.0) + A scale for gradient-clipping: if specified, the normalized gradients + over the whole model will be clipped to have 2-norm equal to + `clipping_scale` times the median 2-norm over the most recent period + of `clipping_update_period` minibatches. By "normalized gradients", + we mean after multiplying by the rms parameter value for this tensor + [for non-scalars]; this is appropriate because our update is scaled + by this quantity. + betas: beta1,beta2 are momentum constants for regular momentum, and moving sum-sq grad. + Must satisfy 0 < beta <= beta2 < 1. + scalar_lr_scale: A scaling factor on the learning rate, that we use to update the + scale of each parameter tensor and scalar parameters of the mode.. + If each parameter were decomposed + as p * p_scale.exp(), where (p**2).mean().sqrt() == 1.0, scalar_lr_scale + would be a the scaling factor on the learning rate of p_scale. + eps: A general-purpose epsilon to prevent division by zero + param_min_rms: Minimum root-mean-square value of parameter tensor, for purposes of + learning the scale on the parameters (we'll constrain the rms of each non-scalar + parameter tensor to be >= this value) + param_max_rms: Maximum root-mean-square value of parameter tensor, for purposes of + learning the scale on the parameters (we'll constrain the rms of each non-scalar + parameter tensor to be <= this value) + scalar_max: Maximum absolute value for scalar parameters (applicable if your + model has any parameters with numel() == 1). + size_update_period: The periodicity, in steps, with which we update the size (scale) + of the parameter tensor. This is provided to save a little time + in the update. + clipping_update_period: if clipping_scale is specified, this is the period + """ + + def __init__( + self, + params, + lr=3e-02, + clipping_scale=None, + betas=(0.9, 0.98), + scalar_lr_scale=0.1, + eps=1.0e-08, + param_min_rms=1.0e-05, + param_max_rms=3.0, + scalar_max=10.0, + size_update_period=4, + clipping_update_period=100, + parameters_names=None, + show_dominant_parameters=True, + ): + + assert parameters_names is not None, ( + "Please prepare parameters_names," + "which is a List[List[str]]. Each List[str] is for a group" + "and each str is for a parameter" + ) + defaults = dict( + lr=lr, + clipping_scale=clipping_scale, + betas=betas, + scalar_lr_scale=scalar_lr_scale, + eps=eps, + param_min_rms=param_min_rms, + param_max_rms=param_max_rms, + scalar_max=scalar_max, + size_update_period=size_update_period, + clipping_update_period=clipping_update_period, + ) + + super(ScaledAdam, self).__init__(params, defaults) + assert len(self.param_groups) == len(parameters_names) + self.parameters_names = parameters_names + self.show_dominant_parameters = show_dominant_parameters + + def __setstate__(self, state): + super(ScaledAdam, self).__setstate__(state) + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + batch = True + + for group, group_params_names in zip( + self.param_groups, self.parameters_names + ): + + with self.batched_params( + group["params"], group_params_names + ) as batches: + + # batches is list of pairs (stacked_param, state). stacked_param is like + # a regular parameter, and will have a .grad, but the 1st dim corresponds to + # a stacking dim, it is not a real dim. + + if ( + len(batches[0][1]) == 0 + ): # if len(first state) == 0: not yet initialized + clipping_scale = 1 + else: + clipping_scale = self._get_clipping_scale(group, batches) + + for p, state, _ in batches: + # Perform optimization step. + # grad is not going to be None, we handled that when creating the batches. + grad = p.grad + if grad.is_sparse: + raise RuntimeError( + "ScaledAdam optimizer does not support sparse gradients" + ) + # State initialization + if len(state) == 0: + self._init_state(group, p, state) + + self._step_one_batch(group, p, state, clipping_scale) + + return loss + + def _init_state(self, group: dict, p: Tensor, state: dict): + """ + Initializes state dict for parameter 'p'. Assumes that dim 0 of tensor p + is actually the batch dimension, corresponding to batched-together + parameters of a given shape. + + + Args: + group: Dict to look up configuration values. + p: The parameter that we are initializing the state for + state: Dict from string to whatever state we are initializing + """ + size_update_period = group["size_update_period"] + + state["step"] = 0 + + kwargs = {"device": p.device, "dtype": p.dtype} + + # 'delta' implements conventional momentum. There are + # several different kinds of update going on, so rather than + # compute "exp_avg" like in Adam, we store and decay a + # parameter-change "delta", which combines all forms of + # update. this is equivalent to how it's done in Adam, + # except for the first few steps. + state["delta"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + batch_size = p.shape[0] + numel = p.numel() // batch_size + numel = p.numel() + + if numel > 1: + # "param_rms" just periodically records the scalar root-mean-square value of + # the parameter tensor. + # it has a shape like (batch_size, 1, 1, 1, 1) + param_rms = ( + (p ** 2).mean(dim=list(range(1, p.ndim)), keepdim=True).sqrt() + ) + state["param_rms"] = param_rms + + state["scale_exp_avg_sq"] = torch.zeros_like(param_rms) + state["scale_grads"] = torch.zeros( + size_update_period, *param_rms.shape, **kwargs + ) + + # exp_avg_sq is the weighted sum of scaled gradients. as in Adam. + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + def _get_clipping_scale( + self, group: dict, tuples: List[Tuple[Tensor, dict, List[str]]] + ) -> float: + """ + Returns a scalar factor <= 1.0 that dictates gradient clipping, i.e. we will scale the gradients + by this amount before applying the rest of the update. + + Args: + group: the parameter group, an item in self.param_groups + tuples: a list of tuples of (param, state, param_names) + where param is a batched set of parameters, + with a .grad (1st dim is batch dim) + and state is the state-dict where optimization parameters are kept. + param_names is a List[str] while each str is name for a parameter + in batched set of parameters "param". + """ + assert len(tuples) >= 1 + clipping_scale = group["clipping_scale"] + (first_p, first_state, _) = tuples[0] + step = first_state["step"] + if clipping_scale is None or step == 0: + # no clipping. return early on step == 0 because the other + # parameters' state won't have been initialized yet. + return 1.0 + clipping_update_period = group["clipping_update_period"] + + tot_sumsq = torch.tensor(0.0, device=first_p.device) + for (p, state, param_names) in tuples: + grad = p.grad + if grad.is_sparse: + raise RuntimeError( + "ScaledAdam optimizer does not support sparse gradients" + ) + if p.numel() == p.shape[0]: # a batch of scalars + tot_sumsq += ( + grad ** 2 + ).sum() # sum() to change shape [1] to [] + else: + tot_sumsq += ((grad * state["param_rms"]) ** 2).sum() + + tot_norm = tot_sumsq.sqrt() + if "model_norms" not in first_state: + first_state["model_norms"] = torch.zeros( + clipping_update_period, device=p.device + ) + first_state["model_norms"][step % clipping_update_period] = tot_norm + + if step % clipping_update_period == 0: + # Print some stats. + # We don't reach here if step == 0 because we would have returned + # above. + sorted_norms = first_state["model_norms"].sort()[0].to("cpu") + quartiles = [] + for n in range(0, 5): + index = min( + clipping_update_period - 1, + (clipping_update_period // 4) * n, + ) + quartiles.append(sorted_norms[index].item()) + + median = quartiles[2] + threshold = clipping_scale * median + first_state["model_norm_threshold"] = threshold + percent_clipped = ( + first_state["num_clipped"] * 100.0 / clipping_update_period + if "num_clipped" in first_state + else 0.0 + ) + first_state["num_clipped"] = 0 + quartiles = " ".join(["%.3e" % x for x in quartiles]) + logging.info( + f"Clipping_scale={clipping_scale}, grad-norm quartiles {quartiles}, " + f"threshold={threshold:.3e}, percent-clipped={percent_clipped:.1f}" + ) + + if step < clipping_update_period: + return 1.0 # We have not yet estimated a norm to clip to. + else: + try: + model_norm_threshold = first_state["model_norm_threshold"] + except KeyError: + logging.info( + "Warning: model_norm_threshold not in state: possibly " + "you changed config when restarting, adding clipping_scale option?" + ) + return 1.0 + ans = min(1.0, (model_norm_threshold / (tot_norm + 1.0e-20)).item()) + if ans < 1.0: + first_state["num_clipped"] += 1 + if ans < 0.1: + logging.warn( + f"Scaling gradients by {ans}, model_norm_threshold={model_norm_threshold}" + ) + if self.show_dominant_parameters: + assert p.shape[0] == len(param_names) + self._show_gradient_dominating_parameter(tuples, tot_sumsq) + return ans + + def _show_gradient_dominating_parameter( + self, tuples: List[Tuple[Tensor, dict, List[str]]], tot_sumsq: Tensor + ): + """ + Show information of parameter wihch dominanting tot_sumsq. + + Args: + tuples: a list of tuples of (param, state, param_names) + where param is a batched set of parameters, + with a .grad (1st dim is batch dim) + and state is the state-dict where optimization parameters are kept. + param_names is a List[str] while each str is name for a parameter + in batched set of parameters "param". + tot_sumsq: sumsq of all parameters. Though it's could be calculated + from tuples, we still pass it to save some time. + """ + all_sumsq_orig = {} + for (p, state, batch_param_names) in tuples: + # p is a stacked batch parameters. + batch_grad = p.grad + if p.numel() == p.shape[0]: # a batch of scalars + batch_sumsq_orig = batch_grad ** 2 + # Dummpy values used by following `zip` statement. + batch_rms_orig = torch.ones(p.shape[0]) + else: + batch_rms_orig = state["param_rms"] + batch_sumsq_orig = ((batch_grad * batch_rms_orig) ** 2).sum( + dim=list(range(1, batch_grad.ndim)) + ) + + for name, sumsq_orig, rms, grad in zip( + batch_param_names, batch_sumsq_orig, batch_rms_orig, batch_grad + ): + + proportion_orig = sumsq_orig / tot_sumsq + all_sumsq_orig[name] = (proportion_orig, sumsq_orig, rms, grad) + + assert torch.isclose( + sum([value[0] for value in all_sumsq_orig.values()]).cpu(), + torch.tensor(1.0), + ) + sorted_by_proportion = { + k: v + for k, v in sorted( + all_sumsq_orig.items(), + key=lambda item: item[1][0], + reverse=True, + ) + } + dominant_param_name = next(iter(sorted_by_proportion)) + ( + dominant_proportion, + dominant_sumsq, + dominant_rms, + dominant_grad, + ) = sorted_by_proportion[dominant_param_name] + logging.info( + f"Parameter Dominanting tot_sumsq {dominant_param_name}" + f" with proportion {dominant_proportion:.2f}," + f" where dominant_sumsq=(grad_sumsq*orig_rms_sq)" + f"={dominant_sumsq:.3e}," + f" grad_sumsq = {(dominant_grad**2).sum():.3e}," + f" orig_rms_sq={(dominant_rms**2).item():.3e}" + ) + + def _step_one_batch( + self, group: dict, p: Tensor, state: dict, clipping_scale: float + ): + """ + Do the step for one parameter, which is actually going to be a batch of + `real` parameters, with dim 0 as the batch dim. + Args: + group: dict to look up configuration values + p: parameter to update (actually multiple parameters stacked together + as a batch) + state: state-dict for p, to look up the optimizer state + """ + lr = group["lr"] + size_update_period = group["size_update_period"] + beta1 = group["betas"][0] + + grad = p.grad + if clipping_scale != 1.0: + grad = grad * clipping_scale + step = state["step"] + delta = state["delta"] + + delta.mul_(beta1) + batch_size = p.shape[0] + numel = p.numel() // batch_size + if numel > 1: + # Update the size/scale of p, and set param_rms + scale_grads = state["scale_grads"] + scale_grads[step % size_update_period] = (p * grad).sum( + dim=list(range(1, p.ndim)), keepdim=True + ) + if step % size_update_period == size_update_period - 1: + param_rms = state["param_rms"] # shape: (batch_size, 1, 1, ..) + param_rms.copy_( + (p ** 2) + .mean(dim=list(range(1, p.ndim)), keepdim=True) + .sqrt() + ) + if step > 0: + # self._size_update() learns the overall scale on the + # parameter, by shrinking or expanding it. + self._size_update(group, scale_grads, p, state) + + if numel == 1: + # For parameters with 1 element we just use regular Adam. + # Updates delta. + self._step_scalar(group, p, state) + else: + self._step(group, p, state) + + state["step"] = step + 1 + + def _size_update( + self, group: dict, scale_grads: Tensor, p: Tensor, state: dict + ) -> None: + """ + Called only where p.numel() > 1, this updates the scale of the parameter. + If we imagine: p = underlying_param * scale.exp(), and we are doing + gradient descent on underlying param and on scale, this function does the update + on `scale`. + + Args: + group: dict to look up configuration values + scale_grads: a tensor of shape (size_update_period, batch_size, 1, 1,...) containing + grads w.r.t. the scales. + p: The parameter to update + state: The state-dict of p + """ + + param_rms = state["param_rms"] + beta1, beta2 = group["betas"] + size_lr = group["lr"] * group["scalar_lr_scale"] + param_min_rms = group["param_min_rms"] + param_max_rms = group["param_max_rms"] + eps = group["eps"] + step = state["step"] + batch_size = p.shape[0] + + size_update_period = scale_grads.shape[0] + # correct beta2 for the size update period: we will have + # faster decay at this level. + beta2_corr = beta2 ** size_update_period + + scale_exp_avg_sq = state[ + "scale_exp_avg_sq" + ] # shape: (batch_size, 1, 1, ..) + scale_exp_avg_sq.mul_(beta2_corr).add_( + (scale_grads ** 2).mean( + dim=0 + ), # mean over dim `size_update_period` + alpha=1 - beta2_corr, + ) # shape is (batch_size, 1, 1, ...) + + # The 1st time we reach here is when size_step == 1. + size_step = (step + 1) // size_update_period + bias_correction2 = 1 - beta2_corr ** size_step + # we don't bother with bias_correction1; this will help prevent divergence + # at the start of training. + + denom = scale_exp_avg_sq.sqrt() + eps + + scale_step = ( + -size_lr + * (bias_correction2 ** 0.5) + * scale_grads.sum(dim=0) + / denom + ) + + is_too_small = param_rms < param_min_rms + is_too_large = param_rms > param_max_rms + + # when the param gets too small, just don't shrink it any further. + scale_step.masked_fill_(is_too_small, 0.0) + # when it gets too large, stop it from getting any larger. + scale_step.masked_fill_(is_too_large, -size_lr * size_update_period) + delta = state["delta"] + # the factor of (1-beta1) relates to momentum. + delta.add_(p * scale_step, alpha=(1 - beta1)) + + def _step(self, group: dict, p: Tensor, state: dict): + """ + This function does the core update of self.step(), in the case where the members of + the batch have more than 1 element. + + Args: + group: A dict which will be used to look up configuration values + p: The parameter to be updated + grad: The grad of p + state: The state-dict corresponding to parameter p + + This function modifies p. + """ + grad = p.grad + lr = group["lr"] + beta1, beta2 = group["betas"] + eps = group["eps"] + param_min_rms = group["param_min_rms"] + step = state["step"] + + exp_avg_sq = state["exp_avg_sq"] + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=(1 - beta2)) + + this_step = state["step"] - ( + state["zero_step"] if "zero_step" in state else 0 + ) + bias_correction2 = 1 - beta2 ** (this_step + 1) + if bias_correction2 < 0.99: + # note: not in-place. + exp_avg_sq = exp_avg_sq * (1.0 / bias_correction2) + + denom = exp_avg_sq.sqrt() + denom += eps + grad = grad / denom + + alpha = -lr * (1 - beta1) * state["param_rms"].clamp(min=param_min_rms) + + delta = state["delta"] + delta.add_(grad * alpha) + p.add_(delta) + + def _step_scalar(self, group: dict, p: Tensor, state: dict): + """ + A simplified form of the core update for scalar tensors, where we cannot get a good + estimate of the parameter rms. + """ + beta1, beta2 = group["betas"] + scalar_max = group["scalar_max"] + eps = group["eps"] + lr = group["lr"] * group["scalar_lr_scale"] + grad = p.grad + + exp_avg_sq = state["exp_avg_sq"] # shape: (batch_size,) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + + # bias_correction2 is like in Adam. Don't bother with bias_correction1; + # slower update at the start will help stability anyway. + bias_correction2 = 1 - beta2 ** (state["step"] + 1) + denom = (exp_avg_sq / bias_correction2).sqrt() + eps + + delta = state["delta"] + delta.add_(grad / denom, alpha=-lr * (1 - beta1)) + p.clamp_(min=-scalar_max, max=scalar_max) + p.add_(delta) + + +class LRScheduler(object): + """ + Base-class for learning rate schedulers where the learning-rate depends on both the + batch and the epoch. + """ + + def __init__(self, optimizer: Optimizer, verbose: bool = False): + # Attach optimizer + if not isinstance(optimizer, Optimizer): + raise TypeError( + "{} is not an Optimizer".format(type(optimizer).__name__) + ) + self.optimizer = optimizer + self.verbose = verbose + + for group in optimizer.param_groups: + group.setdefault("base_lr", group["lr"]) + + self.base_lrs = [group["base_lr"] for group in optimizer.param_groups] + + self.epoch = 0 + self.batch = 0 + + def state_dict(self): + """Returns the state of the scheduler as a :class:`dict`. + + It contains an entry for every variable in self.__dict__ which + is not the optimizer. + """ + return { + "base_lrs": self.base_lrs, + "epoch": self.epoch, + "batch": self.batch, + } + + def load_state_dict(self, state_dict): + """Loads the schedulers state. + + Args: + state_dict (dict): scheduler state. Should be an object returned + from a call to :meth:`state_dict`. + """ + self.__dict__.update(state_dict) + + def get_last_lr(self) -> List[float]: + """Return last computed learning rate by current scheduler. Will be a list of float.""" + return self._last_lr + + def get_lr(self): + # Compute list of learning rates from self.epoch and self.batch and + # self.base_lrs; this must be overloaded by the user. + # e.g. return [some_formula(self.batch, self.epoch, base_lr) for base_lr in self.base_lrs ] + raise NotImplementedError + + def step_batch(self, batch: Optional[int] = None) -> None: + # Step the batch index, or just set it. If `batch` is specified, it + # must be the batch index from the start of training, i.e. summed over + # all epochs. + # You can call this in any order; if you don't provide 'batch', it should + # of course be called once per batch. + if batch is not None: + self.batch = batch + else: + self.batch = self.batch + 1 + self._set_lrs() + + def step_epoch(self, epoch: Optional[int] = None): + # Step the epoch index, or just set it. If you provide the 'epoch' arg, + # you should call this at the start of the epoch; if you don't provide the 'epoch' + # arg, you should call it at the end of the epoch. + if epoch is not None: + self.epoch = epoch + else: + self.epoch = self.epoch + 1 + self._set_lrs() + + def _set_lrs(self): + values = self.get_lr() + assert len(values) == len(self.optimizer.param_groups) + + for i, data in enumerate(zip(self.optimizer.param_groups, values)): + param_group, lr = data + param_group["lr"] = lr + self.print_lr(self.verbose, i, lr) + self._last_lr = [group["lr"] for group in self.optimizer.param_groups] + + def print_lr(self, is_verbose, group, lr): + """Display the current learning rate.""" + if is_verbose: + logging.info( + f"Epoch={self.epoch}, batch={self.batch}: adjusting learning rate" + f" of group {group} to {lr:.4e}." + ) + + +class Eden(LRScheduler): + """ + Eden scheduler. + The basic formula (before warmup) is: + lr = base_lr * (((batch**2 + lr_batches**2) / lr_batches**2) ** -0.25 * + (((epoch**2 + lr_epochs**2) / lr_epochs**2) ** -0.25)) * warmup + where `warmup` increases from linearly 0.5 to 1 over `warmup_batches` batches + and then stays constant at 1. + + + E.g. suggest base_lr = 0.04 (passed to optimizer) if used with ScaledAdam + + Args: + optimizer: the optimizer to change the learning rates on + lr_batches: the number of batches after which we start significantly + decreasing the learning rate, suggest 5000. + lr_epochs: the number of epochs after which we start significantly + decreasing the learning rate, suggest 6 if you plan to do e.g. + 20 to 40 epochs, but may need smaller number if dataset is huge + and you will do few epochs. + """ + + def __init__( + self, + optimizer: Optimizer, + lr_batches: Union[int, float], + lr_epochs: Union[int, float], + warmup_batches: Union[int, float] = 500.0, + verbose: bool = False, + ): + super(Eden, self).__init__(optimizer, verbose) + self.lr_batches = lr_batches + self.lr_epochs = lr_epochs + self.warmup_batches = warmup_batches + + def get_lr(self): + factor = ( + (self.batch ** 2 + self.lr_batches ** 2) / self.lr_batches ** 2 + ) ** -0.25 * ( + ((self.epoch ** 2 + self.lr_epochs ** 2) / self.lr_epochs ** 2) + ** -0.25 + ) + warmup_factor = ( + 1.0 + if self.batch >= self.warmup_batches + else 0.5 + 0.5 * (self.batch / self.warmup_batches) + ) + + return [x * factor * warmup_factor for x in self.base_lrs] + + +def _test_eden(): + m = torch.nn.Linear(100, 100) + optim = ScaledAdam(m.parameters(), lr=0.03) + + scheduler = Eden(optim, lr_batches=100, lr_epochs=2, verbose=True) + + for epoch in range(10): + scheduler.step_epoch(epoch) # sets epoch to `epoch` + + for step in range(20): + x = torch.randn(200, 100).detach() + x.requires_grad = True + y = m(x) + dy = torch.randn(200, 100).detach() + f = (y * dy).sum() + f.backward() + + optim.step() + scheduler.step_batch() + optim.zero_grad() + + logging.info(f"last lr = {scheduler.get_last_lr()}") + logging.info(f"state dict = {scheduler.state_dict()}") + + +# This is included mostly as a baseline for ScaledAdam. +class Eve(Optimizer): + """ + Implements Eve algorithm. This is a modified version of AdamW with a special + way of setting the weight-decay / shrinkage-factor, which is designed to make the + rms of the parameters approach a particular target_rms (default: 0.1). This is + for use with networks with 'scaled' versions of modules (see scaling.py), which + will be close to invariant to the absolute scale on the parameter matrix. + + The original Adam algorithm was proposed in `Adam: A Method for Stochastic Optimization`_. + The AdamW variant was proposed in `Decoupled Weight Decay Regularization`_. + Eve is unpublished so far. + + Arguments: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1e-3) + betas (Tuple[float, float], optional): coefficients used for computing + running averages of gradient and its square (default: (0.9, 0.999)) + eps (float, optional): term added to the denominator to improve + numerical stability (default: 1e-8) + weight_decay (float, optional): weight decay coefficient (default: 3e-4; + this value means that the weight would decay significantly after + about 3k minibatches. Is not multiplied by learning rate, but + is conditional on RMS-value of parameter being > target_rms. + target_rms (float, optional): target root-mean-square value of + parameters, if they fall below this we will stop applying weight decay. + + + .. _Adam: A Method for Stochastic Optimization: + https://arxiv.org/abs/1412.6980 + .. _Decoupled Weight Decay Regularization: + https://arxiv.org/abs/1711.05101 + .. _On the Convergence of Adam and Beyond: + https://openreview.net/forum?id=ryQu7f-RZ + """ + + def __init__( + self, + params, + lr=1e-3, + betas=(0.9, 0.98), + eps=1e-8, + weight_decay=1e-3, + target_rms=0.1, + ): + if not 0.0 <= lr: + raise ValueError("Invalid learning rate: {}".format(lr)) + if not 0.0 <= eps: + raise ValueError("Invalid epsilon value: {}".format(eps)) + if not 0.0 <= betas[0] < 1.0: + raise ValueError( + "Invalid beta parameter at index 0: {}".format(betas[0]) + ) + if not 0.0 <= betas[1] < 1.0: + raise ValueError( + "Invalid beta parameter at index 1: {}".format(betas[1]) + ) + if not 0 <= weight_decay <= 0.1: + raise ValueError( + "Invalid weight_decay value: {}".format(weight_decay) + ) + if not 0 < target_rms <= 10.0: + raise ValueError("Invalid target_rms value: {}".format(target_rms)) + defaults = dict( + lr=lr, + betas=betas, + eps=eps, + weight_decay=weight_decay, + target_rms=target_rms, + ) + super(Eve, self).__init__(params, defaults) + + def __setstate__(self, state): + super(Eve, self).__setstate__(state) + + @torch.no_grad() + def step(self, closure=None): + """Performs a single optimization step. + + Arguments: + closure (callable, optional): A closure that reevaluates the model + and returns the loss. + """ + loss = None + if closure is not None: + with torch.enable_grad(): + loss = closure() + + for group in self.param_groups: + for p in group["params"]: + if p.grad is None: + continue + + # Perform optimization step + grad = p.grad + if grad.is_sparse: + raise RuntimeError( + "AdamW does not support sparse gradients" + ) + + state = self.state[p] + + # State initialization + if len(state) == 0: + state["step"] = 0 + # Exponential moving average of gradient values + state["exp_avg"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + # Exponential moving average of squared gradient values + state["exp_avg_sq"] = torch.zeros_like( + p, memory_format=torch.preserve_format + ) + + exp_avg, exp_avg_sq = state["exp_avg"], state["exp_avg_sq"] + + beta1, beta2 = group["betas"] + + state["step"] += 1 + bias_correction1 = 1 - beta1 ** state["step"] + bias_correction2 = 1 - beta2 ** state["step"] + + # Decay the first and second moment running average coefficient + exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1) + exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2) + denom = (exp_avg_sq.sqrt() * (bias_correction2 ** -0.5)).add_( + group["eps"] + ) + + step_size = group["lr"] / bias_correction1 + target_rms = group["target_rms"] + weight_decay = group["weight_decay"] + + if p.numel() > 1: + # avoid applying this weight-decay on "scaling factors" + # (which are scalar). + is_above_target_rms = p.norm() > ( + target_rms * (p.numel() ** 0.5) + ) + p.mul_(1 - (weight_decay * is_above_target_rms)) + + p.addcdiv_(exp_avg, denom, value=-step_size) + + # if random.random() < 0.0005: + # step = (exp_avg / denom) * step_size + # logging.info( + # f"Delta rms = {(step**2).mean().item()}, shape = {step.shape}" + # ) + + return loss + + +def _test_scaled_adam(hidden_dim: int): + import timeit + + from scaling import ScaledLinear + + E = 100 + B = 4 + T = 2 + logging.info("in test_eve_cain") + # device = torch.device('cuda') + device = torch.device("cpu") + dtype = torch.float32 + + fix_random_seed(42) + # these input_magnitudes and output_magnitudes are to test that + # Abel is working as we expect and is able to adjust scales of + # different dims differently. + input_magnitudes = (1.0 * torch.randn(E, dtype=dtype, device=device)).exp() + output_magnitudes = (1.0 * torch.randn(E, dtype=dtype, device=device)).exp() + + for iter in [1, 0]: + fix_random_seed(42) + Linear = torch.nn.Linear if iter == 0 else ScaledLinear + + m = torch.nn.Sequential( + Linear(E, hidden_dim), + torch.nn.PReLU(), + Linear(hidden_dim, hidden_dim), + torch.nn.PReLU(), + Linear(hidden_dim, E), + ).to(device) + + train_pairs = [ + ( + 100.0 + * torch.randn(B, T, E, device=device, dtype=dtype) + * input_magnitudes, + torch.randn(B, T, E, device=device, dtype=dtype) + * output_magnitudes, + ) + for _ in range(20) + ] + + if iter == 0: + optim = Eve(m.parameters(), lr=0.003) + elif iter == 1: + optim = ScaledAdam(m.parameters(), lr=0.03, clipping_scale=2.0) + scheduler = Eden(optim, lr_batches=200, lr_epochs=5, verbose=False) + + start = timeit.default_timer() + avg_loss = 0.0 + for epoch in range(180): + scheduler.step_epoch() + # if epoch == 100 and iter in [2,3]: + # optim.reset_speedup() # check it doesn't crash. + + # if epoch == 130: + # opts = diagnostics.TensorDiagnosticOptions( + # 2 ** 22 + # ) # allow 4 megabytes per sub-module + # diagnostic = diagnostics.attach_diagnostics(m, opts) + + for n, (x, y) in enumerate(train_pairs): + y_out = m(x) + loss = ((y_out - y) ** 2).mean() * 100.0 + if epoch == 0 and n == 0: + avg_loss = loss.item() + else: + avg_loss = 0.98 * avg_loss + 0.02 * loss.item() + if n == 0 and epoch % 5 == 0: + # norm1 = '%.2e' % (m[0].weight**2).mean().sqrt().item() + # norm1b = '%.2e' % (m[0].bias**2).mean().sqrt().item() + # norm2 = '%.2e' % (m[2].weight**2).mean().sqrt().item() + # norm2b = '%.2e' % (m[2].bias**2).mean().sqrt().item() + # scale1 = '%.2e' % (m[0].weight_scale.exp().item()) + # scale1b = '%.2e' % (m[0].bias_scale.exp().item()) + # scale2 = '%.2e' % (m[2].weight_scale.exp().item()) + # scale2b = '%.2e' % (m[2].bias_scale.exp().item()) + lr = scheduler.get_last_lr()[0] + logging.info( + f"Iter {iter}, epoch {epoch}, batch {n}, avg_loss {avg_loss:.4g}, lr={lr:.4e}" + ) # , norms={norm1,norm1b,norm2,norm2b}") # scales={scale1,scale1b,scale2,scale2b} + loss.log().backward() + optim.step() + optim.zero_grad() + scheduler.step_batch() + + # diagnostic.print_diagnostics() + + stop = timeit.default_timer() + logging.info(f"Iter={iter}, Time taken: {stop - start}") + + logging.info(f"last lr = {scheduler.get_last_lr()}") + # logging.info("state dict = ", scheduler.state_dict()) + # logging.info("optim state_dict = ", optim.state_dict()) + logging.info(f"input_magnitudes = {input_magnitudes}") + logging.info(f"output_magnitudes = {output_magnitudes}") + + +if __name__ == "__main__": + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + logging.getLogger().setLevel(logging.INFO) + import subprocess + + s = subprocess.check_output( + "git status -uno .; git log -1; git diff HEAD .", shell=True + ) + logging.info(s) + import sys + + if len(sys.argv) > 1: + hidden_dim = int(sys.argv[1]) + else: + hidden_dim = 200 + + _test_scaled_adam(hidden_dim) + _test_eden() diff --git a/tts/VALL-E-X/modules/scaling.py b/tts/VALL-E-X/modules/scaling.py new file mode 100644 index 0000000..e944cf5 --- /dev/null +++ b/tts/VALL-E-X/modules/scaling.py @@ -0,0 +1,1401 @@ +# Copyright 2022 Xiaomi Corp. (authors: Daniel Povey) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import collections +import logging +import random +import math +from functools import reduce +from itertools import repeat +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch import Tensor +from torch.nn import Embedding as ScaledEmbedding + +from utils import Transpose + + +class ActivationBalancerFunction(torch.autograd.Function): + @staticmethod + def forward( + ctx, + x: Tensor, + scale_factor: Tensor, + sign_factor: Optional[Tensor], + channel_dim: int, + ) -> Tensor: + if channel_dim < 0: + channel_dim += x.ndim + ctx.channel_dim = channel_dim + xgt0 = x > 0 + if sign_factor is None: + ctx.save_for_backward(xgt0, scale_factor) + else: + ctx.save_for_backward(xgt0, scale_factor, sign_factor) + return x + + @staticmethod + def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None]: + if len(ctx.saved_tensors) == 3: + xgt0, scale_factor, sign_factor = ctx.saved_tensors + for _ in range(ctx.channel_dim, x_grad.ndim - 1): + scale_factor = scale_factor.unsqueeze(-1) + sign_factor = sign_factor.unsqueeze(-1) + factor = sign_factor + scale_factor * (xgt0.to(x_grad.dtype) - 0.5) + else: + xgt0, scale_factor = ctx.saved_tensors + for _ in range(ctx.channel_dim, x_grad.ndim - 1): + scale_factor = scale_factor.unsqueeze(-1) + factor = scale_factor * (xgt0.to(x_grad.dtype) - 0.5) + neg_delta_grad = x_grad.abs() * factor + return ( + x_grad - neg_delta_grad, + None, + None, + None, + ) + + +def _compute_scale_factor( + x: Tensor, + channel_dim: int, + min_abs: float, + max_abs: float, + gain_factor: float, + max_factor: float, +) -> Tensor: + if channel_dim < 0: + channel_dim += x.ndim + sum_dims = [d for d in range(x.ndim) if d != channel_dim] + x_abs_mean = torch.mean(x.abs(), dim=sum_dims).to(torch.float32) + + if min_abs == 0.0: + below_threshold = 0.0 + else: + # below_threshold is 0 if x_abs_mean > min_abs, can be at most max_factor if + # x_abs)_mean , min_abs. + below_threshold = ( + (min_abs - x_abs_mean) * (gain_factor / min_abs) + ).clamp(min=0, max=max_factor) + + above_threshold = ((x_abs_mean - max_abs) * (gain_factor / max_abs)).clamp( + min=0, max=max_factor + ) + + return below_threshold - above_threshold + + +def _compute_sign_factor( + x: Tensor, + channel_dim: int, + min_positive: float, + max_positive: float, + gain_factor: float, + max_factor: float, +) -> Tensor: + if channel_dim < 0: + channel_dim += x.ndim + sum_dims = [d for d in range(x.ndim) if d != channel_dim] + proportion_positive = torch.mean((x > 0).to(torch.float32), dim=sum_dims) + if min_positive == 0.0: + factor1 = 0.0 + else: + # 0 if proportion_positive >= min_positive, else can be + # as large as max_factor. + factor1 = ( + (min_positive - proportion_positive) * (gain_factor / min_positive) + ).clamp_(min=0, max=max_factor) + + if max_positive == 1.0: + factor2 = 0.0 + else: + # 0 if self.proportion_positive <= max_positive, else can be + # as large as -max_factor. + factor2 = ( + (proportion_positive - max_positive) + * (gain_factor / (1.0 - max_positive)) + ).clamp_(min=0, max=max_factor) + sign_factor = factor1 - factor2 + # require min_positive != 0 or max_positive != 1: + assert not isinstance(sign_factor, float) + return sign_factor + + +class ActivationScaleBalancerFunction(torch.autograd.Function): + """ + This object is used in class ActivationBalancer when the user specified + min_positive=0, max_positive=1, so there are no constraints on the signs + of the activations and only the absolute value has a constraint. + """ + + @staticmethod + def forward( + ctx, + x: Tensor, + sign_factor: Tensor, + scale_factor: Tensor, + channel_dim: int, + ) -> Tensor: + if channel_dim < 0: + channel_dim += x.ndim + ctx.channel_dim = channel_dim + xgt0 = x > 0 + ctx.save_for_backward(xgt0, sign_factor, scale_factor) + return x + + @staticmethod + def backward(ctx, x_grad: Tensor) -> Tuple[Tensor, None, None, None]: + xgt0, sign_factor, scale_factor = ctx.saved_tensors + for _ in range(ctx.channel_dim, x_grad.ndim - 1): + sign_factor = sign_factor.unsqueeze(-1) + scale_factor = scale_factor.unsqueeze(-1) + + factor = sign_factor + scale_factor * (xgt0.to(x_grad.dtype) - 0.5) + neg_delta_grad = x_grad.abs() * factor + return ( + x_grad - neg_delta_grad, + None, + None, + None, + ) + + +class RandomClampFunction(torch.autograd.Function): + @staticmethod + def forward( + ctx, + x: Tensor, + min: Optional[float], + max: Optional[float], + prob: float, + reflect: float, + ) -> Tensor: + x_clamped = torch.clamp(x, min=min, max=max) + mask = torch.rand_like(x) < prob + ans = torch.where(mask, x_clamped, x) + if x.requires_grad: + ctx.save_for_backward(ans == x) + ctx.reflect = reflect + if reflect != 0.0: + ans = ans * (1.0 + reflect) - (x * reflect) + return ans + + @staticmethod + def backward( + ctx, ans_grad: Tensor + ) -> Tuple[Tensor, None, None, None, None]: + (is_same,) = ctx.saved_tensors + x_grad = ans_grad * is_same.to(ans_grad.dtype) + reflect = ctx.reflect + if reflect != 0.0: + x_grad = x_grad * (1.0 + reflect) - (ans_grad * reflect) + return x_grad, None, None, None, None + + +def random_clamp( + x: Tensor, + min: Optional[float] = None, + max: Optional[float] = None, + prob: float = 0.5, + reflect: float = 0.0, +): + return RandomClampFunction.apply(x, min, max, prob, reflect) + + +def random_cast_to_half(x: Tensor, min_abs: float = 5.0e-06) -> Tensor: + """ + A randomized way of casting a floating point value to half precision. + """ + if x.dtype == torch.float16: + return x + x_abs = x.abs() + is_too_small = x_abs < min_abs + # for elements where is_too_small is true, random_val will contain +-min_abs with + # probability (x.abs() / min_abs), and 0.0 otherwise. [so this preserves expectations, + # for those elements]. + random_val = min_abs * x.sign() * (torch.rand_like(x) * min_abs < x_abs) + return torch.where(is_too_small, random_val, x).to(torch.float16) + + +class RandomGradFunction(torch.autograd.Function): + """ + Does nothing in forward pass; in backward pass, gets rid of very small grads using + randomized approach that preserves expectations (intended to reduce roundoff). + """ + + @staticmethod + def forward(ctx, x: Tensor, min_abs: float) -> Tensor: + ctx.min_abs = min_abs + return x + + @staticmethod + def backward(ctx, ans_grad: Tensor) -> Tuple[Tensor, None]: + if ans_grad.dtype == torch.float16: + return ( + random_cast_to_half( + ans_grad.to(torch.float32), min_abs=ctx.min_abs + ), + None, + ) + else: + return ans_grad, None + + +class RandomGrad(torch.nn.Module): + """ + Gets rid of very small gradients using an expectation-preserving method, intended to increase + accuracy of training when using amp (automatic mixed precision) + """ + + def __init__(self, min_abs: float = 5.0e-06): + super(RandomGrad, self).__init__() + self.min_abs = min_abs + + def forward(self, x: Tensor): + if ( + torch.jit.is_scripting() + or not self.training + or torch.jit.is_tracing() + ): + return x + else: + return RandomGradFunction.apply(x, self.min_abs) + + +class SoftmaxFunction(torch.autograd.Function): + """ + Tries to handle half-precision derivatives in a randomized way that should + be more accurate for training than the default behavior. + """ + + @staticmethod + def forward(ctx, x: Tensor, dim: int): + ans = x.softmax(dim=dim) + # if x dtype is float16, x.softmax() returns a float32 because + # (presumably) that op does not support float16, and autocast + # is enabled. + if torch.is_autocast_enabled(): + ans = ans.to(torch.float16) + ctx.save_for_backward(ans) + ctx.x_dtype = x.dtype + ctx.dim = dim + return ans + + @staticmethod + def backward(ctx, ans_grad: Tensor): + (ans,) = ctx.saved_tensors + with torch.cuda.amp.autocast(enabled=False): + ans_grad = ans_grad.to(torch.float32) + ans = ans.to(torch.float32) + x_grad = ans_grad * ans + x_grad = x_grad - ans * x_grad.sum(dim=ctx.dim, keepdim=True) + return x_grad, None + + +def softmax(x: Tensor, dim: int): + if torch.jit.is_scripting() or torch.jit.is_tracing(): + return x.softmax(dim) + + return SoftmaxFunction.apply(x, dim) + + +class MaxEigLimiterFunction(torch.autograd.Function): + @staticmethod + def forward( + ctx, + x: Tensor, + coeffs: Tensor, + direction: Tensor, + channel_dim: int, + grad_scale: float, + ) -> Tensor: + ctx.channel_dim = channel_dim + ctx.grad_scale = grad_scale + ctx.save_for_backward(x.detach(), coeffs.detach(), direction.detach()) + return x + + @staticmethod + def backward(ctx, x_grad, *args): + with torch.enable_grad(): + (x_orig, coeffs, new_direction) = ctx.saved_tensors + x_orig.requires_grad = True + num_channels = x_orig.shape[ctx.channel_dim] + x = x_orig.transpose(ctx.channel_dim, -1).reshape(-1, num_channels) + new_direction.requires_grad = False + x = x - x.mean(dim=0) + x_var = (x ** 2).mean() + x_residual = x - coeffs * new_direction + x_residual_var = (x_residual ** 2).mean() + # `variance_proportion` is the proportion of the variance accounted for + # by the top eigen-direction. This is to be minimized. + variance_proportion = (x_var - x_residual_var) / (x_var + 1.0e-20) + variance_proportion.backward() + x_orig_grad = x_orig.grad + x_extra_grad = ( + x_orig.grad + * ctx.grad_scale + * x_grad.norm() + / (x_orig_grad.norm() + 1.0e-20) + ) + return x_grad + x_extra_grad.detach(), None, None, None, None + + +class BasicNorm(torch.nn.Module): + """ + This is intended to be a simpler, and hopefully cheaper, replacement for + LayerNorm. The observation this is based on, is that Transformer-type + networks, especially with pre-norm, sometimes seem to set one of the + feature dimensions to a large constant value (e.g. 50), which "defeats" + the LayerNorm because the output magnitude is then not strongly dependent + on the other (useful) features. Presumably the weight and bias of the + LayerNorm are required to allow it to do this. + + So the idea is to introduce this large constant value as an explicit + parameter, that takes the role of the "eps" in LayerNorm, so the network + doesn't have to do this trick. We make the "eps" learnable. + + Args: + num_channels: the number of channels, e.g. 512. + channel_dim: the axis/dimension corresponding to the channel, + interprted as an offset from the input's ndim if negative. + shis is NOT the num_channels; it should typically be one of + {-2, -1, 0, 1, 2, 3}. + eps: the initial "epsilon" that we add as ballast in: + scale = ((input_vec**2).mean() + epsilon)**-0.5 + Note: our epsilon is actually large, but we keep the name + to indicate the connection with conventional LayerNorm. + learn_eps: if true, we learn epsilon; if false, we keep it + at the initial value. + eps_min: float + eps_max: float + """ + + def __init__( + self, + num_channels: int, + channel_dim: int = -1, # CAUTION: see documentation. + eps: float = 0.25, + learn_eps: bool = True, + eps_min: float = -3.0, + eps_max: float = 3.0, + ) -> None: + super(BasicNorm, self).__init__() + self.num_channels = num_channels + self.channel_dim = channel_dim + if learn_eps: + self.eps = nn.Parameter(torch.tensor(eps).log().detach()) + else: + self.register_buffer("eps", torch.tensor(eps).log().detach()) + self.eps_min = eps_min + self.eps_max = eps_max + + def forward(self, x: Tensor) -> Tensor: + assert x.shape[self.channel_dim] == self.num_channels + eps = self.eps + if self.training and random.random() < 0.25: + # with probability 0.25, in training mode, clamp eps between the min + # and max; this will encourage it to learn parameters within the + # allowed range by making parameters that are outside the allowed + # range noisy. + + # gradients to allow the parameter to get back into the allowed + # region if it happens to exit it. + eps = eps.clamp(min=self.eps_min, max=self.eps_max) + scales = ( + torch.mean(x ** 2, dim=self.channel_dim, keepdim=True) + eps.exp() + ) ** -0.5 + return x * scales + + +def ScaledLinear(*args, initial_scale: float = 1.0, **kwargs) -> nn.Linear: + """ + Behaves like a constructor of a modified version of nn.Linear + that gives an easy way to set the default initial parameter scale. + + Args: + Accepts the standard args and kwargs that nn.Linear accepts + e.g. in_features, out_features, bias=False. + + initial_scale: you can override this if you want to increase + or decrease the initial magnitude of the module's output + (affects the initialization of weight_scale and bias_scale). + Another option, if you want to do something like this, is + to re-initialize the parameters. + """ + ans = nn.Linear(*args, **kwargs) + with torch.no_grad(): + ans.weight[:] *= initial_scale + if ans.bias is not None: + torch.nn.init.uniform_( + ans.bias, -0.1 * initial_scale, 0.1 * initial_scale + ) + return ans + + +def ScaledConv1d( + *args, + initial_scale: float = 1.0, + kernel_size: int = 3, + padding: str = "same", + **kwargs, +) -> nn.Conv1d: + """ + Behaves like a constructor of a modified version of nn.Conv1d + that gives an easy way to set the default initial parameter scale. + + Args: + Accepts the standard args and kwargs that nn.Linear accepts + e.g. in_features, out_features, bias=False. + + initial_scale: you can override this if you want to increase + or decrease the initial magnitude of the module's output + (affects the initialization of weight_scale and bias_scale). + Another option, if you want to do something like this, is + to re-initialize the parameters. + """ + ans = nn.Conv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs) + with torch.no_grad(): + ans.weight[:] *= initial_scale + if ans.bias is not None: + torch.nn.init.uniform_( + ans.bias, -0.1 * initial_scale, 0.1 * initial_scale + ) + return ans + + +def TransposeScaledConv1d( + *args, + initial_scale: float = 1.0, + kernel_size: int = 3, + padding: str = "same", + **kwargs, +) -> nn.Sequential: + """ + Transpose -> ScaledConv1d + """ + return nn.Sequential( + Transpose(), + ScaledConv1d( + *args, + initial_scale=initial_scale, + kernel_size=kernel_size, + padding=padding, + **kwargs, + ), + ) + + +def ScaledConv1dTranspose( + *args, + initial_scale: float = 1.0, + kernel_size: int = 3, + padding: str = "same", + **kwargs, +) -> nn.Sequential: + """ + Transpose -> ScaledConv1d + """ + return nn.Sequential( + ScaledConv1d( + *args, + initial_scale=initial_scale, + kernel_size=kernel_size, + padding=padding, + **kwargs, + ), + Transpose(), + ) + + +def TransposeConv1d( + *args, kernel_size: int = 3, padding: str = "same", **kwargs +) -> nn.Sequential: + """ + Transpose -> Conv1d + """ + return nn.Sequential( + Transpose(), + nn.Conv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs), + ) + + +def Conv1dTranspose( + *args, kernel_size: int = 3, padding: str = "same", **kwargs +) -> nn.Sequential: + """ + ScaledConv1d -> Transpose + """ + return nn.Sequential( + nn.Conv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs), + Transpose(), + ) + + +class SRLinear(nn.Linear): + """https://arxiv.org/abs/2303.06296 + Stabilizing Transformer Training by Preventing Attention Entropy Collapse + """ + + def __init__(self, in_features, out_features, bias=True, **kwargs): + super().__init__(in_features, out_features, bias=bias, **kwargs) + self.register_buffer( + "u", nn.functional.normalize(torch.randn(in_features), dim=0) + ) + with torch.no_grad(): + sigma = self.get_sigma() + self.register_buffer("spectral_norm", sigma) + self.sigma = nn.Parameter(torch.ones(1)) + + def get_sigma(self): + with torch.no_grad(): + u = self.u + v = self.weight.mv(u) + v = nn.functional.normalize(v, dim=0) + u = self.weight.T.mv(v) + u = nn.functional.normalize(u, dim=0) + self.u.data.copy_(u) + return torch.einsum("c,cd,d->", v, self.weight, u) + + def get_weight(self): + sigma = self.get_sigma() + if self.training: + self.spectral_norm.data.copy_(sigma) + weight = (self.sigma / sigma) * self.weight + return weight + + def forward(self, x): + return nn.functional.linear(x, self.get_weight(), self.bias) + + +class SRConv1d(SRLinear): + def __init__( + self, + in_features, + out_features, + kernel_size, + stride: int = 1, + padding: str = "same", + bias: bool = True, + **kwargs, + ): + in_features = in_features * kernel_size + super().__init__(in_features, out_features, bias=bias, **kwargs) + nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5)) + self.kernel_size = kernel_size + self.stride = stride + self.padding = padding + + def forward(self, x): + in_features = self.in_features // self.kernel_size + weight = self.get_weight().view( + self.out_features, in_features, self.kernel_size + ) + return nn.functional.conv1d( + x, weight, bias=self.bias, stride=self.stride, padding=self.padding + ) + + +def TransposeSRConv1d( + *args, kernel_size: int = 3, padding: str = "same", **kwargs +) -> nn.Sequential: + """ + Transpose -> SRConv1d + """ + return nn.Sequential( + Transpose(), + SRConv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs), + ) + + +def SRConv1dTranspose( + *args, kernel_size: int = 3, padding: str = "same", **kwargs +) -> nn.Sequential: + """ + SRConv1d -> Transpose + """ + return nn.Sequential( + SRConv1d(*args, kernel_size=kernel_size, padding=padding, **kwargs), + Transpose(), + ) + + +class ActivationBalancer(torch.nn.Module): + """ + Modifies the backpropped derivatives of a function to try to encourage, for + each channel, that it is positive at least a proportion `threshold` of the + time. It does this by multiplying negative derivative values by up to + (1+max_factor), and positive derivative values by up to (1-max_factor), + interpolated from 1 at the threshold to those extremal values when none + of the inputs are positive. + + Args: + num_channels: the number of channels + channel_dim: the dimension/axis corresponding to the channel, e.g. + -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative. + min_positive: the minimum, per channel, of the proportion of the time + that (x > 0), below which we start to modify the derivatives. + max_positive: the maximum, per channel, of the proportion of the time + that (x > 0), above which we start to modify the derivatives. + max_factor: the maximum factor by which we modify the derivatives for + either the sign constraint or the magnitude constraint; + e.g. with max_factor=0.02, the the derivatives would be multiplied by + values in the range [0.98..1.02]. + sign_gain_factor: determines the 'gain' with which we increase the + change in gradient once the constraints on min_positive and max_positive + are violated. + scale_gain_factor: determines the 'gain' with which we increase the + change in gradient once the constraints on min_abs and max_abs + are violated. + min_abs: the minimum average-absolute-value difference from the mean + value per channel, which we allow, before we start to modify + the derivatives to prevent this. + max_abs: the maximum average-absolute-value difference from the mean + value per channel, which we allow, before we start to modify + the derivatives to prevent this. + min_prob: determines the minimum probability with which we modify the + gradients for the {min,max}_positive and {min,max}_abs constraints, + on each forward(). This is done randomly to prevent all layers + from doing it at the same time. Early in training we may use + higher probabilities than this; it will decay to this value. + """ + + def __init__( + self, + num_channels: int, + channel_dim: int, + min_positive: float = 0.05, + max_positive: float = 0.95, + max_factor: float = 0.04, + sign_gain_factor: float = 0.01, + scale_gain_factor: float = 0.02, + min_abs: float = 0.2, + max_abs: float = 100.0, + min_prob: float = 0.1, + ): + super(ActivationBalancer, self).__init__() + self.num_channels = num_channels + self.channel_dim = channel_dim + self.min_positive = min_positive + self.max_positive = max_positive + self.max_factor = max_factor + self.min_abs = min_abs + self.max_abs = max_abs + self.min_prob = min_prob + self.sign_gain_factor = sign_gain_factor + self.scale_gain_factor = scale_gain_factor + + # count measures how many times the forward() function has been called. + # We occasionally sync this to a tensor called `count`, that exists to + # make sure it is synced to disk when we load and save the model. + self.cpu_count = 0 + self.register_buffer("count", torch.tensor(0, dtype=torch.int64)) + + def forward(self, x: Tensor) -> Tensor: + if ( + torch.jit.is_scripting() + or not x.requires_grad + or torch.jit.is_tracing() + ): + return _no_op(x) + + count = self.cpu_count + self.cpu_count += 1 + + if random.random() < 0.01: + # Occasionally sync self.cpu_count with self.count. + # count affects the decay of 'prob'. don't do this on every iter, + # because syncing with the GPU is slow. + self.cpu_count = max(self.cpu_count, self.count.item()) + self.count.fill_(self.cpu_count) + + # the prob of doing some work exponentially decreases from 0.5 till it hits + # a floor at min_prob (==0.1, by default) + prob = max(self.min_prob, 0.5 ** (1 + (count / 4000.0))) + + if random.random() < prob: + sign_gain_factor = 0.5 + if self.min_positive != 0.0 or self.max_positive != 1.0: + sign_factor = _compute_sign_factor( + x, + self.channel_dim, + self.min_positive, + self.max_positive, + gain_factor=self.sign_gain_factor / prob, + max_factor=self.max_factor, + ) + else: + sign_factor = None + + scale_factor = _compute_scale_factor( + x.detach(), + self.channel_dim, + min_abs=self.min_abs, + max_abs=self.max_abs, + gain_factor=self.scale_gain_factor / prob, + max_factor=self.max_factor, + ) + return ActivationBalancerFunction.apply( + x, + scale_factor, + sign_factor, + self.channel_dim, + ) + else: + return _no_op(x) + + +def penalize_abs_values_gt(x: Tensor, limit: float, penalty: float) -> Tensor: + """ + Returns x unmodified, but in backprop will put a penalty for the excess of + the absolute values of elements of x over the limit "limit". E.g. if + limit == 10.0, then if x has any values over 10 it will get a penalty. + + Caution: the value of this penalty will be affected by grad scaling used + in automatic mixed precision training. For this reasons we use this, + it shouldn't really matter, or may even be helpful; we just use this + to disallow really implausible values of scores to be given to softmax. + """ + x_sign = x.sign() + over_limit = (x.abs() - limit) > 0 + # The following is a memory efficient way to penalize the absolute values of + # x that's over the limit. (The memory efficiency comes when you think + # about which items torch needs to cache for the autograd, and which ones it + # can throw away). The numerical value of aux_loss as computed here will + # actually be larger than it should be, by limit * over_limit.sum(), but it + # has the same derivative as the real aux_loss which is penalty * (x.abs() - + # limit).relu(). + aux_loss = penalty * ((x_sign * over_limit).to(torch.int8) * x) + # note: we don't do sum() here on aux)_loss, but it's as if we had done + # sum() due to how with_loss() works. + x = with_loss(x, aux_loss) + # you must use x for something, or this will be ineffective. + return x + + +def _diag(x: Tensor): # like .diag(), but works for tensors with 3 dims. + if x.ndim == 2: + return x.diag() + else: + (batch, dim, dim) = x.shape + x = x.reshape(batch, dim * dim) + x = x[:, :: dim + 1] + assert x.shape == (batch, dim) + return x + + +def _whitening_metric(x: Tensor, num_groups: int): + """ + Computes the "whitening metric", a value which will be 1.0 if all the eigenvalues of + of the centered feature covariance are the same within each group's covariance matrix + and also between groups. + Args: + x: a Tensor of shape (*, num_channels) + num_groups: the number of groups of channels, a number >=1 that divides num_channels + Returns: + Returns a scalar Tensor that will be 1.0 if the data is "perfectly white" and + greater than 1.0 otherwise. + """ + assert x.dtype != torch.float16 + x = x.reshape(-1, x.shape[-1]) + (num_frames, num_channels) = x.shape + assert num_channels % num_groups == 0 + channels_per_group = num_channels // num_groups + x = x.reshape(num_frames, num_groups, channels_per_group).transpose(0, 1) + # x now has shape (num_groups, num_frames, channels_per_group) + # subtract the mean so we use the centered, not uncentered, covariance. + # My experience has been that when we "mess with the gradients" like this, + # it's better not do anything that tries to move the mean around, because + # that can easily cause instability. + x = x - x.mean(dim=1, keepdim=True) + # x_covar: (num_groups, channels_per_group, channels_per_group) + x_covar = torch.matmul(x.transpose(1, 2), x) + x_covar_mean_diag = _diag(x_covar).mean() + # the following expression is what we'd get if we took the matrix product + # of each covariance and measured the mean of its trace, i.e. + # the same as _diag(torch.matmul(x_covar, x_covar)).mean(). + x_covarsq_mean_diag = (x_covar ** 2).sum() / ( + num_groups * channels_per_group + ) + # this metric will be >= 1.0; the larger it is, the less 'white' the data was. + metric = x_covarsq_mean_diag / (x_covar_mean_diag ** 2 + 1.0e-20) + return metric + + +class WhiteningPenaltyFunction(torch.autograd.Function): + @staticmethod + def forward( + ctx, + x: Tensor, + num_groups: int, + whitening_limit: float, + grad_scale: float, + ) -> Tensor: + ctx.save_for_backward(x) + ctx.num_groups = num_groups + ctx.whitening_limit = whitening_limit + ctx.grad_scale = grad_scale + return x + + @staticmethod + def backward(ctx, x_grad: Tensor): + (x_orig,) = ctx.saved_tensors + with torch.enable_grad(): + with torch.cuda.amp.autocast(enabled=False): + x_detached = x_orig.to(torch.float32).detach() + x_detached.requires_grad = True + + metric = _whitening_metric(x_detached, ctx.num_groups) + + if random.random() < 0.005 or __name__ == "__main__": + logging.info( + f"Whitening: num_groups={ctx.num_groups}, num_channels={x_orig.shape[-1]}, " + f"metric={metric.item():.2f} vs. limit={ctx.whitening_limit}" + ) + + (metric - ctx.whitening_limit).relu().backward() + penalty_grad = x_detached.grad + scale = ctx.grad_scale * ( + x_grad.to(torch.float32).norm() + / (penalty_grad.norm() + 1.0e-20) + ) + penalty_grad = penalty_grad * scale + return x_grad + penalty_grad.to(x_grad.dtype), None, None, None + + +class Whiten(nn.Module): + def __init__( + self, + num_groups: int, + whitening_limit: float, + prob: Union[float, Tuple[float, float]], + grad_scale: float, + ): + """ + Args: + num_groups: the number of groups to divide the channel dim into before + whitening. We will attempt to make the feature covariance + within each group, after mean subtraction, as "white" as possible, + while having the same trace across all groups. + whitening_limit: a value greater than 1.0, that dictates how much + freedom we have to violate the constraints. 1.0 would mean perfectly + white, with exactly the same trace across groups; larger values + give more freedom. E.g. 2.0. + prob: the probability with which we apply the gradient modification + (also affects the grad scale). May be supplied as a float, + or as a pair (min_prob, max_prob) + + grad_scale: determines the scale on the gradient term from this object, + relative to the rest of the gradient on the attention weights. + E.g. 0.02 (you may want to use smaller values than this if prob is large) + """ + super(Whiten, self).__init__() + assert num_groups >= 1 + assert whitening_limit >= 1 + assert grad_scale >= 0 + self.num_groups = num_groups + self.whitening_limit = whitening_limit + if isinstance(prob, float): + assert 0 < prob <= 1 + self.prob = prob + else: + (self.min_prob, self.max_prob) = prob + assert 0 < self.min_prob < self.max_prob <= 1 + self.prob = self.max_prob + + self.grad_scale = grad_scale + + def forward(self, x: Tensor) -> Tensor: + """ + In the forward pass, this function just returns the input unmodified. + In the backward pass, it will modify the gradients to ensure that the + distribution in each group has close to (lambda times I) as the covariance + after mean subtraction, with the same lambda across groups. + For whitening_limit > 1, there will be more freedom to violate this + constraint. + + Args: + x: the input of shape (*, num_channels) + + Returns: + x, unmodified. You should make sure + you use the returned value, or the graph will be freed + and nothing will happen in backprop. + """ + if ( + not x.requires_grad + or random.random() > self.prob + or self.grad_scale == 0 + ): + return _no_op(x) + else: + if hasattr(self, "min_prob") and random.random() < 0.25: + # occasionally switch between min_prob and max_prob, based on whether + # we are above or below the threshold. + if ( + _whitening_metric(x.to(torch.float32), self.num_groups) + > self.whitening_limit + ): + # there would be a change to the grad. + self.prob = self.max_prob + else: + self.prob = self.min_prob + + return WhiteningPenaltyFunction.apply( + x, self.num_groups, self.whitening_limit, self.grad_scale + ) + + +class WithLoss(torch.autograd.Function): + @staticmethod + def forward(ctx, x: Tensor, y: Tensor): + ctx.y_shape = y.shape + return x + + @staticmethod + def backward(ctx, ans_grad: Tensor): + return ans_grad, torch.ones( + ctx.y_shape, dtype=ans_grad.dtype, device=ans_grad.device + ) + + +def with_loss(x, y): + if torch.jit.is_scripting() or torch.jit.is_tracing(): + return x + # returns x but adds y.sum() to the loss function. + return WithLoss.apply(x, y) + + +def _no_op(x: Tensor) -> Tensor: + if torch.jit.is_scripting() or torch.jit.is_tracing(): + return x + else: + # a no-op function that will have a node in the autograd graph, + # to avoid certain bugs relating to backward hooks + return x.chunk(1, dim=-1)[0] + + +class Identity(torch.nn.Module): + def __init__(self): + super(Identity, self).__init__() + + def forward(self, x): + return _no_op(x) + + +class MaxEig(torch.nn.Module): + """ + Modifies the backpropped derivatives of a function to try to discourage + that any given direction in activation space accounts for more than + a specified proportion of the covariance (e.g. 0.2). + + + Args: + num_channels: the number of channels + channel_dim: the dimension/axis corresponding to the channel, e.g. + -1, 0, 1, 2; will be interpreted as an offset from x.ndim if negative. + max_var_per_eig: the maximum proportion of the variance of the + features/channels, after mean subtraction, that can come from + any given eigenvalue. + min_prob: the minimum probability with which we apply this during any invocation + of forward(), assuming last time we applied the constraint it was + not active; supplied for speed. + scale: determines the scale with which we modify the gradients, relative + to the existing / unmodified gradients + """ + + def __init__( + self, + num_channels: int, + channel_dim: int, + max_var_per_eig: float = 0.2, + min_prob: float = 0.01, + scale: float = 0.01, + ): + super(MaxEig, self).__init__() + self.num_channels = num_channels + self.channel_dim = channel_dim + self.scale = scale + assert max_var_per_eig == 0.0 or max_var_per_eig > 1.0 / num_channels + self.max_var_per_eig = max_var_per_eig + + # we figure out the dominant direction using the power method: starting with + # a random vector, keep multiplying by the covariance and renormalizing. + with torch.no_grad(): + # arbitrary.. would use randn() but want to leave the rest of the model's + # random parameters unchanged for comparison + direction = torch.arange(num_channels).to(torch.float) + direction = direction / direction.norm() + self.register_buffer("max_eig_direction", direction) + + self.min_prob = min_prob + # cur_prob is the current probability we'll use to apply the ActivationBalancer. + # We'll regress this towards prob, each time we try to apply it and it is not + # active. + self.cur_prob = 1.0 + + def forward(self, x: Tensor) -> Tensor: + if ( + torch.jit.is_scripting() + or self.max_var_per_eig <= 0 + or random.random() > self.cur_prob + or torch.jit.is_tracing() + ): + return _no_op(x) + + with torch.cuda.amp.autocast(enabled=False): + eps = 1.0e-20 + orig_x = x + x = x.to(torch.float32) + with torch.no_grad(): + x = x.transpose(self.channel_dim, -1).reshape( + -1, self.num_channels + ) + x = x - x.mean(dim=0) + new_direction, coeffs = self._find_direction_coeffs( + x, self.max_eig_direction + ) + x_var = (x ** 2).mean() + x_residual = x - coeffs * new_direction + x_residual_var = (x_residual ** 2).mean() + + # `variance_proportion` is the proportion of the variance accounted for + # by the top eigen-direction. + variance_proportion = (x_var - x_residual_var) / ( + x_var + 1.0e-20 + ) + + # ensure new direction is nonzero even if x == 0, by including `direction`. + self._set_direction( + 0.1 * self.max_eig_direction + new_direction + ) + + if random.random() < 0.01 or __name__ == "__main__": + logging.info( + f"variance_proportion = {variance_proportion.item()}, shape={tuple(orig_x.shape)}, cur_prob={self.cur_prob}" + ) + + if variance_proportion >= self.max_var_per_eig: + # The constraint is active. Note, we should quite rarely + # reach here, only near the beginning of training if we are + # starting to diverge, should this constraint be active. + cur_prob = self.cur_prob + self.cur_prob = ( + 1.0 # next time, do the update with probability 1.0. + ) + return MaxEigLimiterFunction.apply( + orig_x, coeffs, new_direction, self.channel_dim, self.scale + ) + else: + # let self.cur_prob exponentially approach self.min_prob, as + # long as the constraint is inactive. + self.cur_prob = 0.75 * self.cur_prob + 0.25 * self.min_prob + return orig_x + + def _set_direction(self, direction: Tensor): + """ + Sets self.max_eig_direction to a normalized version of `direction` + """ + direction = direction.detach() + direction = direction / direction.norm() + direction_sum = direction.sum().item() + if direction_sum - direction_sum == 0: # no inf/nan + self.max_eig_direction[:] = direction + else: + logging.info( + f"Warning: sum of direction in MaxEig is {direction_sum}, " + "num_channels={self.num_channels}, channel_dim={self.channel_dim}" + ) + + def _find_direction_coeffs( + self, x: Tensor, prev_direction: Tensor + ) -> Tuple[Tensor, Tensor, Tensor]: + """ + Figure out (an approximation to) the proportion of the variance of a set of + feature vectors that can be attributed to the top eigen-direction. + Args: + x: a Tensor of shape (num_frames, num_channels), with num_frames > 1. + prev_direction: a Tensor of shape (num_channels,), that is our previous estimate + of the top eigen-direction, or a random direction if this is the first + iteration. Does not have to be normalized, but should be nonzero. + + Returns: (cur_direction, coeffs), where: + cur_direction: a Tensor of shape (num_channels,) that is the current + estimate of the top eigen-direction. + coeffs: a Tensor of shape (num_frames, 1) that minimizes, or + approximately minimizes, (x - coeffs * cur_direction).norm() + """ + (num_frames, num_channels) = x.shape + assert num_channels > 1 and num_frames > 1 + assert prev_direction.shape == (num_channels,) + # `coeffs` are the coefficients of `prev_direction` in x. + # actually represent the coeffs up to a constant positive factor. + coeffs = (x * prev_direction).sum(dim=1, keepdim=True) + 1.0e-10 + cur_direction = (x * coeffs).sum(dim=0) / ( + (coeffs ** 2).sum() + 1.0e-20 + ) + return cur_direction, coeffs + + +class DoubleSwishFunction(torch.autograd.Function): + """ + double_swish(x) = x * torch.sigmoid(x-1) + This is a definition, originally motivated by its close numerical + similarity to swish(swish(x)), where swish(x) = x * sigmoid(x). + + Memory-efficient derivative computation: + double_swish(x) = x * s, where s(x) = torch.sigmoid(x-1) + double_swish'(x) = d/dx double_swish(x) = x * s'(x) + x' * s(x) = x * s'(x) + s(x). + Now, s'(x) = s(x) * (1-s(x)). + double_swish'(x) = x * s'(x) + s(x). + = x * s(x) * (1-s(x)) + s(x). + = double_swish(x) * (1-s(x)) + s(x) + ... so we just need to remember s(x) but not x itself. + """ + + @staticmethod + def forward(ctx, x: Tensor) -> Tensor: + requires_grad = x.requires_grad + x_dtype = x.dtype + if x.dtype == torch.float16: + x = x.to(torch.float32) + + s = torch.sigmoid(x - 1.0) + y = x * s + + if requires_grad: + deriv = y * (1 - s) + s + # notes on derivative of x * sigmoid(x - 1): + # https://www.wolframalpha.com/input?i=d%2Fdx+%28x+*+sigmoid%28x-1%29%29 + # min \simeq -0.043638. Take floor as -0.043637 so it's a lower bund + # max \simeq 1.1990. Take ceil to be 1.2 so it's an upper bound. + # the combination of "+ torch.rand_like(deriv)" and casting to torch.uint8 (which + # floors), should be expectation-preserving. + floor = -0.043637 + ceil = 1.2 + d_scaled = (deriv - floor) * ( + 255.0 / (ceil - floor) + ) + torch.rand_like(deriv) + if __name__ == "__main__": + # for self-testing only. + assert d_scaled.min() >= 0.0 + assert d_scaled.max() < 256.0 + d_int = d_scaled.to(torch.uint8) + ctx.save_for_backward(d_int) + if x.dtype == torch.float16 or torch.is_autocast_enabled(): + y = y.to(torch.float16) + return y + + @staticmethod + def backward(ctx, y_grad: Tensor) -> Tensor: + (d,) = ctx.saved_tensors + # the same constants as used in forward pass. + floor = -0.043637 + ceil = 1.2 + d = d * ((ceil - floor) / 255.0) + floor + return y_grad * d + + +class DoubleSwish(torch.nn.Module): + def forward(self, x: Tensor) -> Tensor: + """Return double-swish activation function which is an approximation to Swish(Swish(x)), + that we approximate closely with x * sigmoid(x-1). + """ + if torch.jit.is_scripting() or torch.jit.is_tracing(): + return x * torch.sigmoid(x - 1.0) + return DoubleSwishFunction.apply(x) + + +def BalancedDoubleSwish( + d_model, channel_dim=-1, max_abs=10.0, min_prob=0.25 +) -> nn.Sequential: + """ + ActivationBalancer -> DoubleSwish + """ + balancer = ActivationBalancer( + d_model, channel_dim=channel_dim, max_abs=max_abs, min_prob=min_prob + ) + return nn.Sequential( + balancer, + DoubleSwish(), + ) + + +def _test_max_eig(): + for proportion in [0.1, 0.5, 10.0]: + logging.info(f"proportion = {proportion}") + x = torch.randn(100, 128) + direction = torch.randn(128) + coeffs = torch.randn(100, 1) + x += proportion * direction * coeffs + + x.requires_grad = True + + num_channels = 128 + m = MaxEig( + num_channels, 1, 0.5, scale=0.1 # channel_dim # max_var_per_eig + ) # grad_scale + + for _ in range(4): + y = m(x) + + y_grad = torch.randn_like(x) + y.backward(gradient=y_grad) + + if proportion < 0.2: + assert torch.allclose(x.grad, y_grad, atol=1.0e-02) + elif proportion > 1.0: + assert not torch.allclose(x.grad, y_grad) + + +def _test_whiten(): + for proportion in [0.1, 0.5, 10.0]: + logging.info(f"_test_whiten(): proportion = {proportion}") + x = torch.randn(100, 128) + direction = torch.randn(128) + coeffs = torch.randn(100, 1) + x += proportion * direction * coeffs + + x.requires_grad = True + + num_channels = 128 + m = Whiten( + 1, 5.0, prob=1.0, grad_scale=0.1 # num_groups # whitening_limit, + ) # grad_scale + + for _ in range(4): + y = m(x) + + y_grad = torch.randn_like(x) + y.backward(gradient=y_grad) + + if proportion < 0.2: + assert torch.allclose(x.grad, y_grad) + elif proportion > 1.0: + assert not torch.allclose(x.grad, y_grad) + + +def _test_activation_balancer_sign(): + probs = torch.arange(0, 1, 0.01) + N = 1000 + x = 1.0 * ( + (2.0 * (torch.rand(probs.numel(), N) < probs.unsqueeze(-1))) - 1.0 + ) + x = x.detach() + x.requires_grad = True + m = ActivationBalancer( + probs.numel(), + channel_dim=0, + min_positive=0.05, + max_positive=0.95, + max_factor=0.2, + min_abs=0.0, + ) + + y_grad = torch.sign(torch.randn(probs.numel(), N)) + + y = m(x) + y.backward(gradient=y_grad) + print("_test_activation_balancer_sign: x = ", x) + print("_test_activation_balancer_sign: y grad = ", y_grad) + print("_test_activation_balancer_sign: x grad = ", x.grad) + + +def _test_activation_balancer_magnitude(): + magnitudes = torch.arange(0, 1, 0.01) + N = 1000 + x = torch.sign(torch.randn(magnitudes.numel(), N)) * magnitudes.unsqueeze( + -1 + ) + x = x.detach() + x.requires_grad = True + m = ActivationBalancer( + magnitudes.numel(), + channel_dim=0, + min_positive=0.0, + max_positive=1.0, + max_factor=0.2, + min_abs=0.2, + max_abs=0.8, + min_prob=1.0, + ) + + y_grad = torch.sign(torch.randn(magnitudes.numel(), N)) + + y = m(x) + y.backward(gradient=y_grad) + print("_test_activation_balancer_magnitude: x = ", x) + print("_test_activation_balancer_magnitude: y grad = ", y_grad) + print("_test_activation_balancer_magnitude: x grad = ", x.grad) + + +def _test_basic_norm(): + num_channels = 128 + m = BasicNorm(num_channels=num_channels, channel_dim=1) + + x = torch.randn(500, num_channels) + + y = m(x) + + assert y.shape == x.shape + x_rms = (x ** 2).mean().sqrt() + y_rms = (y ** 2).mean().sqrt() + print("x rms = ", x_rms) + print("y rms = ", y_rms) + assert y_rms < x_rms + assert y_rms > 0.5 * x_rms + + +def _test_double_swish_deriv(): + x = torch.randn(10, 12, dtype=torch.double) * 3.0 + x.requires_grad = True + m = DoubleSwish() + + tol = (1.2 - (-0.043637)) / 255.0 + torch.autograd.gradcheck(m, x, atol=tol) + + # for self-test. + x = torch.randn(1000, 1000, dtype=torch.double) * 3.0 + x.requires_grad = True + y = m(x) + + +def _test_softmax(): + a = torch.randn(2, 10, dtype=torch.float64) + b = a.clone() + a.requires_grad = True + b.requires_grad = True + a.softmax(dim=1)[:, 0].sum().backward() + print("a grad = ", a.grad) + softmax(b, dim=1)[:, 0].sum().backward() + print("b grad = ", b.grad) + assert torch.allclose(a.grad, b.grad) + + +if __name__ == "__main__": + logging.getLogger().setLevel(logging.INFO) + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + _test_softmax() + _test_whiten() + _test_max_eig() + _test_activation_balancer_sign() + _test_activation_balancer_magnitude() + _test_basic_norm() + _test_double_swish_deriv() diff --git a/tts/VALL-E-X/modules/scheduler.py b/tts/VALL-E-X/modules/scheduler.py new file mode 100644 index 0000000..df8ffb4 --- /dev/null +++ b/tts/VALL-E-X/modules/scheduler.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# Copyright 2023 (authors: Feiteng Li) +# +# See ../../../../LICENSE for clarification regarding multiple authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import torch + +from modules.optim import Eden + + +def calc_lr(step, dim_embed, warmup_steps): + return dim_embed ** (-0.5) * min( + step ** (-0.5), step * warmup_steps ** (-1.5) + ) + + +class NoamScheduler(torch.optim.lr_scheduler._LRScheduler): + def __init__( + self, + base_lr: float, + optimizer: torch.optim.Optimizer, + dim_embed: int, + warmup_steps: int, + last_epoch: int = -1, + verbose: bool = False, + ) -> None: + + self.dim_embed = dim_embed + self.base_lr = base_lr + self.warmup_steps = warmup_steps + self.num_param_groups = len(optimizer.param_groups) + + super().__init__(optimizer, last_epoch, verbose) + + def get_lr(self) -> float: + lr = self.base_lr * calc_lr( + self._step_count, self.dim_embed, self.warmup_steps + ) + return [lr] * self.num_param_groups + + def set_step(self, step: int): + self._step_count = step + + +def get_scheduler(params, optimizer): + if params.scheduler_name.lower() == "eden": + scheduler = Eden(optimizer, 5000, 4, warmup_batches=params.warmup_steps) + elif params.scheduler_name.lower() == "noam": + scheduler = NoamScheduler( + params.base_lr, + optimizer, + params.decoder_dim, + warmup_steps=params.warmup_steps, + ) + # scheduler.set_step(params.start_batch or params.batch_idx_train) + elif params.scheduler_name.lower() == "cosine": + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( + params.warmup_steps, + optimizer, + eta_min=params.base_lr, + ) + else: + raise NotImplementedError(f"{params.scheduler_name}") + + return scheduler diff --git a/tts/VALL-E-X/modules/transformer.py b/tts/VALL-E-X/modules/transformer.py new file mode 100644 index 0000000..ea8826b --- /dev/null +++ b/tts/VALL-E-X/modules/transformer.py @@ -0,0 +1,683 @@ +import copy +import numbers +from functools import partial +from typing import Any, Callable, List, Optional, Tuple, Union + +import torch +from torch import Tensor, nn +from torch.nn import functional as F + +from .activation import MultiheadAttention +from .scaling import ActivationBalancer, BalancedDoubleSwish +from .scaling import BasicNorm as _BasicNorm + +_shape_t = Union[int, List[int], torch.Size] + + +class LayerNorm(nn.Module): + __constants__ = ["normalized_shape", "eps", "elementwise_affine"] + normalized_shape: Tuple[int, ...] + eps: float + elementwise_affine: bool + + def __init__( + self, + normalized_shape: _shape_t, + eps: float = 1e-5, + elementwise_affine: bool = True, + device=None, + dtype=None, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(LayerNorm, self).__init__() + if isinstance(normalized_shape, numbers.Integral): + # mypy error: incompatible types in assignment + normalized_shape = (normalized_shape,) # type: ignore[assignment] + self.normalized_shape = tuple(normalized_shape) # type: ignore[arg-type] + self.eps = eps + self.elementwise_affine = elementwise_affine + if self.elementwise_affine: + self.weight = nn.Parameter( + torch.empty(self.normalized_shape, **factory_kwargs) + ) + self.bias = nn.Parameter( + torch.empty(self.normalized_shape, **factory_kwargs) + ) + else: + self.register_parameter("weight", None) + self.register_parameter("bias", None) + + self.reset_parameters() + + def reset_parameters(self) -> None: + if self.elementwise_affine: + nn.init.ones_(self.weight) + nn.init.zeros_(self.bias) + + def forward(self, input: Tensor, embedding: Any = None) -> Tensor: + if isinstance(input, tuple): + input, embedding = input + return ( + F.layer_norm( + input, + self.normalized_shape, + self.weight, + self.bias, + self.eps, + ), + embedding, + ) + + assert embedding is None + return F.layer_norm( + input, self.normalized_shape, self.weight, self.bias, self.eps + ) + + def extra_repr(self) -> str: + return ( + "{normalized_shape}, eps={eps}, " + "elementwise_affine={elementwise_affine}".format(**self.__dict__) + ) + + +class AdaptiveLayerNorm(nn.Module): + r"""Adaptive Layer Normalization""" + + def __init__(self, d_model, norm) -> None: + super(AdaptiveLayerNorm, self).__init__() + self.project_layer = nn.Linear(d_model, 2 * d_model) + self.norm = norm + self.d_model = d_model + self.eps = self.norm.eps + + def forward(self, input: Tensor, embedding: Tensor = None) -> Tensor: + if isinstance(input, tuple): + input, embedding = input + weight, bias = torch.split( + self.project_layer(embedding), + split_size_or_sections=self.d_model, + dim=-1, + ) + return (weight * self.norm(input) + bias, embedding) + + weight, bias = torch.split( + self.project_layer(embedding), + split_size_or_sections=self.d_model, + dim=-1, + ) + return weight * self.norm(input) + bias + + +class BasicNorm(_BasicNorm): + def __init__( + self, + d_model: int, + eps: float = 1e-5, + device=None, + dtype=None, + ): + super(BasicNorm, self).__init__(d_model, eps=eps) + + def forward(self, input: Tensor, embedding: Any = None) -> Tensor: + if isinstance(input, tuple): + input, embedding = input + return ( + super(BasicNorm, self).forward(input), + embedding, + ) + + assert embedding is None + return super(BasicNorm, self).forward(input) + + +class BalancedBasicNorm(nn.Module): + def __init__( + self, + d_model: int, + eps: float = 1e-5, + device=None, + dtype=None, + ): + super(BalancedBasicNorm, self).__init__() + self.balancer = ActivationBalancer( + d_model, + channel_dim=-1, + min_positive=0.45, + max_positive=0.55, + max_abs=6.0, + ) + self.norm = BasicNorm(d_model, eps, device=device, dtype=dtype) + + def forward(self, input: Tensor, embedding: Any = None) -> Tensor: + if isinstance(input, tuple): + input, embedding = input + return self.norm((self.balancer(input), embedding)) + + assert embedding is None + return self.norm(self.balancer(input)) + + +class IdentityNorm(nn.Module): + def __init__( + self, + d_model: int, + eps: float = 1e-5, + device=None, + dtype=None, + ) -> None: + super(IdentityNorm, self).__init__() + + def forward(self, input: Tensor, embedding: Any = None) -> Tensor: + if isinstance(input, tuple): + return input + + assert embedding is None + return input + + +class TransformerEncoderLayer(nn.Module): + __constants__ = ["batch_first", "norm_first"] + + def __init__( + self, + d_model: int, + nhead: int, + dim_feedforward: int = 2048, + dropout: float = 0.1, + activation: Union[str, Callable[[Tensor], Tensor]] = F.relu, + batch_first: bool = False, + norm_first: bool = False, + device=None, + dtype=None, + linear1_self_attention_cls: nn.Module = nn.Linear, + linear2_self_attention_cls: nn.Module = nn.Linear, + linear1_feedforward_cls: nn.Module = nn.Linear, + linear2_feedforward_cls: nn.Module = nn.Linear, + layer_norm_cls: nn.Module = LayerNorm, + layer_norm_eps: float = 1e-5, + adaptive_layer_norm=False, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(TransformerEncoderLayer, self).__init__() + self.self_attn = MultiheadAttention( + d_model, + nhead, + dropout=dropout, + batch_first=batch_first, + linear1_cls=linear1_self_attention_cls, + linear2_cls=linear2_self_attention_cls, + **factory_kwargs, + ) + + # Implementation of Feedforward model + self.linear1 = linear1_feedforward_cls( + d_model, dim_feedforward, **factory_kwargs + ) + self.dropout = nn.Dropout(dropout) + self.linear2 = linear2_feedforward_cls( + dim_feedforward, d_model, **factory_kwargs + ) + + self.norm_first = norm_first + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + + # Legacy string support for activation function. + if isinstance(activation, str): + activation = _get_activation_fn(activation) + elif isinstance(activation, partial): + activation = activation(d_model) + elif activation == BalancedDoubleSwish: + activation = BalancedDoubleSwish(d_model) + + # # We can't test self.activation in forward() in TorchScript, + # # so stash some information about it instead. + # if activation is F.relu or isinstance(activation, torch.nn.ReLU): + # self.activation_relu_or_gelu = 1 + # elif activation is F.gelu or isinstance(activation, torch.nn.GELU): + # self.activation_relu_or_gelu = 2 + # else: + # self.activation_relu_or_gelu = 0 + self.activation = activation + + norm1 = layer_norm_cls(d_model, eps=layer_norm_eps, **factory_kwargs) + if layer_norm_cls == IdentityNorm: + norm2 = BalancedBasicNorm( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + else: + norm2 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + + if adaptive_layer_norm: + self.norm1 = AdaptiveLayerNorm(d_model, norm1) + self.norm2 = AdaptiveLayerNorm(d_model, norm2) + else: + self.norm1 = norm1 + self.norm2 = norm2 + + def __setstate__(self, state): + super(TransformerEncoderLayer, self).__setstate__(state) + if not hasattr(self, "activation"): + self.activation = F.relu + + def forward( + self, + src: Tensor, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + ) -> Tensor: + r"""Pass the input through the encoder layer. + + Args: + src: the sequence to the encoder layer (required). + src_mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + x, stage_embedding = src, None + is_src_tuple = False + if isinstance(src, tuple): + x, stage_embedding = src + is_src_tuple = True + + if src_key_padding_mask is not None: + _skpm_dtype = src_key_padding_mask.dtype + if _skpm_dtype != torch.bool and not torch.is_floating_point( + src_key_padding_mask + ): + raise AssertionError( + "only bool and floating types of key_padding_mask are supported" + ) + + if self.norm_first: + x = x + self._sa_block( + self.norm1(x, stage_embedding), + src_mask, + src_key_padding_mask, + ) + x = x + self._ff_block(self.norm2(x, stage_embedding)) + else: + x = self.norm1( + x + self._sa_block(x, src_mask, src_key_padding_mask), + stage_embedding, + ) + x = self.norm2(x + self._ff_block(x), stage_embedding) + + if is_src_tuple: + return (x, stage_embedding) + return x + + def infer( + self, + src: Tensor, + src_mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + past_kv: Optional[Tensor] = None, + use_cache: bool = False, + ): + x, stage_embedding = src, None + is_src_tuple = False + if isinstance(src, tuple): + x, stage_embedding = src + is_src_tuple = True + + if src_key_padding_mask is not None: + _skpm_dtype = src_key_padding_mask.dtype + if _skpm_dtype != torch.bool and not torch.is_floating_point( + src_key_padding_mask + ): + raise AssertionError( + "only bool and floating types of key_padding_mask are supported" + ) + + if self.norm_first: + x_attn_out, kv = self.self_attn.infer( + self.norm1(x, stage_embedding), + attn_mask=src_mask, + key_padding_mask=src_key_padding_mask, + need_weights=False, + past_kv=past_kv, + use_cache=use_cache, + ) + x = x + x_attn_out + x = x + self._ff_block(self.norm2(x, stage_embedding)) + + if is_src_tuple: + return (x, stage_embedding) + return (x, kv) + + # self-attention block + def _sa_block( + self, + x: Tensor, + attn_mask: Optional[Tensor], + key_padding_mask: Optional[Tensor], + ) -> Tensor: + x = self.self_attn( + x, + x, + x, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask, + need_weights=False, + )[0] + return self.dropout1(x) + + # feed forward block + def _ff_block(self, x: Tensor) -> Tensor: + x = self.linear2(self.dropout(self.activation(self.linear1(x)))) + return self.dropout2(x) + + +class TransformerEncoder(nn.Module): + r"""TransformerEncoder is a stack of N encoder layers. Users can build the + BERT(https://arxiv.org/abs/1810.04805) model with corresponding parameters. + + Args: + encoder_layer: an instance of the TransformerEncoderLayer() class (required). + num_layers: the number of sub-encoder-layers in the encoder (required). + norm: the layer normalization component (optional). + enable_nested_tensor: if True, input will automatically convert to nested tensor + (and convert back on output). This will improve the overall performance of + TransformerEncoder when padding rate is high. Default: ``True`` (enabled). + + Examples:: + >>> encoder_layer = TransformerEncoderLayer(d_model=512, nhead=8) + >>> transformer_encoder = TransformerEncoder(encoder_layer, num_layers=6) + >>> src = torch.rand(10, 32, 512) + >>> out = transformer_encoder(src) + """ + __constants__ = ["norm"] + + def __init__(self, encoder_layer, num_layers, norm=None): + super(TransformerEncoder, self).__init__() + self.layers = _get_clones(encoder_layer, num_layers) + self.num_layers = num_layers + self.norm = norm + + def forward( + self, + src: Tensor, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + return_layer_states: bool = False, + ) -> Tensor: + r"""Pass the input through the encoder layers in turn. + + Args: + src: the sequence to the encoder (required). + mask: the mask for the src sequence (optional). + src_key_padding_mask: the mask for the src keys per batch (optional). + return_layer_states: return layers' state (optional). + + Shape: + see the docs in Transformer class. + """ + if return_layer_states: + layer_states = [] # layers' output + output = src + for mod in self.layers: + output = mod( + output, + src_mask=mask, + src_key_padding_mask=src_key_padding_mask, + ) + layer_states.append(output[0]) + + if self.norm is not None: + output = self.norm(output) + + return layer_states, output + + output = src + for mod in self.layers: + output = mod( + output, src_mask=mask, src_key_padding_mask=src_key_padding_mask + ) + + if self.norm is not None: + output = self.norm(output) + + return output + + def infer( + self, + src: Tensor, + mask: Optional[Tensor] = None, + src_key_padding_mask: Optional[Tensor] = None, + return_layer_states: bool = False, + past_kv: Optional[Tensor] = None, + use_cache: bool = False, + ): + if past_kv is None: + past_length = 0 + past_kv = tuple([None] * self.num_layers) + else: + past_length = past_kv[0][0].size(-2) + new_kv = () if use_cache else None + output = src + for mod, past_layer_kv in zip(self.layers, past_kv): + output, kv = mod.infer( + output, src_mask=mask, src_key_padding_mask=src_key_padding_mask, past_kv=past_layer_kv, use_cache=use_cache + ) + if use_cache: + new_kv = new_kv + (kv,) + + if self.norm is not None: + output = self.norm(output) + + return output, new_kv + + +class TransformerDecoderLayer(nn.Module): + __constants__ = ["batch_first", "norm_first"] + + def __init__( + self, + d_model: int, + nhead: int, + dim_feedforward: int = 2048, + dropout: float = 0.1, + activation: Union[str, Callable[[Tensor], Tensor]] = F.relu, + linear1_self_attention_cls: nn.Module = nn.Linear, + linear2_self_attention_cls: nn.Module = nn.Linear, + linear1_feedforward_cls: nn.Module = nn.Linear, + linear2_feedforward_cls: nn.Module = nn.Linear, + batch_first: bool = False, + norm_first: bool = False, + device=None, + dtype=None, + layer_norm_cls: nn.Module = LayerNorm, + layer_norm_eps: float = 1e-5, + adaptive_layer_norm=False, + ) -> None: + factory_kwargs = {"device": device, "dtype": dtype} + super(TransformerDecoderLayer, self).__init__() + self.self_attn = MultiheadAttention( + d_model, + nhead, + dropout=dropout, + batch_first=batch_first, + linear1_cls=linear1_self_attention_cls, + linear2_cls=linear2_self_attention_cls, + **factory_kwargs, + ) + self.multihead_attn = MultiheadAttention( + d_model, + nhead, + dropout=dropout, + batch_first=batch_first, + linear1_cls=linear1_self_attention_cls, + linear2_cls=linear2_self_attention_cls, + **factory_kwargs, + ) + # Implementation of Feedforward model + self.linear1 = linear1_feedforward_cls( + d_model, dim_feedforward, **factory_kwargs + ) + self.dropout = nn.Dropout(dropout) + self.linear2 = linear2_feedforward_cls( + dim_feedforward, d_model, **factory_kwargs + ) + + self.norm_first = norm_first + self.dropout1 = nn.Dropout(dropout) + self.dropout2 = nn.Dropout(dropout) + self.dropout3 = nn.Dropout(dropout) + + # Legacy string support for activation function. + if isinstance(activation, str): + self.activation = _get_activation_fn(activation) + elif isinstance(activation, partial): + self.activation = activation(d_model) + elif activation == BalancedDoubleSwish: + self.activation = BalancedDoubleSwish(d_model) + else: + self.activation = activation + + if adaptive_layer_norm: + norm1 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + norm2 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + norm3 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + + self.norm1 = AdaptiveLayerNorm(d_model, norm1) + self.norm2 = AdaptiveLayerNorm(d_model, norm2) + self.norm3 = AdaptiveLayerNorm(d_model, norm3) + else: + self.norm1 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + self.norm2 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + if layer_norm_cls == IdentityNorm: + self.norm3 = BalancedBasicNorm( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + else: + self.norm3 = layer_norm_cls( + d_model, eps=layer_norm_eps, **factory_kwargs + ) + + def forward( + self, + tgt: Tensor, + memory: Tensor, + tgt_mask: Optional[Tensor] = None, + memory_mask: Optional[Tensor] = None, + tgt_key_padding_mask: Optional[Tensor] = None, + memory_key_padding_mask: Optional[Tensor] = None, + ) -> Tensor: + r"""Pass the inputs (and mask) through the decoder layer. + + Args: + tgt: the sequence to the decoder layer (required). + memory: the sequence from the last layer of the encoder (required). + tgt_mask: the mask for the tgt sequence (optional). + memory_mask: the mask for the memory sequence (optional). + tgt_key_padding_mask: the mask for the tgt keys per batch (optional). + memory_key_padding_mask: the mask for the memory keys per batch (optional). + + Shape: + see the docs in Transformer class. + """ + tgt_is_tuple = False + if isinstance(tgt, tuple): + x, stage_embedding = tgt + tgt_is_tuple = True + else: + x, stage_embedding = tgt, None + + if self.norm_first: + x = x + self._sa_block( + self.norm1(x, stage_embedding), tgt_mask, tgt_key_padding_mask + ) + x = x + self._mha_block( + self.norm2(x, stage_embedding), + memory, + memory_mask, + memory_key_padding_mask, + ) + x = x + self._ff_block(self.norm3(x, stage_embedding)) + else: + x = self.norm1( + x + self._sa_block(x, tgt_mask, tgt_key_padding_mask), + stage_embedding, + ) + x = self.norm2( + x + + self._mha_block( + x, memory, memory_mask, memory_key_padding_mask + ), + stage_embedding, + ) + x = self.norm3(x + self._ff_block(x), stage_embedding) + + if tgt_is_tuple: + return (x, stage_embedding) + return x + + # self-attention block + def _sa_block( + self, + x: Tensor, + attn_mask: Optional[Tensor], + key_padding_mask: Optional[Tensor], + ) -> Tensor: + x = self.self_attn( + x, + x, + x, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask, + need_weights=False, + )[0] + return self.dropout1(x) + + # multihead attention block + def _mha_block( + self, + x: Tensor, + mem: Tensor, + attn_mask: Optional[Tensor], + key_padding_mask: Optional[Tensor], + ) -> Tensor: + x = self.multihead_attn( + x, + mem, + mem, + attn_mask=attn_mask, + key_padding_mask=key_padding_mask, + need_weights=False, + )[0] + return self.dropout2(x) + + # feed forward block + def _ff_block(self, x: Tensor) -> Tensor: + x = self.linear2(self.dropout(self.activation(self.linear1(x)))) + return self.dropout3(x) + + +def _get_clones(module, N): + return nn.ModuleList([copy.deepcopy(module) for i in range(N)]) + + +def _get_activation_fn(activation: str) -> Callable[[Tensor], Tensor]: + if activation == "relu": + return F.relu + elif activation == "gelu": + return F.gelu + + raise RuntimeError( + "activation should be relu/gelu, not {}".format(activation) + ) diff --git a/tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/README b/tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/README new file mode 100644 index 0000000..49a637c --- /dev/null +++ b/tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/README @@ -0,0 +1,98 @@ +Pretrained Punkt Models -- Jan Strunk (New version trained after issues 313 and 514 had been corrected) + +Most models were prepared using the test corpora from Kiss and Strunk (2006). Additional models have +been contributed by various people using NLTK for sentence boundary detection. + +For information about how to use these models, please confer the tokenization HOWTO: +http://nltk.googlecode.com/svn/trunk/doc/howto/tokenize.html +and chapter 3.8 of the NLTK book: +http://nltk.googlecode.com/svn/trunk/doc/book/ch03.html#sec-segmentation + +There are pretrained tokenizers for the following languages: + +File Language Source Contents Size of training corpus(in tokens) Model contributed by +======================================================================================================================================================================= +czech.pickle Czech Multilingual Corpus 1 (ECI) Lidove Noviny ~345,000 Jan Strunk / Tibor Kiss + Literarni Noviny +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +danish.pickle Danish Avisdata CD-Rom Ver. 1.1. 1995 Berlingske Tidende ~550,000 Jan Strunk / Tibor Kiss + (Berlingske Avisdata, Copenhagen) Weekend Avisen +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +dutch.pickle Dutch Multilingual Corpus 1 (ECI) De Limburger ~340,000 Jan Strunk / Tibor Kiss +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +english.pickle English Penn Treebank (LDC) Wall Street Journal ~469,000 Jan Strunk / Tibor Kiss + (American) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +estonian.pickle Estonian University of Tartu, Estonia Eesti Ekspress ~359,000 Jan Strunk / Tibor Kiss +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +finnish.pickle Finnish Finnish Parole Corpus, Finnish Books and major national ~364,000 Jan Strunk / Tibor Kiss + Text Bank (Suomen Kielen newspapers + Tekstipankki) + Finnish Center for IT Science + (CSC) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +french.pickle French Multilingual Corpus 1 (ECI) Le Monde ~370,000 Jan Strunk / Tibor Kiss + (European) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +german.pickle German Neue Zürcher Zeitung AG Neue Zürcher Zeitung ~847,000 Jan Strunk / Tibor Kiss + (Switzerland) CD-ROM + (Uses "ss" + instead of "ß") +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +greek.pickle Greek Efstathios Stamatatos To Vima (TO BHMA) ~227,000 Jan Strunk / Tibor Kiss +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +italian.pickle Italian Multilingual Corpus 1 (ECI) La Stampa, Il Mattino ~312,000 Jan Strunk / Tibor Kiss +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +norwegian.pickle Norwegian Centre for Humanities Bergens Tidende ~479,000 Jan Strunk / Tibor Kiss + (Bokmål and Information Technologies, + Nynorsk) Bergen +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +polish.pickle Polish Polish National Corpus Literature, newspapers, etc. ~1,000,000 Krzysztof Langner + (http://www.nkjp.pl/) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +portuguese.pickle Portuguese CETENFolha Corpus Folha de São Paulo ~321,000 Jan Strunk / Tibor Kiss + (Brazilian) (Linguateca) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +slovene.pickle Slovene TRACTOR Delo ~354,000 Jan Strunk / Tibor Kiss + Slovene Academy for Arts + and Sciences +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +spanish.pickle Spanish Multilingual Corpus 1 (ECI) Sur ~353,000 Jan Strunk / Tibor Kiss + (European) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +swedish.pickle Swedish Multilingual Corpus 1 (ECI) Dagens Nyheter ~339,000 Jan Strunk / Tibor Kiss + (and some other texts) +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- +turkish.pickle Turkish METU Turkish Corpus Milliyet ~333,000 Jan Strunk / Tibor Kiss + (Türkçe Derlem Projesi) + University of Ankara +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- + +The corpora contained about 400,000 tokens on average and mostly consisted of newspaper text converted to +Unicode using the codecs module. + +Kiss, Tibor and Strunk, Jan (2006): Unsupervised Multilingual Sentence Boundary Detection. +Computational Linguistics 32: 485-525. + +---- Training Code ---- + +# import punkt +import nltk.tokenize.punkt + +# Make a new Tokenizer +tokenizer = nltk.tokenize.punkt.PunktSentenceTokenizer() + +# Read in training corpus (one example: Slovene) +import codecs +text = codecs.open("slovene.plain","Ur","iso-8859-2").read() + +# Train tokenizer +tokenizer.train(text) + +# Dump pickled tokenizer +import pickle +out = open("slovene.pickle","wb") +pickle.dump(tokenizer, out) +out.close() + +--------- diff --git a/tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/czech.pickle b/tts/VALL-E-X/nltk_data/tokenizers/punkt/PY3/czech.pickle new file mode 100644 index 0000000000000000000000000000000000000000..d848a550a50a1a1a41c6195bf0bb2bbe5866ef13 GIT binary patch literal 1119050 zcmaI92Y6ja@&0d&2_~4(dozv+p(OO)Oz6FXsqvBSm2`Fc+>%_;+a;9HTL?`+00AaI zLJK$)S@hm}@9?Gf`v1K19_c6wzyFgw*E;Xc?%6UsJ2N}G=ZP1b?X-r?d5z0+8# zAKx2wn!_XSr{4$GJHvWsw*H_pEspGOyY`d%7kFHM`?33v+_r7oiLIhDf8w}^LI3}! zn0rAqD(34Cf$qru4obMtJufh`{|FV3MfZ>Rf7c-ij_hA}|Aj@~E23x*`aA#M^!F1* zeOTA4F2bvhd|dyc$L`-n=Gktm)txPdjc#Yq-!(fkYP5!p&fv&FeRyR5Vvp({b?p8N zh^&9{{X6-$n2BPcXczrUO#7G2l%@WqvKGzm+H$KN{Yy{#m&tB8`smK6JyVaPkM3VK z^A3jHR(-g7uz$H}|MHplBy&p?~#he|IJn{cGf#YU6GUnyZKUd#3%Px0bj!fmVa+^om-&-M{9vf6P{* zYh~x0)GBI|L8~~ads4T5?6iOFtz_59+6~uEJHCJ2Y5#g#laJefNh&`W)#}4;=kV$6 z`Ye^A!pBeh*Ut)%`#0EsAxIXAF%7)ow11<88@M-H#b`j?ZanSZBqgXZ|~>BA09 zxXZME*R3RX%eK@;Gt~L+)BZiSrr$HWzuSw|5lsMn)dI#mFzy*{q08Q`2KyT z{rhcAzJCI1%fl!JGoxtsA297daN%m1X}&ug_8&CuKX_}Vhh)1uMRb>#`knqmr~Uo3 zTg*Rm`1IE5%0jpQuxbC{VCuwiV+`*$03R{!Ka#>l*wa)i7VG^-P5X~7N3>nULU+{a zpE&IwfJ}m-U9{@;arSO8?a!2NS!~eiL9^)3PW!cmn`UhXow-)OKJCwKt?j(WdiCI7 zc=+`G!nEI5xOmoUwuopPetOjVi_?CytV6BU9W5{|TGM_TNSxOcqxCz}es^opy-XiO zk1^^``$6?2Z5T8dr`aD&`@^kSM#+o4)#VzM8BhBsZ6!E435KhOYMpw2GVL#wg|@p( zMGTAnW2XJb!oS;j&;0TI$4&c>-&(>GvR3tB(Hr%jIPE{F?4VX-zUV)B+JDMc`ln{4 z8Ma}aiTAW=f4Y_8=~<_?y0F_nW!itnR)S|H0isJKJZsv2_WFp{51-m7qJQeN|D1B* zXBzbulf2P??zI0rs+JvAUu2+*{`05(7m!o@(RR+F|H5hiMO$C@;v8{xV*e%6{!6ze zzbt|A>Y?Z!KDC21J#E^5`BtLSv&yUIbh=UWUoq{!ax1~B%)BmzLS8-Xzoz8Ba)68e z^0fcjt@&P;Ojc>WWCitbJ~CV)(r1RhAC?OcTW3fEnF(Qc!2m}9yR-CPy6p; zbhNJ4>Vw97r~mG0|2aVw|HN2dMNtvL@{U_^J&K&R-hP5U1$`DW1Tc9y!)|Jbzu@vXT& zk?*sJ9rZss?SE=3#iw(C*H^hw|1;D6XBR%4*#=#f)9$$cxoQ9NTeE#3t5dR8v;W0u z|4Um*zMNg5BKlvM_P@H$HRCP=N`HTC+W-32EZ^ARiK73_Y5!YW)4!eRYvir*1`#&l!shbB<2A05j}=nfCv>mF90rvpy7~{@3XaWpbTMqUAGwuJk93`~3+UoM*Q)3%S-83%1@0Q%LU6|xn){b@Xf=RmLe~?~i z9a|iA8mV!u#1Tn-VScMYTxiA4gfGHx{b_L||2!Zr%J2S*@uC3=^@)Qa3jq3Q>;hi6 z*rVcN{CjLxq&-5BY1iW@Wsvw>WN}6-8^^_kXtC&Z;}ZH?1G8A~z)5jQJ&TLljc68e zsddu!BrdJLrB)6)MJ+C)r;GB$ir*Vfm>H`paan;E1Z0Vh%jrQ~GG05kHd#9(E-ygu z9d_di`a4(B+GJ)DSKLS{;!65kgt(J==||3Q;+=)q~AR zTuqO9=dg<`U0aT;3sCzy#TdOVcCXVnsnWS|4L#~zi?w>}*?3r8j-&OU7tiU%d)J>> zfd_F-J&B{bOP$r_cB56qF@jXHneM`fj%>xXgqY zlN!gakh&Ed-5>u3_L3Xus)mi-;uTYTrp}5YmM3Ak#QHL zZrFp34$xhJo)&lG_W^Ntemy^Dm?Ijldw|??cTS~F{iKNnLObqR!2{Z=Wq%gxdl^=( zS`Ey^qSaXHj*DU3+X?C*Mhd-b2zVcZvRFIydAd>zGP@A>HLhWrU6{_WF9Et$2XIeL2R?d?+3$Xiw4&TgBm1>EFe8kW19dF}2Z;8i!A1*2aT{ z?{I?)*^(Vu2l0@KoLSIVTkdoRL!{6{h40&c6vYhcGxc6~I1X$K3erkLu;G$Vi> zRzXWBKDF*B9&Si%(yF&wjd+BB9a&ikYt^6?j}&-dQ=?YmQO+pV9raq1&ZsqsM^{*v z<%Idri4%?4G#N05Y7HuPph9I6A&Vi7VpD$9;K8slj2UB^L)~7zNeyR>h?U1nMJsB0 zp2;5T&cOObNA_^kO_qXa++u7o`C`tXdcC%GW~;t-F0*hjjCtc~O>gbYL3f5XF9-}}HM=Z15>FkIi=`!^QSzA>*sjra!lo*>qGwE_S+r(G?dS{Gll7&p zjn%U{5rPhQk(|-ImaQm;8JN~W1T`1SbQnXIm=i~{EV{KKMg}#eU1ShbtVxB&Mm185 z9GXp>BxFa*zY$Ey3UuksCC=jaXenYF!`j3V89Mn&&Ly7^5$l@;)zw6 zcyqdP35ji=WH<%*;nSNMh9_6h9FD98YQa+s<=~@k;}9ujOFN$GB)tqtJ4Se~o)b?K zxX+B%=(W0SWYhHggoonk2G#Gi?x2uvOQlXZKk0HjqXJuvnp8G~#WMwZJZ48!s~gWU zvbjDh@oeY46tBs?ZXs3>PAY_cJ+&%%y$J{7IRZUbm_LiMOyar5w3KQ(JU$>t*sI4 zhAMTsAj|Ej&$a4J7Fk;O3R9{X>+e{JS5{!VuuLrCRR&$(ZoFEL+sk@%;x$f9)waM) zU=A#{CApTJpe`IYYIQV-QLtjQ>g{-~lhorw97k6s?Tc*yrC+zIqIbwFWx9@M=BSqXVoV0CO_*Xs28eMtAnnyQe|B` z7gvO;|7KxxUPF1trA%$!V!~Y1dfnEji+8B#tex48w>m}LDS^JYwoH`{3iSHgE6Zra z+p03UGqu%~7E{HwJyR*$Q_|vOT|J0yOyY99y(*h!rHR>1+3zrzO=0|dUG-us-YGb8 z$|Bku2+$(V61F2T1P1M68lUa#no8rzLa;Qg94A#V|B81xRjpaC_$1yfaGy1+8TANq zqCnEcdrU3i1N*KmBWd0%$V`nmlsXviGnM#ZsX!rKI|T*E0{$FROF+*S1B}*qzYz(P zCDu6pr-JbT7oi1q^_+1h4psST-64`N&K0u9imi}(d{9U&=gCG%%kd$XWDRRk#-j$T z9Updny%$gF2ra+KT=|Hpg|dWdyfoyiLhob)punLPhXrhIbSu^bnN5*NsDANLA^TjP z0zRk5I+OUA(A`;Ybc;3$`NxIq%tc|`?bPV(lwF??d{hns&bOuojr6cjx`b*@A!$A( z%&u+5YbY05Jw9zhF%F(OmZj6E#V9^w_!7VfH8k_3)q|;;Fua}7Sp!l zbH;O&FfoRs-Ym=6=Z%)^upran3x-lD?+qJmX;3L3zvu+9A?$i`W6&NQhZ?~e$Y1d# zr*3M~!T7SV^SrYiUok2UY8R*=llZC-i@i21NdF=CiuhVZxO(=W&E)^Ok?fDgQ0jAm zV+@qc;6CwAcu53Su6$}8c>gKn^u+Y1-2#1Uwp@y zs(DhkHE1^CyF#qhuD_-o-*bV|5Ej{0>c;m4>`Z04iGUrfol$fKLu7vZz@+NgIg}Rp z=W|VsA1a~bD5@Ea_>mD!>Tapx6!Bw$JG_iUy*6&3e7E8!Ce>UWAk+umnn->Br%qBA zph`1ws6{(|W>k&sU?8DW74h?mvKX-?goL8PzYuDr7>2=$koLI}zcjHVE9NnyP-BxX z)zV)%CCvr39WnfCL1l-p4?_|EXH%`ir<@5yupez*I=VU*4!Ve5^Cs~4D26;c(q`jODjPK_>;+0p%hQ#{IenT zUNh?LIy^w`zZg~T(fjEB7!Q9Hw!@177)&|LIrq1Umb~Z4*&cs4CMKZgnv_~kWhMTh zgq^8n_qyXQ3rhUcw5n7aU6_*NUxLhoGu_2foBDTEW~WZ$(eG{W{l^JvLZ@Dum#z43 zMNV6_AH-%_ojJS>heOGS<4JGSY{dmCWU**=aU*RPvY2b_QZdl##(HrR`kKp2QJCyzC(+5rCahT-aEu6ResT!r&N)JDpo|quG^I zoMo@$A{B^beXxdwT#q9SCja;9mXIrP(Ta2c`-`R9v8y5-h2(rrpot3naac2U6Xml14Tc|4I93Ld_!F*O$f&1Sij;JBR7U6vZi_X0g0mc6`-&DT>Y zhD9(=L*we)QaCWeSFAv$P7HJVbkF2_}? z0-1vBOvo{Gwex^DPGuQg-QbpbPGYxk+ob6sK`XsRm4ChJhhmRFn?+?mwc==DwvxwP zmd?YcA}Hw7y)Hog$GEzuhH=f!#fsrO*z3nsAR1J6fKgwMYgPG~gw0Wmq;VZOhBEyB# zomoWTNPmUZ+vAPTcWmgZE20j85W&EQ`u;sjjnvE>E zgOCHI;WC5RQmH2xROgo6Q^XyG&6Jb4CIwMC_ML>*?3gK*=-uVGvmnp5nHY7jGh1;N zVb&W4jp0bkB3*D-p}X_l8%svqO`thY-j{68-A!25hc=>R-@}+%j*keViLDRghv z2a_XKR|at}L00WXVuDhz-rM=5w9JcFsAjuZ?6Sa}z*#v3w5aeh=q zSuSsK&H5KO&Yf`WexJbQ77QClR6ediWqctDxuzOrnondyEak?Y| zzWhcfW(6FPmeb)=WrRdch+{|Ar3*S%-5IqA4OY(~&|@QNba2PX=tI5{K=OBOtpj2F@cc zVsmPcE6&iEWPKA|2wh+##C?Zihq@hOAqUbk)Lx7@$yw6iCo}_1HTOIuWKLGn_B3vZ z9#W4SK9x0Z;v(f~YA?>kk{{K~B{rlb>sb*W<6?z&YuYejNgu{zjjG>ggOvx><#Em~ zh!`9^t3^ECR1%}|KrFAFk&~IJjDqwPnr4c$#6gnN>BW6%{h3L0{$C@^r(h zA66IfD0TFzQw-*qU?LDo^^7X_{AgUqRk0#p-!l!XE-dxz2|r6f$@dH09;@`TO`={U zcnsfru|!{=YE(nWUV5WTNC;9QMg4P}wDIzKJhv)hu|8%eC_C|aMpg55c~Fn%8}vXQ zA1~13j0I)UCTuJHN;q#`Xga-w@$WI+UnImeZ`B9l))xyokPi59V-|~~zJy;74@Ns) z;_PZv4Fez=Gb0LpX_a9nFt!+K^oU$+UgiuMTyJO4d8ZlExNQhTnB(PwZJ)9Ii>I!% zn@%^mSZs-%9E>i?S3O_hgzQUM>cf#3I+d4KI!T<`9u;jSSG>x_tLhX_5woMkdc0bw z8EqI8BGctHRptRov;2Yft-Q%#+?Q?9*WmHry9Gl z#L?I@1g|$14{mBrIGx@g+-3-hWa`|!{*5j~y@*>9wTXh?R25vk?_jL_FL)*1Y&bRa z`goGj9W9ivw-{GP(JSoZ9gMdMvNBK=yc`EjDeF<2zf`$&IiaPzO=;cAJOW!3-sn<>Q+GS5H#mq_|>!GoqDu#vNSVUI|UhbmLM}I9gQO-&K9z})U&9C z1lcg&CDdAx!VFTWdbcUX?t`L(t1aFmWM|F{s;b>SGUJSj_iiqf4Q*;o%iedMLeaii zR?aad<4^hH{eDh?L;roik9wzEz1_t)BxG^%Xj$}~{+e^X=PX{b%c;*mect&}BFGPB!{7@>%el68ri|V%R+Kmi zG4OM0f5~VnK;>L*#Fs0O9ktbim=HKq1=_mD=%tg<16Sg!LTy*n=g=34VwjJw3AEXW zx`Fh;3-NWADUNDkHp-W3Dj_w!WgOoWa$ehm+4U`>YK)=_d38I!EzBPDENv3s5n|J0 zW5##myFyEUCp${rVSH~3GK%jTQ5Ve4E;Kr+3jM&vOuV2DH@%PN)_>ws+=_CW?>C+=1+}lh?9DN=|xDw&#K~?&+I^J z(?0haeqI&7F&T^ag;2}B@r1Z9f9d+}>OsW;F8 znIO^@{?DLlMgS_Jm*X8czJDY5!2U}ED?kXbj?Y4yKeqdR>snN2wEeoZ&E4^Lr>F)aHqsYyZjIs}0zF}gSCV`GpF->wVt1IRo0a&N zAg`VsSbV96ca$mmZ^71M)JqB=amW8h3AW2JdQge~H6}Sc(>OV{@dxh}%ds2~la(4t zJ1!vHCbi1UGbORzpoMm=*BHbFh0J93UYBqWR9lSJB6bM3B^yM}VNrJRj9f^lMK&@Y zk%yraM;O#ZP-t+5i(FWkrP}NQ?rJIO?4a)yyepTb@nosPX&d-gviu^Zmy!e5VVa5cFesOBks4(=*5#CG#3c=?w~N+s#Y}DFwd_*H^(O2mf_CE4LacHzo+Bj7C@y0v z^;DWZGA%AE?7)609BdNEU#!)IX43S_RrKX)lI6I(u+lqSoSf43)eTSGfnM-^dOvk5@@_VGkXnE?@(A;9{?8a-G>BIEyo0 z8wMaXzM2!<1K{Y7$pBrirv9%Y1D)#SJODE9$a8CUFg8sjA5a8INL*A%3?FYTY{ zIw(K*x9V|x1#u!MV=%67h+o;AaRWWhmAllV8TkH&LUyDPgDi$aWuCHbLwqCWl{AL| z<*%1ppM)DbfnN$eHE!ZZ4Kf4^3VUzFO$B=T<2pxk8Z-%ny_wK;83&uR0p}37v)eX*JR>)Iy+lrJu zFZod=T5o66s{bOfY(;L($L)oCzTn+hJ$M|g+b3)$-CbC6OH7!0+`)8NfixvEJnjTj z=FGzK@&^os6oTev4iexM10dhC>)e`;V#0gzb`ekvq_8YDr{eB zh8yLr#NC9}a;54Ny@7E8*bx+f2?7K6}=~Lv3(OkTj z6Er+7vk~`pZb5?{d)xS$?;~h?@)^3Pc;>!a^L68X6*MO=4SjfjLuzj+UK;TLV;Vxj zp@<~H#T^ecn5rsFyMVJ|%u4d0^X0F{gN>;vY#I*h4gjFj9`e7+n8ZV?GMX8gjbK`v$iKa=_Fx(OB4k~D?C2&v9%b{380<~?z z%R<Cjf60EY7P9Xm|^YcDREzFJe%Yq9#3aN zyV}-XiqMnQD&8skmU)F%IM_rriJ_i4$u20~EEQ-9c(Rdvdp(|_r}Awx@zhOev+*=N?Q}^+ zO!c&Tzk+lIIBOLlf}xb-4{JT$sbZEqYM}ERrwH>{4pX@);u$8>+?DbmE$*3yBn8*Q z?m(XdXhs4$m5is!g6tzgPFiRTJfOeUnR@Fs%oc_va_ zhojE?K-TW_tFmf@&nT+=1%e#ptFlJ%LIKAne|UKw*OyxHqN=pyh%Os*FE*A^ICX?k zyu^^2i7Y`eONaVPjpkt7C|>5L6r>qI43{!ury0(rNKr|Hc)1W8zX&z_C@^6oPB)#@ z?)K`tI|#DQe5=qYu6Ttrs6rhAL$qi3N<%8OIbl<66t8mr4P+9pHj*p$q}(=$*BDew zILoi)UW?SS(7n0Jq`pIZr;s^&de+Y5&@I-4R{Ylqw+vy*!Uk*0l0vVqitVt=JU|OW zh@4?mV}(z*h6}$PZ!o4>t)4@a39oO*8!N(garn7xkBs9@!p!`L$JvEZtOzj|*~v4C zH#=i;(P%~|(BmytS&Nt?bI7xJt1!DxJ9V5u2lc!ogEa^~o~)gwc;8%g-saqz&``GG zOjBx9akwx!-)=IE)_IgY1VB)*y6bJw1Fp8<18m=#5mfc*>tv$ za`y$n{4PDtSq7(WhO+u@XW~~1M!ZLli#f5hd=a-0?=^+GBu0Y)2QT7%f~=;a+*_Q- zIRX#tzan%fo9xjI@a5GJ3EksRGxF&Fa=gDP%;neO1BTQi+J#&t#Cc7gQRL1PZM-Aot&0`u?(>`W6l^zLBS}Y%IBR=i~ zHQ1gmO^;6)RMoUrr4cU6`IDOqKdp`en!(7Qs>;xA-zaVBPa70(4P+j{nBLoH$7htZ zuiTv1UTI)>d{*GzoT`f)^=Yjun4hZ(=V;7eY4zQB$v-d9{w5uBkjbxTlrIS0lb)R> z91>qNS+0~_#BOgx;r^0K)hMC%7ByK=Up6LAp44Z$#X@|=`PG?(ll!0)Qu(V@=3dbv z(v)HUnz0-mdUi~CUpJ(MV6b`?Te{y6;Khh@=EEk_{F|mpd53ckLH4aJ$RxgPMC^x- zUPp14&*D2qReGEAWkr0~fJPEWkajyzT)wxZv?9JQ#IEEZ=xroztsOrQT2@W236`iI z3TmZ2ig1ACCNq+eKXQrcxuqHH%M(WMV`2L|e=~O@e&XCB>(tru=;HPMsk3HdkwCM9 z_?ZwdA~SV1sbzgM;^!6JSTcJQzc8ZSB|@O7Ewx`7%6XRI;`sZ1WlTLq@LDjjXBmfI zSCrb#hV2Q_tS|?g)3V=GxpBTT!|)>4pbkLagtxN@z={mQJ_dJHINa zV8~gR`VWHaz+SA+FU-m;ScpHmNcRo?8O@_VIdd+YzNVm@Eu;9elSDqO59V}EA^zeb z)Ri-%7PBkykJRwzf+f_rfk$t!U_wv-%Z2*lBq4>*{fl3LK0$n7&NRY8)wW$9|a{ zsI3i!W3)?+Np93banb4xYCUJ}8032FGOX?=LR_A_xR{H|9%W84U(#Yf%9P@VVxe8I z)p2nXifdvbHz2e85~kAp=++cbLEd3p(x{|l2G}lf7S$8+QWd?b)uSAE@6twd;YyE` z>@OC?%Lw=G6?RPx+j5zxVwS1PD&Z2TmvId*x`CofgI$R!Q~C9H_?$<08lUD z3Ia=?vnM&=6%DFo9C0G@R; zM{!kQ`xfqier0ut^{P|egoCubnzN|q*c6`~(F)M3`yJ|asWC+A?iS(*HM}EOp&SLd zhN(1UsAZ!EztgXS^H8~W3R9Ssq^dIT5gyy*EA(FiK7s6 zaf~0uDy&m7?c-X8R6F)KrEecA#LK2E)now1=h`Mz<2azqS&)3eb&MvbG5GXUZbDqQ zB1S`^^Vp}F#Px)3&lSM0ug7tM$^(ii438IJNy;4?#G1wRU6w|Ycx;Z_AenDqOtmJc zg)p|bp%GQDgRrPKar@NcMnd=HMo?{B6y3pjXVi_INeYi*{@V&nrsodEO_bo8W~bBs zn;P0myAn5ZA>wqOabb|Ps^0v67rq>~P{N+vj@bOxLvc$dXtiGA6wN4ZCE!5n?kpkQ z8KhOYO>k=$uI6fOp}}~><2FV$Ne81D0__T>ZoP#Vx~)>IO${&`i3NiDZzs?P2{xi- z(~!G;RXW}e&Mjt~#y+9mV%LpE8Q}!y-ND4_nC_<2GRtv-6U1!@*a1=CH zXB>4-7^jmoMyO<((2`d76Yu4q)IEh~V0q==%7 zf2WIEPFK$@OxlVXnMB)|7$!~QT%)fpSsfN4~~Mi&7o4HH+YVYOlb zEr+}J;#pMz;%THS@oAHIwkb7=sAsHn0eGrGjZ8V)ai~`Lm)L<>t`qsd6nd^nHHCE6 zo5hg%0M9&6xCKTJ1x``<-FUuKcyP zJ7(+*V`>BwrnVqI!6@F~%xXk$-I8Uz-e_X27HY&W-Xy^NhcuKN!?|iA4OML>eL`CX=kSSin0Se1P#brR-W)Qi!JE zmbxH-(;1{Q5%W|?^I?VH1#fqPI-p(PrkCpR4q>H2PV|H>K$3a%PQzQvSc$V-h8jbM zF}R~R+mQMYCM0%(b+d?f8C7crTU}`SbuRNK4r3$I+Fd9__UBczVd~-5y2_I+rGFA zup%Zp&-GbnRQZa!JIGmKoNAvl>RE+aLGYs(<@18g5jDgBr11qods4ZQF@U1ch%Y*~ zn%C^Mk=KMV$CnK1MX7v>&sXBh6`BVs@Nh6Ozfz&>J;D4LkMvccrOu^umZfO_nqUt% zZ`aWuT}|+Hp{17*xqw45zTy0uolC{M*wRrmjrgXC)m-!xDm==)oNpP{BxGr@%=osj z_0v(Do5$>C1*dPm;{-{09DRAjBSp%0jjQZ2E)mTcO`GpkFd7IR2UrEU^?k#sLgcPR z{J@BIlY@O6EYrr2x@c8v#1EaA+#v=QyS3v-Mm6!r6P#U3!}zfwHHAy%xbBLf|B0}U zJuSh}^FjR7R2sO9GoZ<%_*s>ea3_Vk9E_g}a~Mf^z_c5`FkM!C7Dr(>eksJ~xTPe+ z9`P#^slwbqqhNs=3&O9Bs&>pwg}$`o|BPiLkV+g7kKYK|yoEKB&O9dfZ%wROV~>0de@$!JroZ5>VRL>>%+X?a};OXe%pQ zj^iZ$srL64XUo3 zn#|Z})UV4JRgh3AnmeB^X7J*-pfmUI4UoAr+3GTd3Yc$FT?>39j+rR;tB#B zO^bYnQ51aJWaeH`3ADSd*ZrZkv)hbY-U~QUkpPra2BjiE5KQeHEoT z$WX5EM!9b-R=cV*svGGU`4`!Wx|-?52E3k=cDlhs%lg%oU|kNQzSU(jKK2^PnbMMqHi>IGt5ybfoZCb= zA;MvyV@xW}#+A<26`gu5WBhLGwPZ0K>&z;zRVU1YLs8c@l+2^RU=*WnT*n#J>eX{( zVIkwO_#1IuC!|=G5XSww1>wkimDvG%l1&mg$17pq=-9L|*7c8iTwidhVJ&d1f*ZBj zce{bmoxVeKl~5VFthDUGC%gG6XR<3dQFHF6?jZmFW7hFK+;OY%}l92D}x;4=7#k8sJDufwK;hUVPz;5yC^i* z>hLX1t2wJ%dDwGV?yxs>E5TmPTxv6JZORQ+%ad$6<9M6@oj~-c6ZqxMsL`N^+ZoWT zXmbQG4J=KhcHCYm{wx3}+s--~5c^D@y@2f3o%xfvgFvf}I86v8*8O^Mf=RO@tMRDE z9jhYHT3c3c?<8NR3U@Y(A$ zVs$^C!A2prWk<$`vNv(%EJi-a;q)l}^6pwTf>R~*iYp3CvZp5QZ ztRAk_PpZeG^=!dk<1W<2IMD>jg6u`Wm5j~-gW_X_9cbq+MF=O>oT7RzjVJ8K#7tFy z)NK|Hm~6HJ*Xx2|XfQ`NBZ;$AwwtRm8{k zV_Oy4h=z&z-Nr$z;UE@Gr~%_9AKh%X98F;seL6=qWRDsqXbH7>uj_*G*rfPuSH!pp z7|x94=?L9k?i`l8hT<9Y*shXxr#SN^Kd7eB6F8G=u{NRVTtjLy@i4+3*XzLuL#k_& zy9<_QAR7p?SlFoRD276P7u(`!o)9Q%mOHN_7m*B=yL~Zs0sNxmw9t7^G9XSIXfpF= z*6)*@QD~3YtNMK4T!@L#U8!3t{4!k_OQzI#>aNaZU6v&c;4#8W4Hnu3ChTKPsy2=} zbuK5$;{LU5$}4G~F4&jw zY!X$CI7PUdy9uwyGlV-%apQ_9G0!tyj^?F0d`5F6p5;O|YR8#*A?C4lvXwa1WMUu1 z?nq&Z=a^8wJMng5)>AmRjejKGVMLv& zSjDuK-zjW&DnBp+n}G{*-7sZ&%~?(p6IUWD&K9~iWjO30r;Ow3k-G0v(mt)(+(W`O zRq^hsgaXGBN__*e_ZU?@a1%3rEF2p3+#)L^ZT;`J%d`>wfUT$t+{zS414C} z_`FbC607IPa2>@LOz4@yA_{|i(U3GMnbCZFjAIO``F+VLnr{Q{89>b|&(VC@36jGE z4>x7)c5v6#W#fb5MUxirJ6<~ zk9UP|#Jc>z+%IaJU%vK$voF3~5vE~|xt{Lg?-={#N!X0R5f}MKdK5v8naFzGJotsH9HUpgoFwF zi9p{P!1Wv2TO7tug_V0GDNX9}GnXfC3;w^9Z-kBUdW6Z(ou-jkV5dualc2vanjJta zVvjAy_@ywfejNJL&F07gI_6ix?Rn*pa-DNQEAeaR*RXE{5+Qf~&#*YQP3%hi#zpEi z`9(2p9{twXd9zO9ch2e|VMC}`9LDdBsq5Os$=rj{jz1VuH*t_rJ{24*e-!AUTbC@< z`%hJgGGS7|__Hzf2AVW``v*CR5q}Z7BZYB!Zm0AvB98v5w0)`N!)t6o)cFEL{LSU4 z5gfTF9J<$zzYBISCx?eua+)w3{$Xmpb)$ZV;-5yv=_%+M@h>6!a!X2sIO5LHf14rnS=Uf-tRnA15rj%crd$~nl#07--0tnp~M$qPVA-k;Z zOHmoc1%)_9Cuusd!!*gC`B@mf@EY? zd6rQ2e90rU5*Ksvx&4Mmb+m+jSURPTa+11^tZ7PObzZzeX<05Lbj~Fz6e*#5lEl@Q z6k2*GRMC1|N=V1Y?)WkcSFfMN0sn(>Y3J97#_HKi9d>BqGR7D{2}YLX%No!SYw_h! zR+?Ow6FiekoAls%PdfhP1$yORFN=>fjN%Gac`S`^;1vyI15woVl^-I=u4Ggljz^j8 zvDvt?pq;sDt0tqkifQ=WR^y%x-gi|YmIpH6(JsbuHB)KgOQ6Qp_3XH>1>H%<4cxI? znB~hHBj;oEYp2CEoL7zSaa^#7Jpzu&Iw9Im_KX+7!Qzpd)h-H~^^- zSVr)=E`VPY?2|P9*Ar64RmiERb(i(D5yw@ey=d_HfH>ZWr-+Wep+Twg*B5TtgZ_*j zqp2P@5V~=#j=q1=theKaE<^KE7oTa9sTDUesMZhJ)L9{j?#71nF2vLN-Lpm9M7X1C zhzcN9(J_NJHF?S(_PNq6dNX4w&ahB9;!ADg=0d$u&Q%|}mE{&LQg6>2X;5iy>8u*- z=IRF)I&mujZp?=MP>)*+wROfPOy^jOSx9eVQWcl7fT6sti_6-ff^nve&{ARLo~`u# zGMeS~Ce@p95D^i#&wyI7@n$Xs6}l&9EzT1nV<;=PRPeYy)lHe}q}nC3H6a1{6P zv$!iSiO=fa*Pw)y{DuTEr@nAM!|Ilk8XP}DbG^Ty(y*TB;6&-Ddq7pJt155&0|i>1 z)jNIkkYPMXkT-sE;~G`Safc69g10^~Qq_Ikc!&%2@>s_=qgw^r@leD3;=!8Lp1&46 z7D6@LdK~lpVVkL7pI$t?LTZy#^dBL_PA=A|^?lVxRwzMp8J!5TLiAlKzu3g>5k^E9N%rBrS|1cR zI1s!$^}uLN=5?uXy&XfNVxjp42V;Ba-y;{mZ|>f*12z`2HwA#cPqB8c1eF{LCz(`J z65XD~0Z!%i(KuQ7?py&mW=#nb0lRXei%lnKz8zVy@`ko!$<(TGS~)qhhw+#S&WKS> z132=rhWXt#;?qXh#*Y)?p5>?lo?Of{o*2?wJE(8&vv@bqBAc~)zjWo z1eQ3BnY+>P9DxgFZGu4ca8qw=RaMWcdsBCggh+AD-whM!eF6pNF&_uW|wWrXWO@dbJTRU7W^h^hO-Ot;cJe-y)NHFJ^J$qmZq{vJ&>C zAR@|$dmtw9T0{KeG{CNH$LoanRLK$%k7G{n>w2%(3-)5g2cNiA=M=3g_%DSY$T`R& zG;HAi%KKBbp%8CSqSq35aFNfWPvVVL1~r!oZ+tFpIo{+9+4THogKmn86=zXHQ?!=h zM)77-Yvl79t9VP5ov8C+esqE$QKRA;om)&>ohg6NrRrUrTbRf}-u8O$Bv|NEyYvF=7|i% zc%M;Oer(Jl)aBf=vT}}-)UjDcyuT_TzYBJ5T^A^Qz-Y=8+T@rxT$+YeE3RUZao&z| zjY<7sV{WB`)8d1I*40@#AGx6=K4d!auAL4!hOp&_s{-=6FDBtfT!6}COOXo*j*C@6 zm&nG$vowp?P(cn;GWt*0K-k?~3sJzh}td8RG1)(RTl-05D^ud48 zl&YS-W<$UwE@8@uFF8TojroxduP+PoF{v7^y@I_8>8f8br5i#G;J9{tRiNXtm~MRD zCGW!fnn_cpD~viLDwpHyo8WG3mhZNbG>mUl@K_E`MfI^Je^aoHECT3}>+vliRz~@D zVLFJY*SDQr!^y=kNOD^B9bvu;4=(|+60;mk{jLjCZZ-`_% z>iAcq>a#_<-1k-eOo79<V4Yl!7hSYPMUXrP>5|=inHMCW5Fd!aVA@ zbWwRGab-gqx2TssdmTTzipvxeuAR%J)PhFPEak7nIks2k9G;-$WrW}itYVh#V*G+&rf?W zj&YIvqKe4(*ooi zUR_GOQQTaZH7`O0d)`{Vg+NcKjTq>-<@s{+d3a$*rMT>k@E{E1*3P==9ef{Nn9Y*Z zf7V0IZ!6s24^Pb$vB3qzx^wJyO7UDNT`%;I6;r@qV?id;*O@#@X|f{TurK?cM@z-h`OtQ0KSeS>^e3~Ww4-t!1i}>0jf{8 z)PB-kcUR-N7?yXdGpp}rlwb6F+*^#?-Kzqs8v`(U?qOVgC@Urh^q$64+j4nNMetsN z{bdY%N-J}@rRPJx_pVA|Qm=-`A)ZZK+nsXt-a6WaKw~P$6Ek z@TTFo=c`DDQq18L;#8pQ*9Qu9Un42m2Sh212UWQd=c!mdSkUfV9AJ5U&c*KPhZtAm z@qckA4?XZuV`{vDFDMdwp1VQ~^NU1Rsf~D;pc7KaB4W8~KpupLn@&aQlu3T7(({W=)+IG1%v!@%pCoV_A~@}?}~j4?cr-Sti+;l+f3b= zrEx~pOJQuf2(i(8gDcZ z$L_EemW(B*b0Z$=l3W4t7{T77)rfNe+ha}0@3w_D3qO|A;|!grF^A&u6`?!hP&}a` zMC9cqboBlcD-@rSE4dO+5^4ccE58rFeoEoVPV)P57e#8;Q;a5aX3T+v!c$$gK<-!} z3>vK_o+j`-X3&Tu7*msGKT~K!lJ@BphOyyofm4KSPnDHRLRj@Mpr0XZZ!SlQ7{ye3 zrgO?rCVsPlON=u%gxAKioWKWAwvFQ1dgx?Xy^MuJlALM+Rar(XU)i9Ko+HpI5b7_w zAosE4oOrGhyo|C(p^J}^tIu;08cddY-TOc#`7)9V(LB-RZ%oV=2st@r;Vc`|nk#7Q z&3K^;P#tvQ8SfMEpD!}VZz;_h@nS>r5|NvCOUx)-_)xq=NoG;YK0b~lsFObZu4eCs zr{ZNUM)ft9$=gtm(*)ZD?#`&b^>)16q&do{+&$*%Bu+P|U><^l5XX5{8S{)mdxg?0 zOi~%iK7un|DSYq7`6t~fATD^7i%L1I-z@{MAm8l-yL%a=hk$;)i0{xVV(_ zQtPjMtx=5ub4&reuM@J@#tQb*a;m~q{Pm`k^u<0!Svy03%}Z^361@EelVtbHZ0F3L zV>*cnRg$#{O9+I{n_Pf+l3;D_M_AW~R$PvJ^s{`OUQb?ed2v1PM8CxZ8lXC!JY7(x z_4}=cH5ltsj4Mnx6}A*_)6;e>nA>Uvr^WD!oGHjGyPj(jZx>hw8*(HYw$0Ntqj-lB ztdX&lu>NW<^qm4r=EB0~hNL*lrDhwNakh)gCxSg)Mm}}p-FouZJ!-3Gqo$4GJtp86 z-w*ddqY1uOkhN`2+S6cSJWZuu7|_@Ft#VqOV>m;}#Q%A|sI-3G?=*f1dRHl}_<$hm zDfroaC14yfo!OeW8GIJFMFn~w^u=EJJ3eSSezEWfB0xlcNYHh%hq)M5N7UlOrV{HB zjEh;#0@{v`2;FH$lZ#l$4tlHixzBa6d=XL7cpHLid416j1D1kb4M3<4v+ za>ms8XjP83=5fTzyBHrcxs2|-D)HlbE;l8SHk>p~LG}sf*7VZdX@^f|xz(EbN#kN# zT^)fliXAU?#iyL0`N2&Mgam%t1(#|}y@tTZ@%fAi)xPpdRYZF`J}dY@%3kh6$nRRm z=UfE8L-qKb0Hwh-}xFOsf2)I7c{s z*`Nl9rC+WlMdEy=B1O1s!xB05Rf8!82)l2^*97^=K|vYP8s9OXW^lRC1ih8z@w-)N?Ci3as{#971#%6e z9QSxfzh8keQ@dj=Q$H}MUck+hpX#N-KdeZ{liVZ2=lUao`%=5n=U3-p-AVkoqSStx zju_(&6=?Spd%p7fFz0?+QO>bxlyQFXGlSv_`ixyQSO}o zqAG`JHos8U*W`aG(DH1(!Q5MFO3C>e2W{AI__dK7b4L+1>l~o3N?gxe#BU5r6=}9c zOF2t_TV-F@o#?FJ85PfK9~uS2w@?0F_z}6f5jdvr7A$Kn{=ww@>gY5&P9y#($Xi+) z?SZOCNs|UVf#msx2qx zg_lY)n@w`=9%MfJL-6kOelb8q3Fr`Q&i#KX#dEhiUj#F46#o)tRTYhit*!XC^NVLo zU7s%akF%;7+!V8V&H^8>l7;$T6AR~(G{w1^OSUlzp`b?LvBmlJf}2CT&8O{b&nxvRxdL49^g z);6uUf>ZT@oO>y-_wkxW(?Y*yZk zt5oQ^Xrq-^6}l(GS#(7j2j&GFNLSEC=1E+=LN_w6#BLXP-uL52HGH1;b7sCG)+|%p zw+@&dZCK35x+mt_E5!O|Z-ne?$2EnxGdaIcl#WhW9#erTPyVuYTub2g3>VK0rlk(& zKXbczC5}~^5AYEP%U1)WT3%bAPcJd;8*v>!i^td@=H}x3CJCc+T_yO|OT>wy!dABMs* z_mPzwb|)CjS9IczRYopnl5<4?UUxDorbg{NOx#Y&-8)w>uD~)>p&oZJtk*b_OP_bX zt3XdGv`d{x2MD%<8(TG0OJ)@e8xM3LIj#BBDdyONgn0Um0_%sEt5H0-0`qVz z%L>Qfoj3c`TdlnBp$4Vi$Ar@*Qm7eq5pkxViRFNJf0)318R0c^lX$qEN|&y}atW+? zgmY;c;To@VWv^Jia)i9vqL^?)LmjOC7SgAyO3XKTZ`Nv6oSc0ATp zQiz&)(wNhWBf3L@pi1&LcE>3VhVgh)Yl;t11vPb_U`PUj(d0Akrx0ktF?Ye92smkS zem=>RHg`b~ocfYQJD%(W^_})BV)Yz}n5Vb|)odg1E>-<@Jhdv^HEYGw1bI-1p~hj< zKrNdJGy}j^lKJ&^Jl&Kwg0zgYbHr`TIA50I6s7I4THe*wQyWgFXDG#Hv5vUurk!UB znn|@~hEM|8GH~LvgxWxuCtz-wgZt?;e6|Zn#ihL6OB&dzh9w<36LI}>1UTZ(Ge|7= zTvPG8Eo&t|`}2g^P~f$w^cZ;07gnC1oYl#+7dT^1EIxJ1B0$a;8q_Fy6pDC}AytD( zj)w7K0gI`mA^XVm5&=FDBKD!bQ1e#erOue^11p2hf~i?AGhDrb&{09QyY-55;^n5% z08@8;G>0YbxOp88KV3;~hiDJQD+H8b4}=`1$Z6Kw@k*B=p4$8s>y>y_l_1SFFIY~e z%d4HB`A<7!KA_R#=YEZGiAFrg(mj@iI5-?HF(=+lp{C-O!((fY^y{hu8{Fs=bDWCQ zN3&jUJm+7*`TR5VY!1zhy$nQqgNek9y2d+Ld-%xPx$rrDPNjJPF#$5a-^s= zM~XeGBD4N5iL(WjUf0w-2oW`vhK_9;TsZM=A-mH@Koq`i`OEFa_f*AMYrz+4%8k6- z9$F!MtgF2kagMV|KIu4HVk${0?8hQshbQlpZo&j+Zn@VqMRz zAe~Zgp}&1l&{1W$f>gj|MMQtdgc=&p)U-i;t@4MJW?$uag00!<60gXb^AYD!i(qYu z!BrvdGWTK;hqn;%>0}}9e)emMmPY!c#k*3N%mZ5^V2#e#%+--NyZ8xQuDQrz@;{#Y%j}Sc;hq&0-RttzbTe%~2?= zAKmzzVRZq!nzD^QZ$xsC1M9hq*o-e!WOFnFXWAEythb~SUlLFnaI%%~0uJNLrcz}p z)oc=9F|2p2x4MY08svAqzxeu+uoH4i&F5s}>q5NzBWmdRZwT;~Cjsvbcmy}wHwD`2 z;ChaO%xf(e-!iJkDhdgi{B0rSP^L_#1HNM_3CbFMugezg_xr9BRBL~m2lqsLPvEXx z2RCaU%*gMXT=o_R1DXDc1J3=_A1K9}hvrqd<%gy&mwYTJ1$Jt(|Huja`fCLQ2E>nz zov-MX_=!`rQoxwHvDc1#gv(ErbsM$UZm6B>RO> ze+hsU0ecLOS~{(NsRXyklD#-NekII$8KIVK;v?Z%Erh?W%5INzhKn0|`EpZ*b_(v; z_geECqv~_6-jI4gOMdH;HQ9(&&2Py4&X9UUUf>?byviHVeqWW#XLTBRhuTW~!KnI) z{Toc^RByOkWfXr@k^}v83oOg`Qn7YM{K>FhnqL;gX!^4Y(MzT2tbdCHH|bygU&T-2 zuP%pQ_QJR@T_@Q7Cd~IHp0_qd{9Q@jfPoi@3SB83QvayRv1Ek7|0&Ej#pIpa`u^I# zgjpS9bYH)OjQO`AEpHq>U;krVgFZJ|%8emJ#D9%y2(*|p-U#7sJW{HrKZ^!> z$OQygrlTa<~J$Utv={M)^X{sa2!KC&l9k zJzGz9L&kAoL6)9fzMaC6XY%cIJ{L0+7t!MpS<^Om9CCB0Gim6th!wVI_dYeasI#Vk z;AA5w5OxW%b4!u2v#j~tA1ZK9nq`BSNuBj5=j4}#kLoRR!agqU%=F*3CZT1R`FyI$ zmJZO`e>s;|sj!3Wj8cPL7W&>dApTl_3~c5<0V~7gplR zg7)rLgJVQ=g03IDiZiQE7x*v@g7Kk|&UwG@)*_dFu!$Mbg%BE7g zv0K`wJni4TCff80j>HS|>B_vQ z_?Rl+AaDFZh+a#WS8BX^oB))jeXI%jg%{FZLZDt-pmmV-dlvA%5fmzXcdl?8+EEPc zb&aSlOH7$&hrsRY8PkYzdMXdyVxS*qRO317asL(Uq=7!(u)azrM-)o{Ta)sMWQQ_C zu3weL=g)OkN#^MdgnA{@RmL1P)Zqy!2{$x-8Uh4?;0Vy&87pxk;oGaadN*<9k;{*Y|#alklScp3s@v_A}RNi#J=dv1cCl{wStP37`{LVt%`kd^exQmc7 zjH%plg6^(D_bvRB@l+T0P@R!KcN1Egh9jcyS$lFpzPs?<`}Gl;$^m-#RCHpG|2>?p zy0jWS0vJ$y+Hp^zwmdo3#+QljWk4^?$`x^MA+7=ox^9%aPgN9-LGg#g+I@{mB+H|Y zWwPd*Guv@LC-J*3QlP)Tz`JGB7A7eeA0WgQ;Ce4A7XE?f&%F{45;(X2a(Wx{OZTM^ zz}DhpD;Z72*4cxdO%qG!{&Wd0yQ=Y!3YNNniI8k{K_JPPPov zS~46*(!)%zerobPQ3m_rLLB3y3-1PUkw2oMO5?Q^k2JzBl8n0+#7My_+>8^AOSI?$GJW?W4hXZefly9fTnO;E=Go4eF_6=#yM(c}B#|`h zvNWA^s!SIz*G#B}F{%q#33U@COVEiprRRj$7bBsaqJQ4l2Gt}MoNrS_2?(t)40r7Z z4P()m1`7A9luMR6ZvGT(GcG?|%QDasG~)~NT4(`nJ(ml6=v&hRz>bU4Yq`ft_eSG3 zH=(AeFALP5rt+{_+dI#?dkv0-{uHp3}k6D=15!_sq* z(&HHF*^;h&bSr3~!biNYRsxrbWYE6eoAsiIiZUR0&!` zW&10teNHlXk!~oIjgNBi zc%>Z3g?k2~n=PW2zQ$?y7}a`ORjzu1<#Z}!(c+WJGk#AO(n+m#JQ{;-IdO^PbmKzar8DMdU=L_CruL?OE+o%&||?_P&rvnb7B0V8Rd77F~VMMQ2HIgM+ouL1=#&sQy`pNC9FKJ zs4~wm0d&54IbLZ{P0<0Um5yKZs*0BHaPh5fX)&)hs5;>~%y3!^?AKH!uypzQHws>^ zN?7-RHsZAg)5gaPL&qvZBwptP%Tl&$XzS}MTJGY?+h0Z8jx#D+?Ah)j-2MhZ)@V6R z&gVCnI&Uww@2IPRbkHO1AN#M=aU#NpMJYImlo%>TU+II(umE zs3?76MAtFJJB_N=5?g$|PyzMrI7_(q@s`-zWraBVd}WT}UBax3*9c9*P>=}X_D|C$ z*L6@q*hQsQ?{S&@DzphoHQ>D_-BeXh%Nr9R-Z+Ucw9|78rU>Jk;`Gq_D>9wwKqCPB zfI)Q|_Y~v5ZpERBPX08+2JCjOz!?WDGcWKt%qkS|gQit=Y@5(WA2O!atj`u&^I@a* zGjY|7zTY4lF+QRMFNpZI>1ACNzG`Y!3#SUgtrdp_`Hs^(6TB|!SWebVy{Q(5;-khi z!wB=?+CllcSK?!W{h^GF%HbZZ3dm~@b&=mEjHv^;QI{Qq_@s~nDXMZ`8y^~m^{1-5 z+``l1+Z3(%w6J~ITe`ZJEoI)#cj`^bZ(il4uY4BqS%Z2Hrz^Fxe9nlPlku<$2>!e= zbqM=pn9tk?DyZ}>=clYox7ru0?DQ|Un82rB66Q`;P$RuTkAB$&BrkE#52IYfS1Ll; zw@ggb#IL%DySux){Li!BbNh4>{Nf9i>MPKc;% zsE57vXVLhe<2ly^%)bbAteqesH z0}x{m;{9xIPH&|bRkddl%6S{xBsS5eb~-qOaydFK#!|lG%z%Q_$AAg7!xPwW#XY56uH7%>oOD8FFrsuwCsL)am7@!-D$d6T71c^r${lyfFq6 zMtG6uyn*;H_abPLr~f*iXt(WTC9~qsFPIze+4MKuFfWiw{IqVZ!BsCPcvLoo+NDQ} zUvQz2rt=4!y}PaZ!s6*@t{8aYy@<$$Ki7~eY3S1GqA70=(1A-FpD!lTY5V+CWR~T+ zcnUW%8P>2ak%Ihh(KWr<$2VOvs6AqJN>7&xWKneL%kZ#E2eCI%N>AACV3*^%jBu6_ zDvGg&ox_DX9kho}_{(L5vJ%E~UDj-NjU!U0{jdWsaMBi!-2&=zsno7)sY=MfHLN2; zY?IjSwLe|HFYX*RCRG!Mj|$FzO_^A1(4>+SNi=Sb?r-=>Uomvfu+j2vwL3ZT)s=jZ z`82agBHms(T4=f-fUt{KpgGQu==sHUsFR!@su zbKn$Y#ocvn!3^K7F`0vy_SXrWgVG!p>6Ui3x^8eke-58?Sku0`Ua;9W5Fb<^`qTxz zz2H0OhlTMx}nI~+CjT8W3>>-p80Pip1`48P>#-GST|1jwpg@tG>*<9x=C=` zgOP^`iBs@R#qv2^D>9jQ-ApK_1x9QS;=+~`w10@s!w!=kzIkBViQu_^eX(v4L~~6I zrgckyrgEbdy0ITC6e|y!F!;WoaVx?3W@1c!{-p>NDz|f*F3qX+sowUFTk+ao}N6 zOzYk$+D-|sH|7+(GHmBQdEje@xX2O z0RkBo#aq(3%_hhLQ`iZ2fD3DmY&|HL>p$<4>*dAmKdv%=hb7dWzs7~A&0kHp$Iu7}2g9^FS=FB;a)(D5I~=aqrog*3Au+L_Wi zhQ$CZ25oC^PLl$(I7KW4wpY-9ab=~JMY6JHVc2l{?CVsqJ3g8tz(Va3Jxepr5(T&( z8X~*O$egAFuRz|AAOs&rs@ZMl+Q*Xq`}6H%*TTiTjxj*szpn)G!u2um~WJCOzO!32e(6W+%}B8xmr&Vjyc1a;T&Hy9<$h>+Zbp}1J87X@;6fn|9o2=_&<lcUI8P~WTH{65Re2rYq62PwW(!hSI5ke7YkcnS*y3&(+xyW2wY+UDWSMOn7 zujuJ&VDz8*Kaqp)8@?4GXjFRy!zr+c>BzyyzcL{IlO77W5cE}|on(rC5MiY2)k24d zd5{O>M1OW?D7M+AO5h&*W}=09tv|Ol#yD8Qg2IY>UCOmxsks?N@x4^*DZ07IXfOlv zpPfV}I`+*y^#;*{yQZwx8$;BjkMT=+{LX6?@o*qJ7#`WCY zb+S-64$2YSNb9{JI_;}TTO^!CgsRYFpzg%`LhR>oMqmguZJyHCxx&VK(sI3DBu0t8 z-P8I&UwYPVG?)QpSP#P!dZj)jaLy*lVU}(?W9<)-J=U5%sZ-Bb*Lof3K@T@>TjToh zrm`*PVSPj-5tzCwZmI%4D%yD+doEzv=^6hq;p5sm!?%JjZlT&f9^y7dyEhH$6M`8o ztl{k(qFU#Z!pm)rvZ>dn&R7(G&J^_n4HN}?`HV;ucVwp7fjxs@vv`SoR{V%|5I}*6 zIM5k6K9?Gtk#h*%pZ8}(RwPW0;>i172xeCs=4Db}Oc}?183_co-nM`*1#ad(&I4Yb z8KS}9T=M1A;AbDeFt<9l>A(GeZo*ESZ;eaOpGKB41$Z;5d(iUB@e$`hyvt=VX{8}`hJxek~ z!+sN5JE&X5WG(8;x9DY@)e*#gCmKzs$pJ_5)SMfC-_vvYLc0%N-B*7&L(PNwV=xCE z;ivwTpK`e+6$0du{r@aD~4RCsW-Zxs;w#O)AA>Ze>b3!98BMzKRdBXz+B4a$-k66%&Q2&R|~{{^r`Cl+FGxx2kxwV-qgc~w&@72;gc<`>&7g_ zYx*;@@c;@I^vTSKYX#&#BkJLVWSB6^Yp1BogX>OP31F5}>UG3tLpp4(FeFvDu4wun z-ChgeAq%};4>RxSniyfPpSoMOVJTJ9B!gn~3GE_qM_DaHpFJq@VTdL+#4FjK#WH$~vF$$H7XC381!w)~|)E!wF*S>*H%r-9oD9@`kv$^g%dg}QADs|{nTfy!aU7^bt)dfl$Cx^p%f zvccRwXnZbrQjG!fKae|>i|p4~N-*!j3d}Ktt?kyTFj;9(j?WFD_)Xil4IQASkaj=LY+NBTd z0iNW!=HzL*EtDPTUKCyx2k|q{Mf)WUiMnFXg(ySEl(``OT_s&F3rpWe`cXUkzYJ8?~?FcBRWonI5 zqhoEVMVOLu_0CN+E42`sraT!NrO@zbop87qN*ClDXM}d}ToOyL0gOS*{(58!JuoK4 zgTEf0Uvsr9r8hTlS`Q6qldxK`6kDYw8nrd?2}UTjCJ|R^#V5({kWUu#OMC483_|<@S_@&1s%A{u8llQft9n7|4&T zW>-FDlK&A816kDLq-~}T7wiO*>C$wet-?o~q1I_VGFW3uGp!GZ{WUwhbPJo~!IMH{rznBXn^Quq`IE(?S~8g6I{7`G5>op#Hz`M3>#1Tf z6q5lK(X^f>5}q)Czz9>z;EuuOaZ$;fxJ;Ejy*bMnYeY8-<`JXa{XnP0<0!WCLCb|1d?gur$y z5(R-|7`hh+3uljOJuiGokM2L2m*v>^(!M;tHbW8;t=7wWtocyMD_$PVk5^aBE>f=u z2+X!l0uNBJq#Wv!fDG+CTfk~|g*!nhw1+izgPll82%kZZCVmWQVmDO;`Cr=Hft3dwG z<@J#HDg1=NGUe;Tp%UECchyG%`Vu%N2Q1DBA3Xy$t&at3Up8h}L>m5hUmsGoYCDc_ zpAg*A&VilH*GO5P45bgOHIj|gMmc*T@`<0-!mfZ*kJG{l~s{v)`1TEG{HAgJ?(%16H{^y8*#(}Bu zb+PDp)gaPik=VZx+Q!4!NJ*L|3;d>N%vD%+V5>CR_I)eVwq&bL0qtA9y{U?UQDvJ{ z@SPs*(mQyh`)*$WiLA(W?78iG0o#1x{@@PQY<<7S2p}0_*$;ws2S09w;U)t=8P*Sd z5GNAU=3Z|4kw~YL#^=7pC;0Ihs+`tO&VWtor#&VsRS!}>6U*Fq=!A($I<=n*7QTpMb{<w=hzeO@y1hr~61pXrwO&-B8 zcdub@>151`ZyEQ7>20gV^c0F@&S0(2?!z)V^m}XE|Ll95jRY zg|Ml4{9d_}E*56B%P~b!xV2pxID3d?Yg@-VP+=S*kXWp9Bj&cjYgr)Y=qrsoDYDc# z#bSV$QM9#451%WfehAqOojr&4g;$6jGV&7Uxq~_PR)@QqEJNoW7St7w_sHUexGcju zPr$}{F^Axx8Gt@UPHzqkjWe7turJ25d5FdN{31C(=v$IZ6BiInmINdsnn_Bw`?*&Eu8fQXLVZHagw5kBT#( z4Oti(Q;)RmDcW&P@@K(BxB)$_%ZFBL`Eb3e?)TpKMnMnH||KsK_#|D3${v zN;@xBGBd6uI3Ja%Gq8IRx7B^HrQH-}2nl(I%hz_UTg-H=SIHYr-@;K{H4mIx@ERhr zjrSV-1gmTM|C~H zj$5@Ca93R@;Zl~&NI++yD`C~&AQklAHmw`>bvB6}t>ma~lsfs3&`A847~fXv#vv9T z9&d2yrj}i*n*_I)cMb>3N26hyusCiioJ_fJOS+Ep*3EiSE(@B~klkl{Z4sXF`sN#3 zHy65MQ(7(Iyc%yR&NH`24Ss~N%M~ncDU_M)H9M9&Ai|JAXn*OuX&i#r?(lbL_3 zzK+N^#O&;$aKv%lriVf`s@n?1$s|;xx}8v`D`ir0lqT}r?FAEAW$Pk4zWb6+wKVn) zK1x&zS{Nr_JU3IZ3me7iltH+YNRCw-bbIU0LH&Ao9R|N?-9;z{zYW4Yb=MF&86Chj z{&y3bZ%nF_O%9w`cMoA>l7zb4;hG|$QD2;kJ_voZ{GL7K#*6mVy?RvFS|YtGmEAk2 z?VRiwAQ(>TK7nScnpsdk4z%m#F!Ql;lF5*}C-?JrEV7tRiQ&UUY0=8+{-JEU0unXN zdO&Dh7VUx{%9|baz>u}6p3_=AD0J;Z;V#NA$Yu`?p{#=hJBHis;^sXh53(z9ta13z zHXaw)ek~kX+c{07d| zhCVnvigz8&2DMEjBI|+~NLfN6k@E1bWo6N+J5tUHjo-t}v|W6|C=dLGj(BhF495Q` zt)fdA)k5%&gL*}wx#cYewlinPdt|t!pdE+usDV~wS+Em8%j9#Y*2W=p6d42Vyw@&~ zN48_yjJeS#4?SarWPI-{?3%x?Rzhfd5JN4-S z?LN$+s}ua`o)O4T-6%b%XNqoVu6;=HEIHJmJt6mB4Ibf2e0E=B=HjIOS11yjLy55R zoSukNJK-l+>$xHFwJLtOt*BJrhJmUqSZfld%@OX_w(+flE z7kZqO1H4G+uqG;LS!XD_pxP(&wB~|21t)M8D_u+VdZC?-iLbRYxP$$M&~21l ztT&28WIzOErmWW95ZN-eHo5YidXrer-wX86pxzuJx%bX7_cq|Bib|7uix2V*I~d9} zBA32ZC<;8y%k8i{s<-tutU$QREHNy>q)y5Mm!CTix77nU4l7=jYU#w^(S!R>_F5vt zQ}67-7XHf6AoZ?*ZQkbqXIH#iwCmUdZ=P=_viF42u|-GQE`Xoh*9gP#rXpvr_Xf0w z=11e>N8YJ@U$D;TOQ`2{)DKNvq&jz>Pjd`(jyG2-c0sq{Q~JsX89ZJyDERw>DU7gT z)88aqU|Bc)$_Mhy-r&mO{*zaX=k>vWO&mZRVP&*Q@FC&w1riGCWw>7?W-9Mq3~Qb! zO{WUR(hA>ZRn`HKooy>1LMxVzmUR5Xq7nK?mAl;O(mxW^URyypgaOw_dk9&~<5+#H zFRONAfL@ZZ{&>*VZrHR_!_0@D2<$h`*5jl;8Ib>Q@15L^As(VL{ZoA%9+$d9Z+$wj z!;S(HtvFvgs?YSqoc2)gv>mZN8`y=x;nNM|b3ype;x|%+!W85DdGRP4v39}~)>oQ1 z^9Avo{%1`d)EE0oWVubN=Mm;h;yI7QG1Kj4*Ue`11_M0i}oF305&rE$w z?AY!$*a;(6*+xrKz<+giCcJSeP3q^tU6b%2rykBroonnbd=`@x9kPpzMj>`yw+~of zGNM-fDsa{=>#rRKejU&e%2}xSSpKF*n-4XOpmD$LQ5InHa6>EbJJDS0!KA)@VmRU7 zi{+c)jL{PRA*FqNCp|Ht{}@8Smb4K*Gpav@vUTf_JDZdGa}ZmZ_0Z(AzXWY-pYxRq z=dXdB4J`!L0;;CJiN%6Mr(u-PindgL52<6owm^iSj$_t8@<2oO@R<6izYZftqs?iN zO(ylPP_*gVgbwDyzs2G&&ptvZ8GK<_^8X2G$767WKo5tZ=^$PUGqdTq5VHG_y(mx{ z9`4HwG0qO_;7y=u6dYz_^X@>4tW zqlq}MXRr;`-MM{o@O^z3)-|pzcZ~5c@iY}mu4HhFJvFz%*vE#u zC&NY5p><>cho@-)xS?v(>hgW@COhkruv$k2ZHuEBq3M__h|M*55b|ALo-6m&5}y|PBWvy|!5a5SUUDR^tM=8-HX=xm4(eGBv)A@I z#^3!%F4xrr4sJrn#>Xty)kU*_I(cWYu92!*cet;%B~3f88Qi6A*hfwWcovW9TK|=N zy{;YHnb>w*B5?K++kcd4&s7WcVdff66hj8>{7=^Lp(m3 z;llqF=8BYh(~z_!ttlSY!WL}zX2ESpEC(UqB79`yhdZ6S)Npg5D54c#yv#ZC5x+$$ zaopFicH$tyh4)(qZ1WDgAt&)c9UIIEo~RzXbvnFNUwp+NcDud4b!)-n?rXz(k!w`9 z3GCN%)1HI-YTZ^S>$Iojrg|uC6x!5oKfk@;;W=m#)J3%J5X?8BDCN{jauvAD9aG-+ zb5KB0OQY@-(xwc6#SpBsFH1<{eO`oDp16zfmUiSNV$Dn;$VlBa^mc%bl%)A^{O%Ui zkHUWkw>>fw35CZI?{K07Z=>$fQzG`FQ(muo2D16PXXgUyyqEBC_tgU(O-&2nt%clr zCUtKgCL|^^2xXr-xcl@KdDBZ*f44@zf-ZATGPb1juKV?fhALimf05(bV%;%b!2YF+ zRy`neo`Tpay>`0UspWxr%Kw8_xD8Fl0l14(^+Bnu^#+C`LK4-3LuJ zA+2{2P-~cHwuqC@fjv==Z4-?<8A>0o0&WkTvyh;Y8aL?a9RVFOZxL}*hf!FnQDE07 z>fP=-FB|pFzK(8!vQI7qw3h(YlhmTnA#K*-jAHhjuA`PV(GGTTX=Pf=ftvCpGoUHc z5pBnL;+L6uoIZ4N>$Mu#9-nOGo;0q)LHOS^k@&*mJBwAQcK4)9!{Z6dTMrZI?8=-eW~8vckoq!`3F^?% zR?al^ZaBNg+ZSui--pDXtB4Xuu9S{P{7)*d17h!M)9B%%*?IT?o$;6I5uxP&X_|{y zXdbxu+EEW%vGF2VRV?jC<)IEO#29Zw#Z@r$vrXR2%fw&4j%ZFz|JtP>9%AInrryVr{>A&nzN^#CLU3P zq=)LKX(qD5(}UZZ{skm}Pc@FHiGu2x{yr)?Q^IHxf3lvHlJ>?NY&aLmpWRo7@5%o6 zt?IFj<7i-eQ}?qLR^&Nihc`Ki&eZn^<>hnx@_lor^}Nv9N;Y9To+Uwe&rsV0qeODo z8+rT%f=7fovx%>eqYi(!C@boPDbX5&c(lUt_4N~H`*2{yp zldpR2ESjA5UeTA)hJZ7YAqm#&|2E;w={Q8Atl87!{pA@3{FOb9dJHd#rp27ptAg`i z9W2K6_JA4n>OAoKwh^DstUV_NlDW(c*sHlcFOt*`tm_eJ?1%MQv4ma0hmJ|~I)Nja zfWQQ2owH!qQ^jaxG@L2=)UJJ=gS?0Dm1O3rKjBi)Z2R^uBdYcU#xd*B4R%#xym~?Au_4?dRLG(`PuxL z6>Xz?cVL^RZW`GRnfN_|*(BRNqEVe3GUu9sFbL2VZRmUZ;xn;gRPPIDo5}df`lx*( zb5VG=4_mBLLd1WqFSBHuOzZderQ6E~^?@L#ZTi0Ypzw^}ujvRns1JqI&zZG{Z|Xs| zHWu;DrD>g-(i@H3QwKz%VkQV7^O3Fg!(x$F#@tgvnSeO@kxliX&fmY++dCgkX~zzK zPd494eJpfsPBo(r+vLZ^qV$_pxVJtL!nU6|Sn5@{K_9eDlJTSy4bh(l_7i6{tk