mirror of
https://github.com/storytold/realtime-voice-conversion.git
synced 2026-10-09 00:09:45 +00:00
realtime-voice-conversion based on last year's work
This is based on the work I did in (July - October)ish 2019 to get real time voice conversion working. I've removed it from the `voder` library and made it into an isolable program. I'll work on extending the capabilities.
This commit is contained in:
Brandon Thomas
+23
@@ -0,0 +1,23 @@
|
||||
*~
|
||||
__pycache__/
|
||||
node_modules/
|
||||
target/
|
||||
|
||||
# IntelliJ
|
||||
.idea/
|
||||
voder.iml
|
||||
realtime-voce-conversion.iml
|
||||
|
||||
# Project specific ignores
|
||||
cycle_gan/python
|
||||
tacotron_melgan/python
|
||||
voice_converter_sidecar/cycle_gan/python
|
||||
|
||||
# Secrets
|
||||
secrets.yaml
|
||||
|
||||
# Models and data
|
||||
*.jit
|
||||
*.pt
|
||||
*.torchjit
|
||||
*.wav
|
||||
Generated
+897
@@ -0,0 +1,897 @@
|
||||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
[[package]]
|
||||
name = "addr2line"
|
||||
version = "0.13.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072"
|
||||
dependencies = [
|
||||
"gimli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "adler"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "0.7.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alsa-sys"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0edcbbf9ef68f15ae1b620f722180b82a98b6f0628d30baa6b8d2a5abc87d58"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pkg-config",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ansi_term"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293"
|
||||
dependencies = [
|
||||
"addr2line",
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"miniz_oxide",
|
||||
"object",
|
||||
"rustc-demangle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.51.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ebd71393f1ec0509b553aa012b9b58e81dadbdff7130bd3b8cba576e69b32f75"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"cfg-if",
|
||||
"clang-sys",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"lazy_static",
|
||||
"log",
|
||||
"peeking_take_while",
|
||||
"proc-macro2 1.0.21",
|
||||
"quote 1.0.7",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
"which 3.1.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "99de13bb6361e01e493b3db7928085dcc474b7ba4f5481818e53a89d76b8393f"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"cexpr",
|
||||
"cfg-if",
|
||||
"clang-sys",
|
||||
"lazy_static",
|
||||
"lazycell",
|
||||
"peeking_take_while",
|
||||
"proc-macro2 1.0.21",
|
||||
"quote 1.0.7",
|
||||
"regex",
|
||||
"rustc-hash",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||
|
||||
[[package]]
|
||||
name = "byteorder"
|
||||
version = "1.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"iovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cc"
|
||||
version = "1.0.59"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381"
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.3.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d"
|
||||
dependencies = [
|
||||
"nom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822"
|
||||
|
||||
[[package]]
|
||||
name = "clang-sys"
|
||||
version = "0.28.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853"
|
||||
dependencies = [
|
||||
"glob",
|
||||
"libc",
|
||||
"libloading",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "2.33.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002"
|
||||
dependencies = [
|
||||
"ansi_term",
|
||||
"atty",
|
||||
"bitflags",
|
||||
"strsim",
|
||||
"textwrap",
|
||||
"unicode-width",
|
||||
"vec_map",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "core-foundation-sys"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b"
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f229761965dad3e9b11081668a6ea00f1def7aa46062321b5ec245b834f6e491"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"coreaudio-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "coreaudio-sys"
|
||||
version = "0.2.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6570ee6e089131e928d5ec9236db9e818aa3cf850f48b0eec6ef700571271d4"
|
||||
dependencies = [
|
||||
"bindgen 0.53.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpal"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ded070249be850b5b59e1e3a44a70b8ae395e0e5c65b487131d8909a8208120"
|
||||
dependencies = [
|
||||
"alsa-sys",
|
||||
"core-foundation-sys",
|
||||
"coreaudio-rs",
|
||||
"failure",
|
||||
"lazy_static",
|
||||
"libc",
|
||||
"num-traits",
|
||||
"stdweb",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f"
|
||||
|
||||
[[package]]
|
||||
name = "env_logger"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"humantime",
|
||||
"log",
|
||||
"regex",
|
||||
"termcolor",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "error-chain"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8"
|
||||
|
||||
[[package]]
|
||||
name = "failure"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86"
|
||||
dependencies = [
|
||||
"backtrace",
|
||||
"failure_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "failure_derive"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.21",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.40",
|
||||
"synstructure",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fixedbitset"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33"
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.1.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gimli"
|
||||
version = "0.22.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724"
|
||||
|
||||
[[package]]
|
||||
name = "glob"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
|
||||
dependencies = [
|
||||
"unicode-segmentation",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "humantime"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f"
|
||||
dependencies = [
|
||||
"quick-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iovec"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484"
|
||||
dependencies = [
|
||||
"either",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
|
||||
|
||||
[[package]]
|
||||
name = "lazycell"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.77"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235"
|
||||
|
||||
[[package]]
|
||||
name = "libloading"
|
||||
version = "0.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "log"
|
||||
version = "0.4.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memchr"
|
||||
version = "2.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400"
|
||||
|
||||
[[package]]
|
||||
name = "metadeps"
|
||||
version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "73b122901b3a675fac8cecf68dcb2f0d3036193bc861d1ac0e1c337f7d5254c2"
|
||||
dependencies = [
|
||||
"error-chain",
|
||||
"pkg-config",
|
||||
"toml",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "miniz_oxide"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722"
|
||||
dependencies = [
|
||||
"adler",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "multimap"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2eb04b9f127583ed176e163fb9ec6f3e793b87e21deedd5734a69386a18a0151"
|
||||
|
||||
[[package]]
|
||||
name = "nom"
|
||||
version = "4.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.20.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5"
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
|
||||
|
||||
[[package]]
|
||||
name = "petgraph"
|
||||
version = "0.4.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f"
|
||||
dependencies = [
|
||||
"fixedbitset",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20"
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "0.4.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759"
|
||||
dependencies = [
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c"
|
||||
dependencies = [
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96d14b1c185652833d24aaad41c5832b0be5616a590227c1fbff57c616754b23"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"prost-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-build"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb788126ea840817128183f8f603dce02cb7aea25c2a0b764359d8e20010702e"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"heck",
|
||||
"itertools",
|
||||
"log",
|
||||
"multimap",
|
||||
"petgraph",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"tempfile",
|
||||
"which 2.0.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-derive"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e7dc378b94ac374644181a2247cebf59a6ec1c88b49ac77f3a94b86b79d0e11"
|
||||
dependencies = [
|
||||
"failure",
|
||||
"itertools",
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"syn 0.15.44",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prost-types"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1de482a366941c8d56d19b650fac09ca08508f2a696119ee7513ad590c8bac6f"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-error"
|
||||
version = "1.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "0.6.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.21",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.7.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
"rand_hc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.2.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
|
||||
dependencies = [
|
||||
"getrandom",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_hc"
|
||||
version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
|
||||
dependencies = [
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.1.57"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
|
||||
|
||||
[[package]]
|
||||
name = "regex"
|
||||
version = "1.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6"
|
||||
dependencies = [
|
||||
"aho-corasick",
|
||||
"memchr",
|
||||
"regex-syntax",
|
||||
"thread_local",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "regex-syntax"
|
||||
version = "0.6.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8"
|
||||
|
||||
[[package]]
|
||||
name = "remove_dir_all"
|
||||
version = "0.5.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-demangle"
|
||||
version = "0.1.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783"
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2"
|
||||
|
||||
[[package]]
|
||||
name = "stdweb"
|
||||
version = "0.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef5430c8e36b713e13b48a9f709cc21e046723fe44ce34587b73a830203b533e"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "0.15.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5"
|
||||
dependencies = [
|
||||
"proc-macro2 0.4.30",
|
||||
"quote 0.6.13",
|
||||
"unicode-xid 0.1.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "1.0.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.21",
|
||||
"quote 1.0.7",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
version = "0.12.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701"
|
||||
dependencies = [
|
||||
"proc-macro2 1.0.21",
|
||||
"quote 1.0.7",
|
||||
"syn 1.0.40",
|
||||
"unicode-xid 0.2.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"rand",
|
||||
"redox_syscall",
|
||||
"remove_dir_all",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "termcolor"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "textwrap"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060"
|
||||
dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "thread_local"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "736b60249cb25337bc196faa43ee12c705e426f3d55c214d73a4e7be06f92cb4"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-segmentation"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-width"
|
||||
version = "0.1.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-xid"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
|
||||
|
||||
[[package]]
|
||||
name = "vec_map"
|
||||
version = "0.8.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd"
|
||||
|
||||
[[package]]
|
||||
name = "voice_conversion_sidecar"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"cpal",
|
||||
"failure",
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"wavy",
|
||||
"world-sys",
|
||||
"zmq",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasi"
|
||||
version = "0.9.0+wasi-snapshot-preview1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
|
||||
|
||||
[[package]]
|
||||
name = "wavy"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "feea5ba5ef38798f371bc995c1ca542d5162923d1bcbdaaf89916042bb61f006"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "2.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164"
|
||||
dependencies = [
|
||||
"failure",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "which"
|
||||
version = "3.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
|
||||
dependencies = [
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "world-sys"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"bindgen 0.51.1",
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmq"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aad98a7a617d608cd9e1127147f630d24af07c7cd95ba1533246d96cbdd76c66"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"libc",
|
||||
"log",
|
||||
"zmq-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zmq-sys"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d33a2c51dde24d5b451a2ed4b488266df221a5eaee2ee519933dc46b9a9b3648"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"metadeps",
|
||||
]
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
[package]
|
||||
name = "voice_conversion_sidecar"
|
||||
version = "0.1.0"
|
||||
authors = [ "Brandon Thomas <bt@brand.io>",
|
||||
"Brandon Thomas <echelon@gmail.com>" ]
|
||||
[[bin]]
|
||||
name = "voice_converter_sidecar"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
byteorder = "1.3.2" # byte-level operations
|
||||
bytes = "0.4.12" # required by 'prost'
|
||||
cpal = "0.10.0" # hardware audio out
|
||||
failure = "0.1.5" # required by cpal
|
||||
prost = "0.5.*" # protobuf
|
||||
prost-types = "0.5.0" # protobuf
|
||||
wavy = "0.1.2" # real time audio capture and playback
|
||||
# TODO: Move these into the voder project workspace.
|
||||
world-sys = { path = "/home/bt/dev/world-sys" } # AWESOME WORLD VOCODER BINDINGS HAH HA HAH
|
||||
zmq = "0.9.2" # zeromq
|
||||
|
||||
[build-dependencies]
|
||||
prost-build = "0.5.0"
|
||||
@@ -0,0 +1,34 @@
|
||||
Voice Converter Sidecar
|
||||
=======================
|
||||
This project spins up a Rust binary to handle audio input and output, then
|
||||
ships microphone audio over proto/zeromq to a Python sidecar running
|
||||
CycleGAN-VC to convert it into target speech.
|
||||
|
||||
### CycleGAN
|
||||
I've included a pared down copy of CycleGAN for the purposes of model
|
||||
evaluation. It handles all of the audio buffering / sidecar integration
|
||||
pieces. [The original CycleGAN I used is here](https://github.com/leimao/Voice_Converter_CycleGAN).
|
||||
|
||||
#### Note on Training
|
||||
It's possible to train and evaluate at the same time using dual GPUs
|
||||
(at least my dual 1080Ti setup). To run the sidecar on a particular
|
||||
GPU (0-indexded), use:
|
||||
|
||||
```
|
||||
CUDA_VISIBLE_DEVICES=1 ./sidecar.py
|
||||
```
|
||||
|
||||
### Proto Compilation (for Python)
|
||||
Codegen for Rust is built in. Codegen for Python uses,
|
||||
|
||||
```
|
||||
protoc --python_out=cycle_gan protos/audio.proto
|
||||
```
|
||||
|
||||
### Current Results
|
||||
Currently there is 4.39 seconds of delay between speaking and generated output
|
||||
with the sidecar setup on my desktop computer. This is really great and seems
|
||||
promising.
|
||||
|
||||
This gets up to 6.0 seconds later. Drift continues to accrue, but it's a slow
|
||||
build.
|
||||
@@ -0,0 +1,11 @@
|
||||
extern crate prost_build;
|
||||
|
||||
fn main() {
|
||||
//prost_build::compile_protos(&["protos/audio.proto"],
|
||||
// &["protos/"]).unwrap();
|
||||
prost_build::Config::new()
|
||||
.out_dir("src/protos/")
|
||||
.compile_protos(&["protos/audio.proto"],
|
||||
&["protos/"])
|
||||
.unwrap();
|
||||
}
|
||||
Executable
+278
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Starts a demo HTTP server to capture and transform audio
|
||||
as a live demonstration of the trained model.
|
||||
|
||||
Brandon Thomas 2019-07-29 <bt@brand.io> <echelon@gmail.com>
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import falcon
|
||||
import io
|
||||
import librosa
|
||||
import numpy as np
|
||||
import os
|
||||
import scipy
|
||||
import soundfile
|
||||
import tensorflow as tf
|
||||
import subprocess
|
||||
import tempfile
|
||||
|
||||
from falcon_multipart.middleware import MultipartMiddleware
|
||||
from model import CycleGAN
|
||||
from preprocess import *
|
||||
from wsgiref import simple_server
|
||||
|
||||
print("TensorFlow version: {}".format(tf.version.VERSION))
|
||||
|
||||
INDEX_HTML = '''
|
||||
<!doctype html>
|
||||
<html>
|
||||
<body>
|
||||
<h1>Voice Demo</h1>
|
||||
<script src="./script/recorder.js" type="application/javascript"></script>
|
||||
<script type="application/javascript">
|
||||
var audio_context;
|
||||
var recorder;
|
||||
|
||||
// NB: Capture code originates from 'recorder.js' demo.
|
||||
function startUserMedia(stream) {
|
||||
var input = audio_context.createMediaStreamSource(stream);
|
||||
window.recorder = new Recorder(input);
|
||||
}
|
||||
|
||||
function createDownloadLink() {
|
||||
window.recorder && window.recorder.exportWAV(function(blob) {
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.responseType = 'blob';
|
||||
xhr.onload=function(e) {
|
||||
console.log('response received');
|
||||
var objectUrl = window.URL.createObjectURL(this.response);
|
||||
console.log('audio URL', objectUrl);
|
||||
var audio = document.getElementById('audio');
|
||||
audio.src = objectUrl;
|
||||
audio.play();
|
||||
};
|
||||
|
||||
var fd = new FormData();
|
||||
fd.append("audio_data", blob, "audio_file.wav");
|
||||
console.log(blob);
|
||||
console.log('sending...');
|
||||
xhr.open("POST","/upload",true);
|
||||
xhr.send(fd);
|
||||
console.log('sent');
|
||||
});
|
||||
}
|
||||
|
||||
function startRecording(button) {
|
||||
window.recorder && window.recorder.record();
|
||||
document.getElementById('start').disabled = true;
|
||||
document.getElementById('stop').disabled = false;
|
||||
}
|
||||
|
||||
function stopRecording(button) {
|
||||
window.recorder && window.recorder.stop();
|
||||
document.getElementById('start').disabled = false;
|
||||
document.getElementById('stop').disabled = true;
|
||||
|
||||
createDownloadLink();
|
||||
window.recorder.clear();
|
||||
}
|
||||
|
||||
window.onload = function init() {
|
||||
try {
|
||||
audio_context = new AudioContext();
|
||||
} catch (e) {
|
||||
alert('No web audio support in this browser!');
|
||||
}
|
||||
|
||||
var media = navigator.mediaDevices.getUserMedia({ audio: true, video: false });
|
||||
window.media = media;
|
||||
console.log('media', media);
|
||||
media.then(startUserMedia);
|
||||
};
|
||||
</script>
|
||||
|
||||
<button id="start" onclick="startRecording(this);">record</button>
|
||||
<br>
|
||||
<br>
|
||||
<button id="stop" onclick="stopRecording(this);" disabled>stop</button>
|
||||
<br>
|
||||
<br>
|
||||
<audio controls id="audio">
|
||||
<source id="source" src="" type="audio/wav">
|
||||
Your browser does not support the audio element.
|
||||
</audio>
|
||||
</body>
|
||||
</html>
|
||||
'''
|
||||
|
||||
|
||||
class Converter():
|
||||
def __init__(self, model_dir, model_name):
|
||||
self.num_features = 24
|
||||
self.sampling_rate = 16000
|
||||
self.frame_period = 5.0
|
||||
|
||||
self.model = CycleGAN(num_features = self.num_features, mode = 'test')
|
||||
|
||||
self.model.load(filepath = os.path.join(model_dir, model_name))
|
||||
|
||||
"""
|
||||
# NB: Save the graph
|
||||
definition = self.model.sess.graph_def
|
||||
directory = 'saved_model_2'
|
||||
tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True)
|
||||
|
||||
# https://github.com/tensorflow/models/issues/3530#issuecomment-395968881
|
||||
output_dir = './saved_model/'
|
||||
builder = tf.saved_model.builder.SavedModelBuilder(output_dir)
|
||||
|
||||
builder.add_meta_graph_and_variables(
|
||||
self.model.sess,
|
||||
[tf.saved_model.tag_constants.SERVING],
|
||||
main_op=tf.tables_initializer(),
|
||||
)
|
||||
|
||||
builder.save()
|
||||
"""
|
||||
|
||||
"""
|
||||
builder.add_meta_graph_and_variables(
|
||||
self.model.sess,
|
||||
[tf.saved_model.tag_constants.SERVING],
|
||||
signature_def_map={
|
||||
'predict_images':
|
||||
prediction_signature,
|
||||
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
|
||||
classification_signature,
|
||||
},
|
||||
main_op=tf.tables_initializer())
|
||||
"""
|
||||
|
||||
self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz'))
|
||||
self.mcep_mean_A = self.mcep_normalization_params['mean_A']
|
||||
self.mcep_std_A = self.mcep_normalization_params['std_A']
|
||||
self.mcep_mean_B = self.mcep_normalization_params['mean_B']
|
||||
self.mcep_std_B = self.mcep_normalization_params['std_B']
|
||||
|
||||
self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz'))
|
||||
self.logf0s_mean_A = self.logf0s_normalization_params['mean_A']
|
||||
self.logf0s_std_A = self.logf0s_normalization_params['std_A']
|
||||
self.logf0s_mean_B = self.logf0s_normalization_params['mean_B']
|
||||
self.logf0s_std_B = self.logf0s_normalization_params['std_B']
|
||||
|
||||
def convert(self, wav, conversion_direction='A2B'):
|
||||
wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4)
|
||||
f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period)
|
||||
coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features)
|
||||
coded_sp_transposed = coded_sp.T
|
||||
|
||||
if conversion_direction == 'A2B':
|
||||
f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B)
|
||||
coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A
|
||||
coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0]
|
||||
coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B
|
||||
else:
|
||||
f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A)
|
||||
coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B
|
||||
coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0]
|
||||
coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A
|
||||
|
||||
coded_sp_converted = coded_sp_converted.T
|
||||
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
|
||||
decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate)
|
||||
wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period)
|
||||
|
||||
# For debugging model output, uncomment the following line:
|
||||
# librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate)
|
||||
|
||||
# TODO: Perhaps ditch this. It's probably unnecessary work.
|
||||
upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000)
|
||||
pcm_data = upsampled.astype(np.float64)
|
||||
stereo_pcm_data = np.tile(pcm_data, (2,1)).T
|
||||
|
||||
buf = io.BytesIO()
|
||||
scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32))
|
||||
return buf
|
||||
|
||||
# Set up model
|
||||
# This should live long in memory, so we do it up front.
|
||||
model_dir_default = './model/sf1_tm1'
|
||||
model_name_default = 'sf1_tm1.ckpt'
|
||||
converter = Converter(model_dir_default, model_name_default)
|
||||
|
||||
class IndexHandler():
|
||||
def on_get(self, request, response):
|
||||
response.content_type = 'text/html'
|
||||
response.body = INDEX_HTML
|
||||
|
||||
class ApiHandler():
|
||||
def on_post(self, request, response):
|
||||
# NB: uses middleware to pull out data.
|
||||
form_data = request.params['audio_data'].file
|
||||
data, samplerate = soundfile.read(form_data)
|
||||
|
||||
print('samplerate', samplerate)
|
||||
print('data', data)
|
||||
print('data.shape', data.shape)
|
||||
print('data.dtype', data.dtype)
|
||||
|
||||
# For debugging browser input, uncomment the following line:
|
||||
# scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data)
|
||||
|
||||
# NB: Convert the input stereo signal into mono.
|
||||
# In the future the frontend should be responsible for sampling details.
|
||||
mono = data[:, 0]
|
||||
print('mono', mono)
|
||||
print('mono.shape', mono.shape)
|
||||
print('mono.dtype', mono.dtype)
|
||||
|
||||
# NB: We must downsample to the rate that the network is trained on.
|
||||
downsampled = librosa.resample(mono, samplerate, 16000)
|
||||
print('downsampled', downsampled)
|
||||
print('downsampled.shape', downsampled.shape)
|
||||
print('downsampled.dtype', downsampled.dtype)
|
||||
|
||||
# Evaluate the model
|
||||
print(">>> Converting...")
|
||||
results = converter.convert(downsampled, conversion_direction = 'A2B')
|
||||
|
||||
print('type(results)', type(results))
|
||||
|
||||
temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio')
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix='.wav')
|
||||
|
||||
temp_file.write(results.read())
|
||||
|
||||
out_file = temp_dir.name + '/output.ogg'
|
||||
|
||||
# NB: Browsers have a great deal of trouble decoding WAV files unless they are in the
|
||||
# narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python
|
||||
# tools do a good job of this, so here we shell out to ffmpeg and generate OGG.
|
||||
# It's lazy and messy, but it works for now.
|
||||
# See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape
|
||||
# See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs
|
||||
_stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file])
|
||||
|
||||
response.content_type = 'audio/ogg'
|
||||
with open(out_file, mode='rb') as f:
|
||||
response.data = f.read()
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--port', type=int, default=8000)
|
||||
args = parser.parse_args()
|
||||
|
||||
api = falcon.API(middleware=[MultipartMiddleware()])
|
||||
api.add_route('/', IndexHandler())
|
||||
api.add_route('/upload', ApiHandler())
|
||||
api.add_static_route('/script', os.path.abspath('./script'))
|
||||
api.add_static_route('/sound', os.path.abspath('./sound'))
|
||||
print('Serving on 0.0.0.0:%d' % args.port)
|
||||
simple_server.make_server('0.0.0.0', args.port, api).serve_forever()
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
import os
|
||||
import tensorflow as tf
|
||||
from module import discriminator, generator_gatedcnn
|
||||
from utils import l1_loss, l2_loss, cross_entropy_loss
|
||||
from datetime import datetime
|
||||
|
||||
class CycleGAN(object):
|
||||
|
||||
def __init__(self, num_features, discriminator = discriminator, generator = generator_gatedcnn, mode = 'train', log_dir = './log'):
|
||||
|
||||
self.num_features = num_features
|
||||
self.input_shape = [None, num_features, None] # [batch_size, num_features, num_frames]
|
||||
|
||||
self.discriminator = discriminator
|
||||
self.generator = generator
|
||||
self.mode = mode
|
||||
|
||||
self.build_model()
|
||||
self.optimizer_initializer()
|
||||
|
||||
self.saver = tf.train.Saver()
|
||||
|
||||
#config = tf.ConfigProto(device_count = {'GPU': 1})
|
||||
#self.sess = tf.Session(config=config)
|
||||
self.sess = tf.Session()
|
||||
self.sess.run(tf.global_variables_initializer())
|
||||
|
||||
if self.mode == 'train':
|
||||
self.train_step = 0
|
||||
now = datetime.now()
|
||||
self.log_dir = os.path.join(log_dir, now.strftime('%Y%m%d-%H%M%S'))
|
||||
self.writer = tf.summary.FileWriter(self.log_dir, tf.get_default_graph())
|
||||
self.generator_summaries, self.discriminator_summaries = self.summary()
|
||||
|
||||
def build_model(self):
|
||||
|
||||
# Placeholders for real training samples
|
||||
self.input_A_real = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_real')
|
||||
self.input_B_real = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_real')
|
||||
# Placeholders for fake generated samples
|
||||
self.input_A_fake = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_fake')
|
||||
self.input_B_fake = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_fake')
|
||||
# Placeholder for test samples
|
||||
self.input_A_test = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_test')
|
||||
self.input_B_test = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_test')
|
||||
|
||||
self.generation_B = self.generator(inputs = self.input_A_real, reuse = False, scope_name = 'generator_A2B')
|
||||
self.cycle_A = self.generator(inputs = self.generation_B, reuse = False, scope_name = 'generator_B2A')
|
||||
|
||||
self.generation_A = self.generator(inputs = self.input_B_real, reuse = True, scope_name = 'generator_B2A')
|
||||
self.cycle_B = self.generator(inputs = self.generation_A, reuse = True, scope_name = 'generator_A2B')
|
||||
|
||||
self.generation_A_identity = self.generator(inputs = self.input_A_real, reuse = True, scope_name = 'generator_B2A')
|
||||
self.generation_B_identity = self.generator(inputs = self.input_B_real, reuse = True, scope_name = 'generator_A2B')
|
||||
|
||||
self.discrimination_A_fake = self.discriminator(inputs = self.generation_A, reuse = False, scope_name = 'discriminator_A')
|
||||
self.discrimination_B_fake = self.discriminator(inputs = self.generation_B, reuse = False, scope_name = 'discriminator_B')
|
||||
|
||||
# Cycle loss
|
||||
self.cycle_loss = l1_loss(y = self.input_A_real, y_hat = self.cycle_A) + l1_loss(y = self.input_B_real, y_hat = self.cycle_B)
|
||||
|
||||
# Identity loss
|
||||
self.identity_loss = l1_loss(y = self.input_A_real, y_hat = self.generation_A_identity) + l1_loss(y = self.input_B_real, y_hat = self.generation_B_identity)
|
||||
|
||||
# Place holder for lambda_cycle and lambda_identity
|
||||
self.lambda_cycle = tf.placeholder(tf.float32, None, name = 'lambda_cycle')
|
||||
self.lambda_identity = tf.placeholder(tf.float32, None, name = 'lambda_identity')
|
||||
|
||||
# Generator loss
|
||||
# Generator wants to fool discriminator
|
||||
self.generator_loss_A2B = l2_loss(y = tf.ones_like(self.discrimination_B_fake), y_hat = self.discrimination_B_fake)
|
||||
self.generator_loss_B2A = l2_loss(y = tf.ones_like(self.discrimination_A_fake), y_hat = self.discrimination_A_fake)
|
||||
|
||||
# Merge the two generators and the cycle loss
|
||||
self.generator_loss = self.generator_loss_A2B + self.generator_loss_B2A + self.lambda_cycle * self.cycle_loss + self.lambda_identity * self.identity_loss
|
||||
|
||||
# Discriminator loss
|
||||
self.discrimination_input_A_real = self.discriminator(inputs = self.input_A_real, reuse = True, scope_name = 'discriminator_A')
|
||||
self.discrimination_input_B_real = self.discriminator(inputs = self.input_B_real, reuse = True, scope_name = 'discriminator_B')
|
||||
self.discrimination_input_A_fake = self.discriminator(inputs = self.input_A_fake, reuse = True, scope_name = 'discriminator_A')
|
||||
self.discrimination_input_B_fake = self.discriminator(inputs = self.input_B_fake, reuse = True, scope_name = 'discriminator_B')
|
||||
|
||||
# Discriminator wants to classify real and fake correctly
|
||||
self.discriminator_loss_input_A_real = l2_loss(y = tf.ones_like(self.discrimination_input_A_real), y_hat = self.discrimination_input_A_real)
|
||||
self.discriminator_loss_input_A_fake = l2_loss(y = tf.zeros_like(self.discrimination_input_A_fake), y_hat = self.discrimination_input_A_fake)
|
||||
self.discriminator_loss_A = (self.discriminator_loss_input_A_real + self.discriminator_loss_input_A_fake) / 2
|
||||
|
||||
self.discriminator_loss_input_B_real = l2_loss(y = tf.ones_like(self.discrimination_input_B_real), y_hat = self.discrimination_input_B_real)
|
||||
self.discriminator_loss_input_B_fake = l2_loss(y = tf.zeros_like(self.discrimination_input_B_fake), y_hat = self.discrimination_input_B_fake)
|
||||
self.discriminator_loss_B = (self.discriminator_loss_input_B_real + self.discriminator_loss_input_B_fake) / 2
|
||||
|
||||
# Merge the two discriminators into one
|
||||
self.discriminator_loss = self.discriminator_loss_A + self.discriminator_loss_B
|
||||
|
||||
# Categorize variables because we have to optimize the two sets of the variables separately
|
||||
trainable_variables = tf.trainable_variables()
|
||||
self.discriminator_vars = [var for var in trainable_variables if 'discriminator' in var.name]
|
||||
self.generator_vars = [var for var in trainable_variables if 'generator' in var.name]
|
||||
#for var in t_vars: print(var.name)
|
||||
|
||||
# Reserved for test
|
||||
self.generation_B_test = self.generator(inputs = self.input_A_test, reuse = True, scope_name = 'generator_A2B')
|
||||
self.generation_A_test = self.generator(inputs = self.input_B_test, reuse = True, scope_name = 'generator_B2A')
|
||||
|
||||
|
||||
def optimizer_initializer(self):
|
||||
|
||||
self.generator_learning_rate = tf.placeholder(tf.float32, None, name = 'generator_learning_rate')
|
||||
self.discriminator_learning_rate = tf.placeholder(tf.float32, None, name = 'discriminator_learning_rate')
|
||||
self.discriminator_optimizer = tf.train.AdamOptimizer(learning_rate = self.discriminator_learning_rate, beta1 = 0.5).minimize(self.discriminator_loss, var_list = self.discriminator_vars)
|
||||
self.generator_optimizer = tf.train.AdamOptimizer(learning_rate = self.generator_learning_rate, beta1 = 0.5).minimize(self.generator_loss, var_list = self.generator_vars)
|
||||
|
||||
def train(self, input_A, input_B, lambda_cycle, lambda_identity, generator_learning_rate, discriminator_learning_rate):
|
||||
|
||||
generation_A, generation_B, generator_loss, _, generator_summaries = self.sess.run(
|
||||
[self.generation_A, self.generation_B, self.generator_loss, self.generator_optimizer, self.generator_summaries], \
|
||||
feed_dict = {self.lambda_cycle: lambda_cycle, self.lambda_identity: lambda_identity, self.input_A_real: input_A, self.input_B_real: input_B, self.generator_learning_rate: generator_learning_rate})
|
||||
|
||||
self.writer.add_summary(generator_summaries, self.train_step)
|
||||
|
||||
discriminator_loss, _, discriminator_summaries = self.sess.run([self.discriminator_loss, self.discriminator_optimizer, self.discriminator_summaries], \
|
||||
feed_dict = {self.input_A_real: input_A, self.input_B_real: input_B, self.discriminator_learning_rate: discriminator_learning_rate, self.input_A_fake: generation_A, self.input_B_fake: generation_B})
|
||||
|
||||
self.writer.add_summary(discriminator_summaries, self.train_step)
|
||||
|
||||
self.train_step += 1
|
||||
|
||||
return generator_loss, discriminator_loss
|
||||
|
||||
|
||||
def test(self, inputs, direction):
|
||||
#print(">>> model.test()!")
|
||||
|
||||
if direction == 'A2B':
|
||||
#print("generation_B_test: {}".format(self.generation_B_test))
|
||||
#print("input_A_test: {}".format(self.input_A_test))
|
||||
generation = self.sess.run(self.generation_B_test, feed_dict = {self.input_A_test: inputs})
|
||||
elif direction == 'B2A':
|
||||
#print("generation_A_test: {}".format(self.generation_A_test))
|
||||
#print("input_B_test: {}".format(self.input_B_test))
|
||||
generation = self.sess.run(self.generation_A_test, feed_dict = {self.input_B_test: inputs})
|
||||
else:
|
||||
raise Exception('Conversion direction must be specified.')
|
||||
|
||||
return generation
|
||||
|
||||
|
||||
def save(self, directory, filename):
|
||||
|
||||
if not os.path.exists(directory):
|
||||
os.makedirs(directory)
|
||||
self.saver.save(self.sess, os.path.join(directory, filename))
|
||||
|
||||
return os.path.join(directory, filename)
|
||||
|
||||
def load(self, filepath):
|
||||
|
||||
self.saver.restore(self.sess, filepath)
|
||||
|
||||
|
||||
def summary(self):
|
||||
|
||||
with tf.name_scope('generator_summaries'):
|
||||
cycle_loss_summary = tf.summary.scalar('cycle_loss', self.cycle_loss)
|
||||
identity_loss_summary = tf.summary.scalar('identity_loss', self.identity_loss)
|
||||
generator_loss_A2B_summary = tf.summary.scalar('generator_loss_A2B', self.generator_loss_A2B)
|
||||
generator_loss_B2A_summary = tf.summary.scalar('generator_loss_B2A', self.generator_loss_B2A)
|
||||
generator_loss_summary = tf.summary.scalar('generator_loss', self.generator_loss)
|
||||
generator_summaries = tf.summary.merge([cycle_loss_summary, identity_loss_summary, generator_loss_A2B_summary, generator_loss_B2A_summary, generator_loss_summary])
|
||||
|
||||
with tf.name_scope('discriminator_summaries'):
|
||||
discriminator_loss_A_summary = tf.summary.scalar('discriminator_loss_A', self.discriminator_loss_A)
|
||||
discriminator_loss_B_summary = tf.summary.scalar('discriminator_loss_B', self.discriminator_loss_B)
|
||||
discriminator_loss_summary = tf.summary.scalar('discriminator_loss', self.discriminator_loss)
|
||||
discriminator_summaries = tf.summary.merge([discriminator_loss_A_summary, discriminator_loss_B_summary, discriminator_loss_summary])
|
||||
|
||||
return generator_summaries, discriminator_summaries
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
model = CycleGAN(num_features = 24)
|
||||
#print('Graph Compile Successeded.')
|
||||
@@ -0,0 +1,214 @@
|
||||
import tensorflow as tf
|
||||
|
||||
def gated_linear_layer(inputs, gates, name = None):
|
||||
|
||||
activation = tf.multiply(x = inputs, y = tf.sigmoid(gates), name = name)
|
||||
|
||||
return activation
|
||||
|
||||
def instance_norm_layer(
|
||||
inputs,
|
||||
epsilon = 1e-06,
|
||||
activation_fn = None,
|
||||
name = None):
|
||||
|
||||
instance_norm_layer = tf.contrib.layers.instance_norm(
|
||||
inputs = inputs,
|
||||
epsilon = epsilon,
|
||||
activation_fn = activation_fn)
|
||||
|
||||
return instance_norm_layer
|
||||
|
||||
def conv1d_layer(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides = 1,
|
||||
padding = 'same',
|
||||
activation = None,
|
||||
kernel_initializer = None,
|
||||
name = None):
|
||||
|
||||
conv_layer = tf.layers.conv1d(
|
||||
inputs = inputs,
|
||||
filters = filters,
|
||||
kernel_size = kernel_size,
|
||||
strides = strides,
|
||||
padding = padding,
|
||||
activation = activation,
|
||||
kernel_initializer = kernel_initializer,
|
||||
name = name)
|
||||
|
||||
return conv_layer
|
||||
|
||||
def conv2d_layer(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
padding = 'same',
|
||||
activation = None,
|
||||
kernel_initializer = None,
|
||||
name = None):
|
||||
|
||||
conv_layer = tf.layers.conv2d(
|
||||
inputs = inputs,
|
||||
filters = filters,
|
||||
kernel_size = kernel_size,
|
||||
strides = strides,
|
||||
padding = padding,
|
||||
activation = activation,
|
||||
kernel_initializer = kernel_initializer,
|
||||
name = name)
|
||||
|
||||
return conv_layer
|
||||
|
||||
def residual1d_block(
|
||||
inputs,
|
||||
filters = 1024,
|
||||
kernel_size = 3,
|
||||
strides = 1,
|
||||
name_prefix = 'residule_block_'):
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
h2 = conv1d_layer(inputs = h1_glu, filters = filters // 2, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h2_conv')
|
||||
h2_norm = instance_norm_layer(inputs = h2, activation_fn = None, name = name_prefix + 'h2_norm')
|
||||
|
||||
h3 = inputs + h2_norm
|
||||
|
||||
return h3
|
||||
|
||||
def downsample1d_block(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
name_prefix = 'downsample1d_block_'):
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
|
||||
return h1_glu
|
||||
|
||||
def downsample2d_block(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
name_prefix = 'downsample2d_block_'):
|
||||
|
||||
h1 = conv2d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
h1_gates = conv2d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
|
||||
return h1_glu
|
||||
|
||||
def upsample1d_block(
|
||||
inputs,
|
||||
filters,
|
||||
kernel_size,
|
||||
strides,
|
||||
shuffle_size = 2,
|
||||
name_prefix = 'upsample1d_block_'):
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv')
|
||||
h1_shuffle = pixel_shuffler(inputs = h1, shuffle_size = shuffle_size, name = name_prefix + 'h1_shuffle')
|
||||
h1_norm = instance_norm_layer(inputs = h1_shuffle, activation_fn = None, name = name_prefix + 'h1_norm')
|
||||
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates')
|
||||
h1_shuffle_gates = pixel_shuffler(inputs = h1_gates, shuffle_size = shuffle_size, name = name_prefix + 'h1_shuffle_gates')
|
||||
h1_norm_gates = instance_norm_layer(inputs = h1_shuffle_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates')
|
||||
|
||||
h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu')
|
||||
|
||||
return h1_glu
|
||||
|
||||
def pixel_shuffler(inputs, shuffle_size = 2, name = None):
|
||||
|
||||
n = tf.shape(inputs)[0]
|
||||
w = tf.shape(inputs)[1]
|
||||
c = inputs.get_shape().as_list()[2]
|
||||
|
||||
oc = c // shuffle_size
|
||||
ow = w * shuffle_size
|
||||
|
||||
outputs = tf.reshape(tensor = inputs, shape = [n, ow, oc], name = name)
|
||||
|
||||
return outputs
|
||||
|
||||
def generator_gatedcnn(inputs, reuse = False, scope_name = 'generator_gatedcnn'):
|
||||
|
||||
# inputs has shape [batch_size, num_features, time]
|
||||
# we need to convert it to [batch_size, time, num_features] for 1D convolution
|
||||
inputs = tf.transpose(inputs, perm = [0, 2, 1], name = 'input_transpose')
|
||||
|
||||
with tf.variable_scope(scope_name) as scope:
|
||||
# Discriminator would be reused in CycleGAN
|
||||
if reuse:
|
||||
scope.reuse_variables()
|
||||
else:
|
||||
assert scope.reuse is False
|
||||
|
||||
h1 = conv1d_layer(inputs = inputs, filters = 128, kernel_size = 15, strides = 1, activation = None, name = 'h1_conv')
|
||||
h1_gates = conv1d_layer(inputs = inputs, filters = 128, kernel_size = 15, strides = 1, activation = None, name = 'h1_conv_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1, gates = h1_gates, name = 'h1_glu')
|
||||
|
||||
# Downsample
|
||||
d1 = downsample1d_block(inputs = h1_glu, filters = 256, kernel_size = 5, strides = 2, name_prefix = 'downsample1d_block1_')
|
||||
d2 = downsample1d_block(inputs = d1, filters = 512, kernel_size = 5, strides = 2, name_prefix = 'downsample1d_block2_')
|
||||
|
||||
# Residual blocks
|
||||
r1 = residual1d_block(inputs = d2, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block1_')
|
||||
r2 = residual1d_block(inputs = r1, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block2_')
|
||||
r3 = residual1d_block(inputs = r2, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block3_')
|
||||
r4 = residual1d_block(inputs = r3, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block4_')
|
||||
r5 = residual1d_block(inputs = r4, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block5_')
|
||||
r6 = residual1d_block(inputs = r5, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block6_')
|
||||
|
||||
# Upsample
|
||||
u1 = upsample1d_block(inputs = r6, filters = 1024, kernel_size = 5, strides = 1, shuffle_size = 2, name_prefix = 'upsample1d_block1_')
|
||||
u2 = upsample1d_block(inputs = u1, filters = 512, kernel_size = 5, strides = 1, shuffle_size = 2, name_prefix = 'upsample1d_block2_')
|
||||
|
||||
# Output
|
||||
o1 = conv1d_layer(inputs = u2, filters = 24, kernel_size = 15, strides = 1, activation = None, name = 'o1_conv')
|
||||
o2 = tf.transpose(o1, perm = [0, 2, 1], name = 'output_transpose')
|
||||
|
||||
return o2
|
||||
|
||||
|
||||
def discriminator(inputs, reuse = False, scope_name = 'discriminator'):
|
||||
|
||||
# inputs has shape [batch_size, num_features, time]
|
||||
# we need to add channel for 2D convolution [batch_size, num_features, time, 1]
|
||||
inputs = tf.expand_dims(inputs, -1)
|
||||
|
||||
with tf.variable_scope(scope_name) as scope:
|
||||
# Discriminator would be reused in CycleGAN
|
||||
if reuse:
|
||||
scope.reuse_variables()
|
||||
else:
|
||||
assert scope.reuse is False
|
||||
|
||||
h1 = conv2d_layer(inputs = inputs, filters = 128, kernel_size = [3, 3], strides = [1, 2], activation = None, name = 'h1_conv')
|
||||
h1_gates = conv2d_layer(inputs = inputs, filters = 128, kernel_size = [3, 3], strides = [1, 2], activation = None, name = 'h1_conv_gates')
|
||||
h1_glu = gated_linear_layer(inputs = h1, gates = h1_gates, name = 'h1_glu')
|
||||
|
||||
# Downsample
|
||||
d1 = downsample2d_block(inputs = h1_glu, filters = 256, kernel_size = [3, 3], strides = [2, 2], name_prefix = 'downsample2d_block1_')
|
||||
d2 = downsample2d_block(inputs = d1, filters = 512, kernel_size = [3, 3], strides = [2, 2], name_prefix = 'downsample2d_block2_')
|
||||
d3 = downsample2d_block(inputs = d2, filters = 1024, kernel_size = [6, 3], strides = [1, 2], name_prefix = 'downsample2d_block3_')
|
||||
|
||||
# Output
|
||||
o1 = tf.layers.dense(inputs = d3, units = 1, activation = tf.nn.sigmoid)
|
||||
|
||||
return o1
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
import librosa
|
||||
import numpy as np
|
||||
import os
|
||||
import pyworld
|
||||
|
||||
def load_wavs(wav_dir, sr):
|
||||
|
||||
wavs = list()
|
||||
for file in os.listdir(wav_dir):
|
||||
file_path = os.path.join(wav_dir, file)
|
||||
wav, _ = librosa.load(file_path, sr = sr, mono = True)
|
||||
#wav = wav.astype(np.float64)
|
||||
wavs.append(wav)
|
||||
|
||||
return wavs
|
||||
|
||||
def world_decompose(wav, fs, frame_period = 5.0):
|
||||
|
||||
# Decompose speech signal into f0, spectral envelope and aperiodicity using WORLD
|
||||
wav = wav.astype(np.float64)
|
||||
f0, timeaxis = pyworld.harvest(wav, fs, frame_period = frame_period, f0_floor = 71.0, f0_ceil = 800.0)
|
||||
sp = pyworld.cheaptrick(wav, f0, timeaxis, fs)
|
||||
ap = pyworld.d4c(wav, f0, timeaxis, fs)
|
||||
|
||||
return f0, timeaxis, sp, ap
|
||||
|
||||
def world_encode_spectral_envelop(sp, fs, dim = 24):
|
||||
|
||||
# Get Mel-cepstral coefficients (MCEPs)
|
||||
|
||||
#sp = sp.astype(np.float64)
|
||||
coded_sp = pyworld.code_spectral_envelope(sp, fs, dim)
|
||||
|
||||
return coded_sp
|
||||
|
||||
def world_decode_spectral_envelop(coded_sp, fs):
|
||||
|
||||
fftlen = pyworld.get_cheaptrick_fft_size(fs)
|
||||
#coded_sp = coded_sp.astype(np.float32)
|
||||
#coded_sp = np.ascontiguousarray(coded_sp)
|
||||
decoded_sp = pyworld.decode_spectral_envelope(coded_sp, fs, fftlen)
|
||||
|
||||
return decoded_sp
|
||||
|
||||
|
||||
def world_encode_data(wavs, fs, frame_period = 5.0, coded_dim = 24):
|
||||
|
||||
f0s = list()
|
||||
timeaxes = list()
|
||||
sps = list()
|
||||
aps = list()
|
||||
coded_sps = list()
|
||||
|
||||
for wav in wavs:
|
||||
f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = fs, frame_period = frame_period)
|
||||
coded_sp = world_encode_spectral_envelop(sp = sp, fs = fs, dim = coded_dim)
|
||||
f0s.append(f0)
|
||||
timeaxes.append(timeaxis)
|
||||
sps.append(sp)
|
||||
aps.append(ap)
|
||||
coded_sps.append(coded_sp)
|
||||
|
||||
return f0s, timeaxes, sps, aps, coded_sps
|
||||
|
||||
|
||||
def transpose_in_list(lst):
|
||||
|
||||
transposed_lst = list()
|
||||
for array in lst:
|
||||
transposed_lst.append(array.T)
|
||||
return transposed_lst
|
||||
|
||||
|
||||
def world_decode_data(coded_sps, fs):
|
||||
|
||||
decoded_sps = list()
|
||||
|
||||
for coded_sp in coded_sps:
|
||||
decoded_sp = world_decode_spectral_envelop(coded_sp, fs)
|
||||
decoded_sps.append(decoded_sp)
|
||||
|
||||
return decoded_sps
|
||||
|
||||
|
||||
def world_speech_synthesis(f0, decoded_sp, ap, fs, frame_period):
|
||||
|
||||
#decoded_sp = decoded_sp.astype(np.float64)
|
||||
wav = pyworld.synthesize(f0, decoded_sp, ap, fs, frame_period)
|
||||
# Librosa could not save wav if not doing so
|
||||
wav = wav.astype(np.float32)
|
||||
|
||||
return wav
|
||||
|
||||
|
||||
def world_synthesis_data(f0s, decoded_sps, aps, fs, frame_period):
|
||||
|
||||
wavs = list()
|
||||
|
||||
for f0, decoded_sp, ap in zip(f0s, decoded_sps, aps):
|
||||
wav = world_speech_synthesis(f0, decoded_sp, ap, fs, frame_period)
|
||||
wavs.append(wav)
|
||||
|
||||
return wavs
|
||||
|
||||
|
||||
def coded_sps_normalization_fit_transoform(coded_sps):
|
||||
|
||||
coded_sps_concatenated = np.concatenate(coded_sps, axis = 1)
|
||||
coded_sps_mean = np.mean(coded_sps_concatenated, axis = 1, keepdims = True)
|
||||
coded_sps_std = np.std(coded_sps_concatenated, axis = 1, keepdims = True)
|
||||
|
||||
coded_sps_normalized = list()
|
||||
for coded_sp in coded_sps:
|
||||
coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std)
|
||||
|
||||
return coded_sps_normalized, coded_sps_mean, coded_sps_std
|
||||
|
||||
def coded_sps_normalization_transoform(coded_sps, coded_sps_mean, coded_sps_std):
|
||||
|
||||
coded_sps_normalized = list()
|
||||
for coded_sp in coded_sps:
|
||||
coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std)
|
||||
|
||||
return coded_sps_normalized
|
||||
|
||||
def coded_sps_normalization_inverse_transoform(normalized_coded_sps, coded_sps_mean, coded_sps_std):
|
||||
|
||||
coded_sps = list()
|
||||
for normalized_coded_sp in normalized_coded_sps:
|
||||
coded_sps.append(normalized_coded_sp * coded_sps_std + coded_sps_mean)
|
||||
|
||||
return coded_sps
|
||||
|
||||
def coded_sp_padding(coded_sp, multiple = 4):
|
||||
|
||||
num_features = coded_sp.shape[0]
|
||||
num_frames = coded_sp.shape[1]
|
||||
num_frames_padded = int(np.ceil(num_frames / multiple)) * multiple
|
||||
num_frames_diff = num_frames_padded - num_frames
|
||||
num_pad_left = num_frames_diff // 2
|
||||
num_pad_right = num_frames_diff - num_pad_left
|
||||
coded_sp_padded = np.pad(coded_sp, ((0, 0), (num_pad_left, num_pad_right)), 'constant', constant_values = 0)
|
||||
|
||||
return coded_sp_padded
|
||||
|
||||
def wav_padding(wav, sr, frame_period, multiple = 4):
|
||||
|
||||
assert wav.ndim == 1
|
||||
num_frames = len(wav)
|
||||
num_frames_padded = int((np.ceil((np.floor(num_frames / (sr * frame_period / 1000)) + 1) / multiple + 1) * multiple - 1) * (sr * frame_period / 1000))
|
||||
num_frames_diff = num_frames_padded - num_frames
|
||||
num_pad_left = num_frames_diff // 2
|
||||
num_pad_right = num_frames_diff - num_pad_left
|
||||
wav_padded = np.pad(wav, (num_pad_left, num_pad_right), 'constant', constant_values = 0)
|
||||
|
||||
return wav_padded
|
||||
|
||||
|
||||
def logf0_statistics(f0s):
|
||||
|
||||
log_f0s_concatenated = np.ma.log(np.concatenate(f0s))
|
||||
log_f0s_mean = log_f0s_concatenated.mean()
|
||||
log_f0s_std = log_f0s_concatenated.std()
|
||||
|
||||
return log_f0s_mean, log_f0s_std
|
||||
|
||||
def pitch_conversion(f0, mean_log_src, std_log_src, mean_log_target, std_log_target):
|
||||
|
||||
# Logarithm Gaussian normalization for Pitch Conversions
|
||||
f0_converted = np.exp((np.log(f0) - mean_log_src) / std_log_src * std_log_target + mean_log_target)
|
||||
|
||||
return f0_converted
|
||||
|
||||
def wavs_to_specs(wavs, n_fft = 1024, hop_length = None):
|
||||
|
||||
stfts = list()
|
||||
for wav in wavs:
|
||||
stft = librosa.stft(wav, n_fft = n_fft, hop_length = hop_length)
|
||||
stfts.append(stft)
|
||||
|
||||
return stfts
|
||||
|
||||
|
||||
def wavs_to_mfccs(wavs, sr, n_fft = 1024, hop_length = None, n_mels = 128, n_mfcc = 24):
|
||||
|
||||
mfccs = list()
|
||||
for wav in wavs:
|
||||
mfcc = librosa.feature.mfcc(y = wav, sr = sr, n_fft = n_fft, hop_length = hop_length, n_mels = n_mels, n_mfcc = n_mfcc)
|
||||
mfccs.append(mfcc)
|
||||
|
||||
return mfccs
|
||||
|
||||
|
||||
def mfccs_normalization(mfccs):
|
||||
|
||||
mfccs_concatenated = np.concatenate(mfccs, axis = 1)
|
||||
mfccs_mean = np.mean(mfccs_concatenated, axis = 1, keepdims = True)
|
||||
mfccs_std = np.std(mfccs_concatenated, axis = 1, keepdims = True)
|
||||
|
||||
mfccs_normalized = list()
|
||||
for mfcc in mfccs:
|
||||
mfccs_normalized.append((mfcc - mfccs_mean) / mfccs_std)
|
||||
|
||||
return mfccs_normalized, mfccs_mean, mfccs_std
|
||||
|
||||
|
||||
def sample_train_data(dataset_A, dataset_B, n_frames = 128):
|
||||
|
||||
num_samples = min(len(dataset_A), len(dataset_B))
|
||||
train_data_A_idx = np.arange(len(dataset_A))
|
||||
train_data_B_idx = np.arange(len(dataset_B))
|
||||
np.random.shuffle(train_data_A_idx)
|
||||
np.random.shuffle(train_data_B_idx)
|
||||
train_data_A_idx_subset = train_data_A_idx[:num_samples]
|
||||
train_data_B_idx_subset = train_data_B_idx[:num_samples]
|
||||
|
||||
train_data_A = list()
|
||||
train_data_B = list()
|
||||
|
||||
for idx_A, idx_B in zip(train_data_A_idx_subset, train_data_B_idx_subset):
|
||||
data_A = dataset_A[idx_A]
|
||||
frames_A_total = data_A.shape[1]
|
||||
assert frames_A_total >= n_frames
|
||||
start_A = np.random.randint(frames_A_total - n_frames + 1)
|
||||
end_A = start_A + n_frames
|
||||
train_data_A.append(data_A[:,start_A:end_A])
|
||||
|
||||
data_B = dataset_B[idx_B]
|
||||
frames_B_total = data_B.shape[1]
|
||||
assert frames_B_total >= n_frames
|
||||
start_B = np.random.randint(frames_B_total - n_frames + 1)
|
||||
end_B = start_B + n_frames
|
||||
train_data_B.append(data_B[:,start_B:end_B])
|
||||
|
||||
train_data_A = np.array(train_data_A)
|
||||
train_data_B = np.array(train_data_B)
|
||||
|
||||
return train_data_A, train_data_B
|
||||
@@ -0,0 +1,260 @@
|
||||
# Generated by the protocol buffer compiler. DO NOT EDIT!
|
||||
# source: protos/audio.proto
|
||||
|
||||
import sys
|
||||
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
|
||||
from google.protobuf import descriptor as _descriptor
|
||||
from google.protobuf import message as _message
|
||||
from google.protobuf import reflection as _reflection
|
||||
from google.protobuf import symbol_database as _symbol_database
|
||||
from google.protobuf import descriptor_pb2
|
||||
# @@protoc_insertion_point(imports)
|
||||
|
||||
_sym_db = _symbol_database.Default()
|
||||
|
||||
|
||||
|
||||
|
||||
DESCRIPTOR = _descriptor.FileDescriptor(
|
||||
name='protos/audio.proto',
|
||||
package='voder_audio',
|
||||
syntax='proto3',
|
||||
serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xe6\x04\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x1c\n\x14request_batch_number\x18\x02 \x01(\x03\x12\x43\n\rvocode_params\x18\n \x01(\x0b\x32,.voder_audio.VocodeAudioRequest.VocodeParams\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x1a\x86\x03\n\x0cVocodeParams\x12\x1b\n\x13initial_sample_rate\x18\x01 \x01(\x05\x12\x1c\n\x14original_source_rate\x18\t \x01(\x05\x12!\n\x19original_source_save_file\x18\n \x01(\x08\x12\x1c\n\x14pre_convert_resample\x18\x02 \x01(\x08\x12!\n\x19pre_convert_resample_rate\x18\x03 \x01(\x05\x12&\n\x1epre_convert_resample_save_file\x18\x04 \x01(\x08\x12*\n\"model_hyperparameter_sampling_rate\x18\x05 \x01(\x05\x12\x17\n\x0fmodel_save_file\x18\x0b \x01(\x08\x12\x1d\n\x15post_convert_resample\x18\x06 \x01(\x08\x12\"\n\x1apost_convert_resample_rate\x18\x07 \x01(\x05\x12\'\n\x1fpost_convert_resample_save_file\x18\x08 \x01(\x08\"I\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x1d\n\x15response_batch_number\x18\x02 \x01(\x03\x62\x06proto3')
|
||||
)
|
||||
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
|
||||
|
||||
|
||||
|
||||
|
||||
_VOCODEAUDIOREQUEST_VOCODEPARAMS = _descriptor.Descriptor(
|
||||
name='VocodeParams',
|
||||
full_name='voder_audio.VocodeAudioRequest.VocodeParams',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='initial_sample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.initial_sample_rate', index=0,
|
||||
number=1, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='original_source_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_rate', index=1,
|
||||
number=9, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='original_source_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_save_file', index=2,
|
||||
number=10, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='pre_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample', index=3,
|
||||
number=2, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='pre_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_rate', index=4,
|
||||
number=3, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='pre_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_save_file', index=5,
|
||||
number=4, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='model_hyperparameter_sampling_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_hyperparameter_sampling_rate', index=6,
|
||||
number=5, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='model_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_save_file', index=7,
|
||||
number=11, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='post_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample', index=8,
|
||||
number=6, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='post_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_rate', index=9,
|
||||
number=7, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='post_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_save_file', index=10,
|
||||
number=8, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=260,
|
||||
serialized_end=650,
|
||||
)
|
||||
|
||||
_VOCODEAUDIOREQUEST = _descriptor.Descriptor(
|
||||
name='VocodeAudioRequest',
|
||||
full_name='voder_audio.VocodeAudioRequest',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='float_audio', full_name='voder_audio.VocodeAudioRequest.float_audio', index=0,
|
||||
number=1, type=2, cpp_type=6, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='request_batch_number', full_name='voder_audio.VocodeAudioRequest.request_batch_number', index=1,
|
||||
number=2, type=3, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='vocode_params', full_name='voder_audio.VocodeAudioRequest.vocode_params', index=2,
|
||||
number=10, type=11, cpp_type=10, label=1,
|
||||
has_default_value=False, default_value=None,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='skip_vocode', full_name='voder_audio.VocodeAudioRequest.skip_vocode', index=3,
|
||||
number=4, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='buffer_size_minimum', full_name='voder_audio.VocodeAudioRequest.buffer_size_minimum', index=4,
|
||||
number=6, type=5, cpp_type=1, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=5,
|
||||
number=8, type=8, cpp_type=7, label=1,
|
||||
has_default_value=False, default_value=False,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[_VOCODEAUDIOREQUEST_VOCODEPARAMS, ],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=36,
|
||||
serialized_end=650,
|
||||
)
|
||||
|
||||
|
||||
_VOCODEAUDIORESPONSE = _descriptor.Descriptor(
|
||||
name='VocodeAudioResponse',
|
||||
full_name='voder_audio.VocodeAudioResponse',
|
||||
filename=None,
|
||||
file=DESCRIPTOR,
|
||||
containing_type=None,
|
||||
fields=[
|
||||
_descriptor.FieldDescriptor(
|
||||
name='float_audio', full_name='voder_audio.VocodeAudioResponse.float_audio', index=0,
|
||||
number=1, type=2, cpp_type=6, label=3,
|
||||
has_default_value=False, default_value=[],
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
_descriptor.FieldDescriptor(
|
||||
name='response_batch_number', full_name='voder_audio.VocodeAudioResponse.response_batch_number', index=1,
|
||||
number=2, type=3, cpp_type=2, label=1,
|
||||
has_default_value=False, default_value=0,
|
||||
message_type=None, enum_type=None, containing_type=None,
|
||||
is_extension=False, extension_scope=None,
|
||||
options=None),
|
||||
],
|
||||
extensions=[
|
||||
],
|
||||
nested_types=[],
|
||||
enum_types=[
|
||||
],
|
||||
options=None,
|
||||
is_extendable=False,
|
||||
syntax='proto3',
|
||||
extension_ranges=[],
|
||||
oneofs=[
|
||||
],
|
||||
serialized_start=652,
|
||||
serialized_end=725,
|
||||
)
|
||||
|
||||
_VOCODEAUDIOREQUEST_VOCODEPARAMS.containing_type = _VOCODEAUDIOREQUEST
|
||||
_VOCODEAUDIOREQUEST.fields_by_name['vocode_params'].message_type = _VOCODEAUDIOREQUEST_VOCODEPARAMS
|
||||
DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST
|
||||
DESCRIPTOR.message_types_by_name['VocodeAudioResponse'] = _VOCODEAUDIORESPONSE
|
||||
|
||||
VocodeAudioRequest = _reflection.GeneratedProtocolMessageType('VocodeAudioRequest', (_message.Message,), dict(
|
||||
|
||||
VocodeParams = _reflection.GeneratedProtocolMessageType('VocodeParams', (_message.Message,), dict(
|
||||
DESCRIPTOR = _VOCODEAUDIOREQUEST_VOCODEPARAMS,
|
||||
__module__ = 'protos.audio_pb2'
|
||||
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest.VocodeParams)
|
||||
))
|
||||
,
|
||||
DESCRIPTOR = _VOCODEAUDIOREQUEST,
|
||||
__module__ = 'protos.audio_pb2'
|
||||
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest)
|
||||
))
|
||||
_sym_db.RegisterMessage(VocodeAudioRequest)
|
||||
_sym_db.RegisterMessage(VocodeAudioRequest.VocodeParams)
|
||||
|
||||
VocodeAudioResponse = _reflection.GeneratedProtocolMessageType('VocodeAudioResponse', (_message.Message,), dict(
|
||||
DESCRIPTOR = _VOCODEAUDIORESPONSE,
|
||||
__module__ = 'protos.audio_pb2'
|
||||
# @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioResponse)
|
||||
))
|
||||
_sym_db.RegisterMessage(VocodeAudioResponse)
|
||||
|
||||
|
||||
# @@protoc_insertion_point(module_scope)
|
||||
@@ -0,0 +1,36 @@
|
||||
absl-py==0.7.1
|
||||
astor==0.8.0
|
||||
audioread==2.1.8
|
||||
cffi==1.12.3
|
||||
Cython==0.29.12
|
||||
decorator==4.4.0
|
||||
falcon==2.0.0
|
||||
falcon-multipart==0.2.0
|
||||
gast==0.2.2
|
||||
google-pasta==0.1.7
|
||||
grpcio==1.22.0
|
||||
h5py==2.9.0
|
||||
joblib==0.13.2
|
||||
Keras-Applications==1.0.8
|
||||
Keras-Preprocessing==1.1.0
|
||||
librosa==0.7.0
|
||||
llvmlite==0.29.0
|
||||
Markdown==3.1.1
|
||||
numba==0.45.0
|
||||
numpy==1.16.4
|
||||
protobuf==3.9.0
|
||||
pycparser==2.19
|
||||
pyworld==0.2.8
|
||||
pyzmq==18.1.0
|
||||
resampy==0.2.1
|
||||
scikit-learn==0.21.2
|
||||
scipy==1.3.0
|
||||
six==1.12.0
|
||||
SoundFile==0.10.2
|
||||
tensorboard==1.14.0
|
||||
tensorflow-estimator==1.14.0
|
||||
tensorflow-gpu==1.14.0
|
||||
termcolor==1.1.0
|
||||
Werkzeug==0.15.5
|
||||
wrapt==1.11.2
|
||||
zmq==0.0.0
|
||||
Executable
+264
@@ -0,0 +1,264 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Starts a demo HTTP server to capture and transform audio
|
||||
as a live demonstration of the trained model.
|
||||
|
||||
Brandon Thomas 2019-07-29 <bt@brand.io> <echelon@gmail.com>
|
||||
"""
|
||||
|
||||
# noinspection PyInterpreter
|
||||
import argparse
|
||||
import io
|
||||
import librosa
|
||||
import numpy as np
|
||||
import os
|
||||
import pathlib
|
||||
import scipy
|
||||
import soundfile
|
||||
import struct
|
||||
import subprocess
|
||||
import tempfile
|
||||
import tensorflow as tf
|
||||
import zmq
|
||||
|
||||
from model import CycleGAN
|
||||
from preprocess import *
|
||||
from protos.audio_pb2 import VocodeAudioRequest
|
||||
from protos.audio_pb2 import VocodeAudioResponse
|
||||
|
||||
print("TensorFlow version: {}".format(tf.version.VERSION))
|
||||
|
||||
|
||||
class Converter():
|
||||
def __init__(self, model_dir, model_name):
|
||||
self.num_features = 24
|
||||
self.frame_period = 5.0
|
||||
|
||||
self.model = CycleGAN(num_features = self.num_features, mode = 'test')
|
||||
|
||||
self.model.load(filepath = os.path.join(model_dir, model_name))
|
||||
|
||||
self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz'))
|
||||
self.mcep_mean_A = self.mcep_normalization_params['mean_A']
|
||||
self.mcep_std_A = self.mcep_normalization_params['std_A']
|
||||
self.mcep_mean_B = self.mcep_normalization_params['mean_B']
|
||||
self.mcep_std_B = self.mcep_normalization_params['std_B']
|
||||
|
||||
self.logf0s_normalization_params = np.load(os.path.join(model_dir,
|
||||
'logf0s_normalization.npz'))
|
||||
self.logf0s_mean_A = self.logf0s_normalization_params['mean_A']
|
||||
self.logf0s_std_A = self.logf0s_normalization_params['std_A']
|
||||
self.logf0s_mean_B = self.logf0s_normalization_params['mean_B']
|
||||
self.logf0s_std_B = self.logf0s_normalization_params['std_B']
|
||||
|
||||
def convert_partial(self, wav, conversion_direction='A2B', model_sampling_rate=16000):
|
||||
wav = wav_padding(wav = wav,
|
||||
sr = model_sampling_rate,
|
||||
frame_period = self.frame_period,
|
||||
multiple = 4)
|
||||
|
||||
#librosa.output.write_wav('wav_padding.wav', wav, model_sampling_rate)
|
||||
|
||||
f0, timeaxis, sp, ap = world_decompose(wav = wav,
|
||||
fs = model_sampling_rate,
|
||||
frame_period = self.frame_period)
|
||||
coded_sp = world_encode_spectral_envelop(sp = sp,
|
||||
fs = model_sampling_rate,
|
||||
dim = self.num_features)
|
||||
coded_sp_transposed = coded_sp.T
|
||||
|
||||
f0_converted = pitch_conversion(f0 = f0,
|
||||
mean_log_src = self.logf0s_mean_A,
|
||||
std_log_src = self.logf0s_std_A,
|
||||
mean_log_target = self.logf0s_mean_B,
|
||||
std_log_target = self.logf0s_std_B)
|
||||
coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A
|
||||
|
||||
coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]),
|
||||
direction = conversion_direction)[0]
|
||||
coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B
|
||||
|
||||
coded_sp_converted = coded_sp_converted.T
|
||||
coded_sp_converted = np.ascontiguousarray(coded_sp_converted)
|
||||
decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted,
|
||||
fs = model_sampling_rate)
|
||||
wav_transformed = world_speech_synthesis(f0 = f0_converted,
|
||||
decoded_sp = decoded_sp_converted,
|
||||
ap = ap,
|
||||
fs = model_sampling_rate,
|
||||
frame_period = self.frame_period)
|
||||
|
||||
# For debugging model output, uncomment the following line:
|
||||
#librosa.output.write_wav('model_output.wav', wav_transformed, model_sampling_rate)
|
||||
|
||||
# TODO: Perhaps ditch this. It's probably unnecessary work.
|
||||
#upsampled = librosa.resample(wav_transformed, model_sampling_rate, 48000)
|
||||
#pcm_data = upsampled.astype(np.float64)
|
||||
#stereo_pcm_data = np.tile(pcm_data, (2,1)).T
|
||||
#return stereo_pcm_data.astype(np.float32)
|
||||
#librosa.output.write_wav('model_output.wav', pcm_data, 48000)
|
||||
|
||||
#return wav
|
||||
return wav_transformed
|
||||
|
||||
#def convert(self, wav, conversion_direction='A2B'):
|
||||
# pcm_data = self.convert_partial(wav, conversion_direction=conversion_direction)
|
||||
# buf = io.BytesIO()
|
||||
# # pcm_data: A 1-D or 2-D numpy array of either integer or float data-type.
|
||||
# # To write multiple-channels, use a 2-D array of shape (Nsamples, Nchannels).
|
||||
# scipy.io.wavfile.write(buf, 48000, pcm_data)
|
||||
# return buf
|
||||
|
||||
# Set up model
|
||||
# This should live long in memory, so we do it up front.
|
||||
model_dir_default = './model/sf1_tm1'
|
||||
model_name_default = 'sf1_tm1.ckpt'
|
||||
|
||||
# TODO: UNCOMMENT
|
||||
converter = Converter(model_dir_default, model_name_default)
|
||||
|
||||
TEMP_DIR = tempfile.TemporaryDirectory(prefix='queue_audio')
|
||||
|
||||
def temp_file_name(suffix='.wav'):
|
||||
# NB: Not actually using the tempfile. Just the random name.
|
||||
temp_file = tempfile.NamedTemporaryFile(suffix=suffix)
|
||||
name = os.path.basename(temp_file.name)
|
||||
return os.path.join(TEMP_DIR.name, name)
|
||||
|
||||
def convert(audio,
|
||||
vocode_params=None,
|
||||
request_batch_number=0,
|
||||
skip_vocode=False,
|
||||
save_files=False,
|
||||
discard_vocoded_audio=False):
|
||||
#audio = np.array(audio, dtype=np.int16)
|
||||
#data, samplerate = soundfile.read(audio)
|
||||
#print('samplerate', samplerate)
|
||||
"""
|
||||
samplerate 44100
|
||||
data.shape (77824, 2)
|
||||
data.dtype float64
|
||||
mono [0.0050354 0.00518799 0.0050354 ... 0.11651611 0.11935425 0.1164856 ]
|
||||
mono.shape (77824,)
|
||||
mono.dtype float64
|
||||
downsampled [0.00329925 0.00577342 0.00474898 ... 0.0851728 0.1166483 0. ]
|
||||
downsampled.shape (28236,)
|
||||
downsampled.dtype float64
|
||||
|
||||
1) Data should be float64
|
||||
2) Output is BYTES!! Not floats.
|
||||
3) Result is still somehow mono!?
|
||||
--------------------------------------------------
|
||||
|
||||
demo_server.py
|
||||
|
||||
capture sample rate = 44100
|
||||
downsample for algorithm = 16000
|
||||
upsample after done = 48000
|
||||
|
||||
"""
|
||||
|
||||
audio = np.array(audio, dtype=np.float64)
|
||||
print('audio.shape', audio.shape)
|
||||
print('audio.type', audio.dtype)
|
||||
|
||||
#source_rate = 88000 # Experimentally determined for Rust library 'CPAL'
|
||||
#source_rate = 44100
|
||||
|
||||
if vocode_params.original_source_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_original_source.wav'.format(request_batch_number)
|
||||
print('----- Original wav file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.original_source_rate, audio)
|
||||
|
||||
# NB: Convert the input stereo signal into mono.
|
||||
# In the future the frontend should be responsible for sampling details.
|
||||
#audio = audio[:, 0]
|
||||
|
||||
if vocode_params.pre_convert_resample:
|
||||
print("Resampling audio from {} Hz to {} Hz".format(vocode_params.original_source_rate,
|
||||
vocode_params.pre_convert_resample_rate))
|
||||
audio = librosa.resample(audio, vocode_params.original_source_rate, vocode_params.pre_convert_resample_rate)
|
||||
print('resampled_audio.shape', audio.shape)
|
||||
print('resampled_audio.type', audio.dtype)
|
||||
if vocode_params.pre_convert_resample_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_pre_convert_resample.wav'.format(request_batch_number)
|
||||
print('----- Pre-downsampled file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.pre_convert_resample_rate, audio)
|
||||
|
||||
if skip_vocode:
|
||||
return audio
|
||||
|
||||
results = converter.convert_partial(audio,
|
||||
conversion_direction='A2B',
|
||||
model_sampling_rate=vocode_params.model_hyperparameter_sampling_rate)
|
||||
|
||||
if vocode_params.model_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_model_output.wav'.format(request_batch_number)
|
||||
print('----- Model file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.model_hyperparameter_sampling_rate, results)
|
||||
|
||||
print('results.type', type(results))
|
||||
print('results.len', len(results))
|
||||
|
||||
if vocode_params.post_convert_resample:
|
||||
print("Resampling output audio from {} Hz to {} Hz".format(vocode_params.model_hyperparameter_sampling_rate,
|
||||
vocode_params.post_convert_resample_rate))
|
||||
#consume_rate = 68000 # Experimentally determined for Rust lib 'CPAL'
|
||||
results = librosa.resample(results,
|
||||
vocode_params.model_hyperparameter_sampling_rate,
|
||||
vocode_params.post_convert_resample_rate)
|
||||
if vocode_params.post_convert_resample_save_file:
|
||||
#filename = temp_file_name('.wav')
|
||||
filename = 'debug/{}_post_convert_resample.wav'.format(request_batch_number)
|
||||
print('----- Upsampled (transformed) file out: {}'.format(filename))
|
||||
scipy.io.wavfile.write(filename, vocode_params.post_convert_resample_rate, results)
|
||||
|
||||
if discard_vocoded_audio:
|
||||
return audio
|
||||
else:
|
||||
return results
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--port', type=int, default=5555)
|
||||
args = parser.parse_args()
|
||||
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.REP)
|
||||
socket.bind("tcp://*:{}".format(args.port))
|
||||
print('Running server...')
|
||||
|
||||
queue = []
|
||||
while True:
|
||||
# Wait for next request from client
|
||||
message = socket.recv()
|
||||
|
||||
vocode_request = VocodeAudioRequest.FromString(message)
|
||||
vocode_params = vocode_request.vocode_params
|
||||
|
||||
queue.extend(vocode_request.float_audio)
|
||||
|
||||
if len(queue) >= vocode_request.buffer_size_minimum:
|
||||
#results = queue[:]
|
||||
results = convert(queue,
|
||||
vocode_params=vocode_params,
|
||||
request_batch_number=vocode_request.request_batch_number,
|
||||
skip_vocode=vocode_request.skip_vocode,
|
||||
discard_vocoded_audio=vocode_request.discard_vocoded_audio)
|
||||
queue = []
|
||||
|
||||
vocode_response = VocodeAudioResponse()
|
||||
vocode_response.float_audio[:] = results
|
||||
|
||||
socket.send(vocode_response.SerializeToString())
|
||||
else:
|
||||
# Must send reply back to client
|
||||
socket.send(b"OK")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import tensorflow as tf
|
||||
import os
|
||||
import random
|
||||
import numpy as np
|
||||
|
||||
def l1_loss(y, y_hat):
|
||||
|
||||
return tf.reduce_mean(tf.abs(y - y_hat))
|
||||
|
||||
def l2_loss(y, y_hat):
|
||||
|
||||
return tf.reduce_mean(tf.square(y - y_hat))
|
||||
|
||||
def cross_entropy_loss(logits, labels):
|
||||
return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = logits, labels = labels))
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
syntax = "proto3";
|
||||
package voder_audio;
|
||||
|
||||
message VocodeAudioRequest {
|
||||
repeated float float_audio = 1;
|
||||
|
||||
// An autoincrement sent with each request.
|
||||
int64 request_batch_number = 2;
|
||||
|
||||
message VocodeParams {
|
||||
// The initial sample rate coming from the microphone.
|
||||
int32 initial_sample_rate = 1;
|
||||
|
||||
// Original sample
|
||||
int32 original_source_rate = 9;
|
||||
bool original_source_save_file = 10;
|
||||
|
||||
// Resample before passing to the algorithm?
|
||||
bool pre_convert_resample = 2;
|
||||
int32 pre_convert_resample_rate = 3;
|
||||
bool pre_convert_resample_save_file = 4;
|
||||
|
||||
// The hard-coded hyperparameter of the model.
|
||||
// Probably don't want to change from 16000.
|
||||
int32 model_hyperparameter_sampling_rate = 5;
|
||||
bool model_save_file = 11;
|
||||
|
||||
// Resample after passing to the algorithm?
|
||||
bool post_convert_resample = 6;
|
||||
int32 post_convert_resample_rate = 7;
|
||||
bool post_convert_resample_save_file = 8;
|
||||
}
|
||||
|
||||
VocodeParams vocode_params = 10;
|
||||
|
||||
// We'll resend these params for every batch, but will only use
|
||||
// the last batch's settings. Kind of a dumb API, but it'll work.
|
||||
//int32 sample_rate = 2;
|
||||
|
||||
//bool skip_resample = 3;
|
||||
bool skip_vocode = 4;
|
||||
|
||||
// Debugging
|
||||
//bool save_files = 5;
|
||||
|
||||
// How big we let the buffer grow before running 'convert'.
|
||||
int32 buffer_size_minimum = 6;
|
||||
|
||||
// Requested output rate
|
||||
//int32 output_rate = 7;
|
||||
|
||||
// Discard the vocoded audio and return the original
|
||||
bool discard_vocoded_audio = 8;
|
||||
|
||||
// Rate param for the model
|
||||
//int32 model_sampling_rate = 9;
|
||||
}
|
||||
|
||||
message VocodeAudioResponse {
|
||||
repeated float float_audio = 1;
|
||||
|
||||
// A sidecar-side autoincrement sent with each request.
|
||||
int64 response_batch_number = 2;
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
use cpal::traits::{DeviceTrait, EventLoopTrait, HostTrait};
|
||||
use cpal;
|
||||
use audio_queue::AudioQueue;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// CPAL Audio Device
|
||||
/// Access audio input and output streams.
|
||||
pub struct AudioHardware {
|
||||
host: cpal::Host,
|
||||
event_loop: cpal::EventLoop,
|
||||
microphone_queue: Arc<AudioQueue>,
|
||||
speaker_queue: Arc<AudioQueue>,
|
||||
}
|
||||
|
||||
impl AudioHardware {
|
||||
pub fn new(microphone_queue: Arc<AudioQueue>, speaker_queue: Arc<AudioQueue>) -> Result<Self, failure::Error> {
|
||||
let host = cpal::default_host();
|
||||
let event_loop = host.event_loop();
|
||||
|
||||
let input_device = host.default_input_device().expect("failed to get default input device");
|
||||
let output_device = host.default_output_device().expect("failed to get default output device");
|
||||
|
||||
println!("Using default input device: \"{}\"", input_device.name()?);
|
||||
println!("Using default output device: \"{}\"", output_device.name()?);
|
||||
|
||||
// We'll try and use the same format between streams to keep it simple
|
||||
let mut format = input_device.default_input_format()?;
|
||||
format.data_type = cpal::SampleFormat::F32;
|
||||
|
||||
// Build streams.
|
||||
println!("Attempting to build both streams with `{:?}`.", format);
|
||||
let input_stream_id = event_loop.build_input_stream(&input_device, &format)?;
|
||||
let output_stream_id = event_loop.build_output_stream(&output_device, &format)?;
|
||||
println!("Successfully built streams.");
|
||||
|
||||
Ok(AudioHardware {
|
||||
host,
|
||||
event_loop,
|
||||
microphone_queue,
|
||||
speaker_queue,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn run(&mut self) {
|
||||
self.event_loop.run(|id, result| {
|
||||
let data = match result {
|
||||
Ok(data) => data,
|
||||
Err(err) => {
|
||||
eprintln!("An error occurred on stream {:?}: {}", id, err);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
match data {
|
||||
cpal::StreamData::Input { buffer: cpal::UnknownTypeInputBuffer::F32(buffer) } => {
|
||||
//assert_eq!(id, input_stream_id);
|
||||
let mut output_fell_behind = false;
|
||||
for &sample in buffer.iter() {
|
||||
self.microphone_queue.push_back(sample);
|
||||
}
|
||||
},
|
||||
cpal::StreamData::Output { buffer: cpal::UnknownTypeOutputBuffer::F32(mut buffer) } => {
|
||||
//println!("Audio out buffer len: {}", self.speaker_queue.len());
|
||||
let request_size = buffer.len();
|
||||
let mut drained = self.speaker_queue.drain_size((request_size));
|
||||
match drained {
|
||||
None => {
|
||||
for sample in buffer.iter_mut() {
|
||||
*sample = 0.0;
|
||||
}
|
||||
},
|
||||
Some(mut drained) => {
|
||||
for (i, sample) in buffer.iter_mut().enumerate() {
|
||||
*sample = drained.get(i).copied().unwrap();
|
||||
//*sample = 0.0;
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
_ => panic!("We're expecting f32 data."),
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
use zmq;
|
||||
use zmq::{Error, Socket};
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::{RwLock, PoisonError};
|
||||
|
||||
pub struct AudioQueue {
|
||||
queue: RwLock<VecDeque<f32>>,
|
||||
}
|
||||
|
||||
impl AudioQueue {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
queue: RwLock::new(VecDeque::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn push_back(&self, val: f32) {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
queue.push_back(val);
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (push_back)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn extend(&self, buf: Vec<f32>) {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
queue.extend(buf);
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (extend)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn drain(&self) -> Vec<f32> {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
return queue.drain(..)
|
||||
.collect::<Vec<_>>();
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (drain)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn drain_size(&self, size: usize) -> Option<Vec<f32>> {
|
||||
match self.queue.write() {
|
||||
Ok(mut queue) => {
|
||||
if queue.len() < size {
|
||||
return None;
|
||||
}
|
||||
let vec = queue.drain(0..size)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
return Some(vec);
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (drain_size)");
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
match self.queue.read() {
|
||||
Ok(mut queue) => {
|
||||
return queue.len();
|
||||
},
|
||||
Err(_) => {
|
||||
unreachable!("This shouldn't happen (len)");
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
//!
|
||||
//! This is a process that captures microphone input, ships proto-over-zeromq to a python
|
||||
//! CycleGAN-VC (tensorflow) sidecar server, collects the results, then plays them over
|
||||
//! the speaker.
|
||||
//!
|
||||
|
||||
extern crate bytes;
|
||||
extern crate byteorder;
|
||||
extern crate cpal;
|
||||
extern crate failure;
|
||||
extern crate prost;
|
||||
extern crate prost_types;
|
||||
extern crate wavy;
|
||||
extern crate world_sys;
|
||||
extern crate zmq;
|
||||
|
||||
//include!(concat!(env!("OUT_DIR"), "/voder.audio.rs"));
|
||||
|
||||
pub mod audio_hardware;
|
||||
pub mod audio_queue;
|
||||
//pub mod model;
|
||||
pub mod protos;
|
||||
pub mod sidecar;
|
||||
pub mod synthesis;
|
||||
|
||||
use audio_hardware::AudioHardware;
|
||||
use protos::voder_audio::VocodeAudioRequest;
|
||||
use protos::voder_audio::VocodeAudioResponse;
|
||||
use sidecar::Sidecar;
|
||||
|
||||
use wavy::*;
|
||||
|
||||
use byteorder::{ByteOrder, BigEndian, LittleEndian, ReadBytesExt};
|
||||
use failure::_core::time::Duration;
|
||||
use audio_queue::AudioQueue;
|
||||
use prost::Message;
|
||||
//use model::load_model;
|
||||
//use model::print_version;
|
||||
use std::collections::VecDeque;
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Cursor};
|
||||
use std::path::Path;
|
||||
use std::process::exit;
|
||||
use std::sync::Arc;
|
||||
use std::thread;
|
||||
use zmq::{Error, Socket};
|
||||
use protos::voder_audio::vocode_audio_request::VocodeParams;
|
||||
|
||||
fn main() {
|
||||
//print_version();
|
||||
//load_model(); // TODO: This works. Temporarily commented out
|
||||
run_audio().expect("Should work");
|
||||
}
|
||||
|
||||
const LATENCY_MS: f32 = 50.0;
|
||||
|
||||
fn run_audio() -> Result<(), failure::Error> {
|
||||
let mut microphone_queue = Arc::new(AudioQueue::new());
|
||||
let mut microphone_queue_2 = microphone_queue.clone();
|
||||
|
||||
let mut post_process_queue = Arc::new(AudioQueue::new());
|
||||
let mut post_process_queue_2 = post_process_queue.clone();
|
||||
let mut sidecar = Sidecar::new("tcp://127.0.0.1:5555", microphone_queue_2, post_process_queue);
|
||||
|
||||
thread::spawn(move || {
|
||||
sidecar.run();
|
||||
});
|
||||
|
||||
let mut audio_hardware = AudioHardware::new(microphone_queue, post_process_queue_2).expect("");
|
||||
audio_hardware.run();
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod voder_audio;
|
||||
@@ -0,0 +1,70 @@
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct VocodeAudioRequest {
|
||||
#[prost(float, repeated, tag="1")]
|
||||
pub float_audio: ::std::vec::Vec<f32>,
|
||||
/// An autoincrement sent with each request.
|
||||
#[prost(int64, tag="2")]
|
||||
pub request_batch_number: i64,
|
||||
#[prost(message, optional, tag="10")]
|
||||
pub vocode_params: ::std::option::Option<vocode_audio_request::VocodeParams>,
|
||||
// We'll resend these params for every batch, but will only use
|
||||
// the last batch's settings. Kind of a dumb API, but it'll work.
|
||||
//int32 sample_rate = 2;
|
||||
|
||||
///bool skip_resample = 3;
|
||||
#[prost(bool, tag="4")]
|
||||
pub skip_vocode: bool,
|
||||
// Debugging
|
||||
//bool save_files = 5;
|
||||
|
||||
/// How big we let the buffer grow before running 'convert'.
|
||||
#[prost(int32, tag="6")]
|
||||
pub buffer_size_minimum: i32,
|
||||
// Requested output rate
|
||||
//int32 output_rate = 7;
|
||||
|
||||
/// Discard the vocoded audio and return the original
|
||||
#[prost(bool, tag="8")]
|
||||
pub discard_vocoded_audio: bool,
|
||||
}
|
||||
pub mod vocode_audio_request {
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct VocodeParams {
|
||||
/// The initial sample rate coming from the microphone.
|
||||
#[prost(int32, tag="1")]
|
||||
pub initial_sample_rate: i32,
|
||||
/// Original sample
|
||||
#[prost(int32, tag="9")]
|
||||
pub original_source_rate: i32,
|
||||
#[prost(bool, tag="10")]
|
||||
pub original_source_save_file: bool,
|
||||
/// Resample before passing to the algorithm?
|
||||
#[prost(bool, tag="2")]
|
||||
pub pre_convert_resample: bool,
|
||||
#[prost(int32, tag="3")]
|
||||
pub pre_convert_resample_rate: i32,
|
||||
#[prost(bool, tag="4")]
|
||||
pub pre_convert_resample_save_file: bool,
|
||||
/// The hard-coded hyperparameter of the model.
|
||||
/// Probably don't want to change from 16000.
|
||||
#[prost(int32, tag="5")]
|
||||
pub model_hyperparameter_sampling_rate: i32,
|
||||
#[prost(bool, tag="11")]
|
||||
pub model_save_file: bool,
|
||||
/// Resample after passing to the algorithm?
|
||||
#[prost(bool, tag="6")]
|
||||
pub post_convert_resample: bool,
|
||||
#[prost(int32, tag="7")]
|
||||
pub post_convert_resample_rate: i32,
|
||||
#[prost(bool, tag="8")]
|
||||
pub post_convert_resample_save_file: bool,
|
||||
}
|
||||
}
|
||||
#[derive(Clone, PartialEq, ::prost::Message)]
|
||||
pub struct VocodeAudioResponse {
|
||||
#[prost(float, repeated, tag="1")]
|
||||
pub float_audio: ::std::vec::Vec<f32>,
|
||||
/// A sidecar-side autoincrement sent with each request.
|
||||
#[prost(int64, tag="2")]
|
||||
pub response_batch_number: i64,
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
use audio_queue::AudioQueue;
|
||||
use prost::Message;
|
||||
use protos::voder_audio::VocodeAudioRequest;
|
||||
use protos::voder_audio::VocodeAudioResponse;
|
||||
use protos::voder_audio::vocode_audio_request::VocodeParams;
|
||||
use std::collections::VecDeque;
|
||||
use std::sync::Arc;
|
||||
use zmq::{Error, Socket};
|
||||
use zmq;
|
||||
|
||||
/// Talks a custom protobuf over ZeroMQ protocol to a Python TensorFlow sidecar process
|
||||
/// that evaluates the model and ships back audio byte results.
|
||||
pub struct Sidecar {
|
||||
socket_endpoint: String,
|
||||
microphone_queue: Arc<AudioQueue>,
|
||||
processed_queue: Arc<AudioQueue>,
|
||||
}
|
||||
|
||||
impl Sidecar {
|
||||
pub fn new(socket_endpoint: &str,
|
||||
microphone_queue: Arc<AudioQueue>,
|
||||
processed_queue: Arc<AudioQueue>) -> Self {
|
||||
Self {
|
||||
socket_endpoint: socket_endpoint.into(),
|
||||
microphone_queue,
|
||||
processed_queue,
|
||||
}
|
||||
}
|
||||
|
||||
/// Heuristic to determine if a speaker is speaking.
|
||||
/// Basically just checks audio loudness.
|
||||
fn is_speaking(audio: &Vec<f32>) -> bool {
|
||||
let mut avg_pos = 0.0f32;
|
||||
let mut num_pos = 0;
|
||||
|
||||
for x in audio {
|
||||
if x > &0.0f32 {
|
||||
avg_pos += x;
|
||||
num_pos += 1;
|
||||
}
|
||||
}
|
||||
|
||||
avg_pos = avg_pos / num_pos as f32;
|
||||
avg_pos > 0.006f32
|
||||
}
|
||||
|
||||
pub fn run(&mut self) {
|
||||
let mut context = zmq::Context::new();
|
||||
let mut socket = context.socket(zmq::REQ).unwrap();
|
||||
|
||||
socket.connect(&self.socket_endpoint).unwrap();
|
||||
|
||||
let mut reconnect = false;
|
||||
let mut fail_count = 0;
|
||||
|
||||
const SEND_SIZE : usize = 5000;
|
||||
|
||||
let mut request_batch_number = 0i64;
|
||||
|
||||
let mut activated = false;
|
||||
let mut frames_activated = 0;
|
||||
let mut frames_deactivated = 0;
|
||||
|
||||
let mut ring_buffer = VecDeque::with_capacity(5);
|
||||
|
||||
loop {
|
||||
let mut drained = match self.microphone_queue.drain_size(SEND_SIZE) {
|
||||
None => { continue; },
|
||||
Some(d) => d,
|
||||
};
|
||||
|
||||
/*
|
||||
Heuristic for audio. Don't record when silent (using threshold).
|
||||
|
||||
mic: act act act dec act act act act dec dec dec dec dec act dec
|
||||
sent: | ok ok ok ok ok ok ok ok ok ok ok|
|
||||
*/
|
||||
|
||||
|
||||
// NB: We don't want to lose audio right at the activating edge.
|
||||
// Save some bounded history to replay once activated.
|
||||
ring_buffer.push_back(drained.clone());
|
||||
if ring_buffer.len() > 5 {
|
||||
ring_buffer.pop_front();
|
||||
}
|
||||
|
||||
if activated {
|
||||
if !Self::is_speaking(&drained) {
|
||||
frames_deactivated += 1;
|
||||
} else {
|
||||
frames_deactivated = 0;
|
||||
}
|
||||
|
||||
if frames_deactivated > 20 {
|
||||
println!("Deactivating edge");
|
||||
activated = false;
|
||||
frames_activated = 0;
|
||||
frames_deactivated = 0;
|
||||
}
|
||||
|
||||
} else {
|
||||
if Self::is_speaking(&drained) {
|
||||
frames_activated += 1;
|
||||
} else {
|
||||
frames_activated = 0;
|
||||
}
|
||||
|
||||
if frames_activated > 3 {
|
||||
println!("Activating edge");
|
||||
activated = true;
|
||||
frames_activated = 0;
|
||||
frames_deactivated = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if !activated {
|
||||
continue;
|
||||
}
|
||||
|
||||
let drained = ring_buffer.drain(..)
|
||||
.flat_map(|sample| sample)
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
request_batch_number += 1;
|
||||
|
||||
let mut vocode_request = VocodeAudioRequest::default();
|
||||
//vocode_request.sample_rate = 16000;
|
||||
//vocode_request.output_rate = 16000;
|
||||
//vocode_request.model_sampling_rate = 88000; // This is close!
|
||||
//vocode_request.model_sampling_rate = 88000;
|
||||
//vocode_request.skip_resample = false;
|
||||
vocode_request.skip_vocode = false;
|
||||
vocode_request.discard_vocoded_audio = false;
|
||||
//vocode_request.buffer_size_minimum = 5000; // AWFUL. SO CHOPPY.
|
||||
//vocode_request.buffer_size_minimum = 50000; // Practically real time, but lots more phase distortion.
|
||||
//vocode_request.buffer_size_minimum = 30000; // Hm, hmm... not bad
|
||||
vocode_request.buffer_size_minimum = 70000; // Sounds pretty good.
|
||||
//vocode_request.buffer_size_minimum = 100000; // This sounds good! A bit slow ~3seconds.
|
||||
//vocode_request.buffer_size_minimum = 200000;
|
||||
vocode_request.float_audio = drained.clone();
|
||||
vocode_request.request_batch_number = request_batch_number;
|
||||
|
||||
let mut vocode_params = VocodeParams::default();
|
||||
vocode_params.original_source_rate = 88000; // This is correct for saving
|
||||
|
||||
vocode_params.pre_convert_resample = true;
|
||||
vocode_params.pre_convert_resample_rate = 16000;
|
||||
|
||||
vocode_params.model_hyperparameter_sampling_rate = 16000;
|
||||
|
||||
vocode_params.post_convert_resample = true;
|
||||
vocode_params.post_convert_resample_rate = 88000;
|
||||
|
||||
vocode_params.original_source_save_file = false;
|
||||
vocode_params.pre_convert_resample_save_file = false;
|
||||
vocode_params.model_save_file = false;
|
||||
vocode_params.post_convert_resample_save_file = false;
|
||||
|
||||
vocode_request.vocode_params = Some(vocode_params);
|
||||
|
||||
let mut encoded_bytes = Vec::with_capacity(vocode_request.encoded_len());
|
||||
vocode_request.encode(&mut encoded_bytes).unwrap();
|
||||
|
||||
if reconnect {
|
||||
reconnect = false;
|
||||
println!("RECONNECT");
|
||||
|
||||
//thread::sleep(Duration::from_millis(200));
|
||||
|
||||
socket = match context.socket(zmq::REQ) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
println!("Error creating socket: {:?}", e);
|
||||
continue
|
||||
},
|
||||
};
|
||||
|
||||
match socket.connect("tcp://127.0.0.1:5555") {
|
||||
Ok(_) => {},
|
||||
Err(err) => {
|
||||
println!("Err B: {:?}", err);
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
match socket.send(&encoded_bytes, 0) {
|
||||
Ok(_) => {
|
||||
//println!("Sent len: {}", bytes.len());
|
||||
},
|
||||
Err(e) => {
|
||||
println!("send err {}: {:?}", e.to_raw(), e);
|
||||
fail_count += 1;
|
||||
},
|
||||
}
|
||||
|
||||
match socket.recv_bytes(0) {
|
||||
Ok(buf) => {
|
||||
//println!("---> Buf len: {}", buf.len());
|
||||
if buf.len() > 2 {
|
||||
// Receive data condition.
|
||||
let vocode_response = VocodeAudioResponse::decode(buf).unwrap();
|
||||
self.processed_queue.extend(vocode_response.float_audio);
|
||||
}
|
||||
},
|
||||
Err(e) => {
|
||||
fail_count += 1;
|
||||
println!("recv err: {:?}", e);
|
||||
},
|
||||
}
|
||||
|
||||
if fail_count > 5 {
|
||||
fail_count = 0;
|
||||
reconnect = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
use world_sys::core::synthesize::SynthesizeError;
|
||||
use world_sys::core::synthesize::SynthesizeResult;
|
||||
use world_sys::core::synthesize::synthesize;
|
||||
|
||||
use world_sys::core::spectral::code_spectral_envelope;
|
||||
|
||||
/*
|
||||
TODO: ALGORITHM TO GET VOICE TRANSFORMATION:
|
||||
|
||||
- [mono]
|
||||
- [librosa.resample (downsample)]
|
||||
- core convert()
|
||||
- wav_padding()
|
||||
- [maths]
|
||||
- [np.pad]
|
||||
- world_decompose()
|
||||
- pyworld.harvest
|
||||
- pyworld.cheaptrick
|
||||
- pyworld.d4c
|
||||
- world_encode_spectral_envelope()
|
||||
- pyworld.code_spectral_envelope
|
||||
- [matrix transpose]
|
||||
- pitch_conversion()
|
||||
- [maths]
|
||||
- [maths : coded_sp_norm]
|
||||
- << evaluate ML model >>
|
||||
- [maths : coded_sp_converted]
|
||||
- [matrix transpose]
|
||||
- [np.ascontiguousarray]
|
||||
- world_decode_spectral_envelop
|
||||
- pyworld.get_cheaptrick_fft_size
|
||||
- pyworld.decode_spectral_envelope
|
||||
- world_speech_synthesis
|
||||
- pyworld.synthesize
|
||||
- [librosa.resample (upsample)]
|
||||
- [wav things]
|
||||
*/
|
||||
|
||||
Reference in New Issue
Block a user