From 4a222ad81553153e4f5aceb97fb9d046c16ef87a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Sep 2020 09:06:48 -0400 Subject: [PATCH] realtime-voice-conversion based on last year's work This is based on the work I did in (July - October)ish 2019 to get real time voice conversion working. I've removed it from the `voder` library and made it into an isolable program. I'll work on extending the capabilities. --- .gitignore | 23 + Cargo.lock | 897 ++++++++++++++++++++++++++++++++++ Cargo.toml | 23 + README.md | 34 ++ build.rs | 11 + cycle_gan/demo_server.py | 278 +++++++++++ cycle_gan/model.py | 183 +++++++ cycle_gan/module.py | 214 ++++++++ cycle_gan/preprocess.py | 238 +++++++++ cycle_gan/protos/audio_pb2.py | 260 ++++++++++ cycle_gan/requirements.txt | 36 ++ cycle_gan/sidecar.py | 264 ++++++++++ cycle_gan/utils.py | 17 + protos/audio.proto | 64 +++ src/audio_hardware.rs | 84 ++++ src/audio_queue.rs | 78 +++ src/main.rs | 72 +++ src/protos/mod.rs | 1 + src/protos/voder_audio.rs | 70 +++ src/sidecar.rs | 217 ++++++++ src/synthesis.rs | 38 ++ 21 files changed, 3102 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.lock create mode 100644 Cargo.toml create mode 100644 README.md create mode 100644 build.rs create mode 100755 cycle_gan/demo_server.py create mode 100644 cycle_gan/model.py create mode 100644 cycle_gan/module.py create mode 100644 cycle_gan/preprocess.py create mode 100644 cycle_gan/protos/audio_pb2.py create mode 100644 cycle_gan/requirements.txt create mode 100755 cycle_gan/sidecar.py create mode 100644 cycle_gan/utils.py create mode 100644 protos/audio.proto create mode 100644 src/audio_hardware.rs create mode 100644 src/audio_queue.rs create mode 100644 src/main.rs create mode 100644 src/protos/mod.rs create mode 100644 src/protos/voder_audio.rs create mode 100644 src/sidecar.rs create mode 100644 src/synthesis.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bc6f3de --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +*~ +__pycache__/ +node_modules/ +target/ + +# IntelliJ +.idea/ +voder.iml +realtime-voce-conversion.iml + +# Project specific ignores +cycle_gan/python +tacotron_melgan/python +voice_converter_sidecar/cycle_gan/python + +# Secrets +secrets.yaml + +# Models and data +*.jit +*.pt +*.torchjit +*.wav diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..700cfd3 --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,897 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "addr2line" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6a2d3371669ab3ca9797670853d61402b03d0b4b9ebf33d677dfa720203072" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "aho-corasick" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043164d8ba5c4c3035fec9bbee8647c0261d788f3474306f93bb65901cae0e86" +dependencies = [ + "memchr", +] + +[[package]] +name = "alsa-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0edcbbf9ef68f15ae1b620f722180b82a98b6f0628d30baa6b8d2a5abc87d58" +dependencies = [ + "libc", + "pkg-config", +] + +[[package]] +name = "ansi_term" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" +dependencies = [ + "winapi", +] + +[[package]] +name = "atty" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" +dependencies = [ + "hermit-abi", + "libc", + "winapi", +] + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "backtrace" +version = "0.3.50" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46254cf2fdcdf1badb5934448c1bcbe046a56537b3987d96c51a7afc5d03f293" +dependencies = [ + "addr2line", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "bindgen" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebd71393f1ec0509b553aa012b9b58e81dadbdff7130bd3b8cba576e69b32f75" +dependencies = [ + "bitflags", + "cexpr", + "cfg-if", + "clang-sys", + "clap", + "env_logger", + "lazy_static", + "log", + "peeking_take_while", + "proc-macro2 1.0.21", + "quote 1.0.7", + "regex", + "rustc-hash", + "shlex", + "which 3.1.1", +] + +[[package]] +name = "bindgen" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99de13bb6361e01e493b3db7928085dcc474b7ba4f5481818e53a89d76b8393f" +dependencies = [ + "bitflags", + "cexpr", + "cfg-if", + "clang-sys", + "lazy_static", + "lazycell", + "peeking_take_while", + "proc-macro2 1.0.21", + "quote 1.0.7", + "regex", + "rustc-hash", + "shlex", +] + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "cc" +version = "1.0.59" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66120af515773fb005778dc07c261bd201ec8ce50bd6e7144c927753fe013381" + +[[package]] +name = "cexpr" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce5b5fb86b0c57c20c834c1b412fd09c77c8a59b9473f86272709e78874cd1d" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "clang-sys" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81de550971c976f176130da4b2978d3b524eaa0fd9ac31f3ceb5ae1231fb4853" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "clap" +version = "2.33.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37e58ac78573c40708d45522f0d80fa2f01cc4f9b4e2bf749807255454312002" +dependencies = [ + "ansi_term", + "atty", + "bitflags", + "strsim", + "textwrap", + "unicode-width", + "vec_map", +] + +[[package]] +name = "core-foundation-sys" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" + +[[package]] +name = "coreaudio-rs" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f229761965dad3e9b11081668a6ea00f1def7aa46062321b5ec245b834f6e491" +dependencies = [ + "bitflags", + "coreaudio-sys", +] + +[[package]] +name = "coreaudio-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6570ee6e089131e928d5ec9236db9e818aa3cf850f48b0eec6ef700571271d4" +dependencies = [ + "bindgen 0.53.1", +] + +[[package]] +name = "cpal" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded070249be850b5b59e1e3a44a70b8ae395e0e5c65b487131d8909a8208120" +dependencies = [ + "alsa-sys", + "core-foundation-sys", + "coreaudio-rs", + "failure", + "lazy_static", + "libc", + "num-traits", + "stdweb", + "winapi", +] + +[[package]] +name = "either" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56b59865bce947ac5958779cfa508f6c3b9497cc762b7e24a12d11ccde2c4f" + +[[package]] +name = "env_logger" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcde04e90a5226a6443b7aabdb016ba2f8307c847d524724bd9b346dd1a2d3" +dependencies = [ + "atty", + "humantime", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "error-chain" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" + +[[package]] +name = "failure" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d32e9bd16cc02eae7db7ef620b392808b89f6a5e16bb3497d159c6b92a0f4f86" +dependencies = [ + "backtrace", + "failure_derive", +] + +[[package]] +name = "failure_derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa4da3c766cd7a0db8242e326e9e4e081edd567072893ed320008189715366a4" +dependencies = [ + "proc-macro2 1.0.21", + "quote 1.0.7", + "syn 1.0.40", + "synstructure", +] + +[[package]] +name = "fixedbitset" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86d4de0081402f5e88cdac65c8dcdcc73118c1a7a465e2a05f0da05843a8ea33" + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gimli" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" + +[[package]] +name = "glob" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574" + +[[package]] +name = "heck" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "hermit-abi" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3deed196b6e7f9e44a2ae8d94225d80302d81208b1bb673fd21fe634645c85a9" +dependencies = [ + "libc", +] + +[[package]] +name = "humantime" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" +dependencies = [ + "quick-error", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "itertools" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56a2d0bc861f9165be4eb3442afd3c236d8a98afd426f65d92324ae1091a484" +dependencies = [ + "either", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + +[[package]] +name = "libc" +version = "0.2.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f96b10ec2560088a8e76961b00d47107b3a625fecb76dedb29ee7ccbf98235" + +[[package]] +name = "libloading" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2b111a074963af1d37a139918ac6d49ad1d0d5e47f72fd55388619691a7d753" +dependencies = [ + "cc", + "winapi", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "memchr" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" + +[[package]] +name = "metadeps" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b122901b3a675fac8cecf68dcb2f0d3036193bc861d1ac0e1c337f7d5254c2" +dependencies = [ + "error-chain", + "pkg-config", + "toml", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d7559a8a40d0f97e1edea3220f698f78b1c5ab67532e49f68fde3910323b722" +dependencies = [ + "adler", +] + +[[package]] +name = "multimap" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eb04b9f127583ed176e163fb9ec6f3e793b87e21deedd5734a69386a18a0151" + +[[package]] +name = "nom" +version = "4.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad2a91a8e869eeb30b9cb3119ae87773a8f4ae617f41b1eb9c154b2905f7bd6" +dependencies = [ + "memchr", + "version_check", +] + +[[package]] +name = "num-traits" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac267bcc07f48ee5f8935ab0d24f316fb722d7a1292e2913f0cc196b29ffd611" +dependencies = [ + "autocfg", +] + +[[package]] +name = "object" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab52be62400ca80aa00285d25253d7f7c437b7375c4de678f5405d3afe82ca5" + +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + +[[package]] +name = "petgraph" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3659d1ee90221741f65dd128d9998311b0e40c5d3c23a62445938214abce4f" +dependencies = [ + "fixedbitset", +] + +[[package]] +name = "pkg-config" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d36492546b6af1463394d46f0c834346f31548646f6ba10849802c9c9a27ac33" + +[[package]] +name = "ppv-lite86" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" + +[[package]] +name = "proc-macro2" +version = "0.4.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d2011ab5c909338f7887f4fc896d35932e29146c12c8d01da6b22a80ba759" +dependencies = [ + "unicode-xid 0.1.0", +] + +[[package]] +name = "proc-macro2" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36e28516df94f3dd551a587da5357459d9b36d945a7c37c3557928c1c2ff2a2c" +dependencies = [ + "unicode-xid 0.2.1", +] + +[[package]] +name = "prost" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d14b1c185652833d24aaad41c5832b0be5616a590227c1fbff57c616754b23" +dependencies = [ + "byteorder", + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb788126ea840817128183f8f603dce02cb7aea25c2a0b764359d8e20010702e" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "tempfile", + "which 2.0.1", +] + +[[package]] +name = "prost-derive" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7dc378b94ac374644181a2247cebf59a6ec1c88b49ac77f3a94b86b79d0e11" +dependencies = [ + "failure", + "itertools", + "proc-macro2 0.4.30", + "quote 0.6.13", + "syn 0.15.44", +] + +[[package]] +name = "prost-types" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1de482a366941c8d56d19b650fac09ca08508f2a696119ee7513ad590c8bac6f" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + +[[package]] +name = "quote" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce23b6b870e8f94f81fb0a363d65d86675884b34a09043c81e5562f11c1f8e1" +dependencies = [ + "proc-macro2 0.4.30", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +dependencies = [ + "proc-macro2 1.0.21", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "regex" +version = "1.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3780fcf44b193bc4d09f36d2a3c87b251da4a046c87795a0d35f4f927ad8e6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", + "thread_local", +] + +[[package]] +name = "regex-syntax" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26412eb97c6b088a6997e05f69403a802a92d520de2f8e63c2b65f9e0f47c4e8" + +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "rustc-demangle" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c691c0e608126e00913e33f0ccf3727d5fc84573623b8d65b2df340b5201783" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "shlex" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" + +[[package]] +name = "stdweb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef5430c8e36b713e13b48a9f709cc21e046723fe44ce34587b73a830203b533e" + +[[package]] +name = "strsim" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" + +[[package]] +name = "syn" +version = "0.15.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ca4b3b69a77cbe1ffc9e198781b7acb0c7365a883670e8f1c1bc66fba79a5c5" +dependencies = [ + "proc-macro2 0.4.30", + "quote 0.6.13", + "unicode-xid 0.1.0", +] + +[[package]] +name = "syn" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "963f7d3cc59b59b9325165add223142bbf1df27655d07789f109896d353d8350" +dependencies = [ + "proc-macro2 1.0.21", + "quote 1.0.7", + "unicode-xid 0.2.1", +] + +[[package]] +name = "synstructure" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +dependencies = [ + "proc-macro2 1.0.21", + "quote 1.0.7", + "syn 1.0.40", + "unicode-xid 0.2.1", +] + +[[package]] +name = "tempfile" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + +[[package]] +name = "termcolor" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "textwrap" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d326610f408c7a4eb6f51c37c330e496b08506c9457c9d34287ecc38809fb060" +dependencies = [ + "unicode-width", +] + +[[package]] +name = "thread_local" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "toml" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736b60249cb25337bc196faa43ee12c705e426f3d55c214d73a4e7be06f92cb4" + +[[package]] +name = "unicode-segmentation" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83e153d1053cbb5a118eeff7fd5be06ed99153f00dbcd8ae310c5fb2b22edc0" + +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + +[[package]] +name = "unicode-xid" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "vec_map" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" + +[[package]] +name = "version_check" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "914b1a6776c4c929a602fafd8bc742e06365d4bcbe48c30f9cca5824f70dc9dd" + +[[package]] +name = "voice_conversion_sidecar" +version = "0.1.0" +dependencies = [ + "byteorder", + "bytes", + "cpal", + "failure", + "prost", + "prost-build", + "prost-types", + "wavy", + "world-sys", + "zmq", +] + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "wavy" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feea5ba5ef38798f371bc995c1ca542d5162923d1bcbdaaf89916042bb61f006" +dependencies = [ + "libc", +] + +[[package]] +name = "which" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b57acb10231b9493c8472b20cb57317d0679a49e0bdbee44b3b803a6473af164" +dependencies = [ + "failure", + "libc", +] + +[[package]] +name = "which" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d011071ae14a2f6671d0b74080ae0cd8ebf3a6f8c9589a2cd45f23126fe29724" +dependencies = [ + "libc", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" +dependencies = [ + "winapi", +] + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "world-sys" +version = "0.1.0" +dependencies = [ + "bindgen 0.51.1", + "cc", +] + +[[package]] +name = "zmq" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aad98a7a617d608cd9e1127147f630d24af07c7cd95ba1533246d96cbdd76c66" +dependencies = [ + "bitflags", + "libc", + "log", + "zmq-sys", +] + +[[package]] +name = "zmq-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d33a2c51dde24d5b451a2ed4b488266df221a5eaee2ee519933dc46b9a9b3648" +dependencies = [ + "libc", + "metadeps", +] diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..44a4c41 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "voice_conversion_sidecar" +version = "0.1.0" +authors = [ "Brandon Thomas ", + "Brandon Thomas " ] +[[bin]] +name = "voice_converter_sidecar" +path = "src/main.rs" + +[dependencies] +byteorder = "1.3.2" # byte-level operations +bytes = "0.4.12" # required by 'prost' +cpal = "0.10.0" # hardware audio out +failure = "0.1.5" # required by cpal +prost = "0.5.*" # protobuf +prost-types = "0.5.0" # protobuf +wavy = "0.1.2" # real time audio capture and playback +# TODO: Move these into the voder project workspace. +world-sys = { path = "/home/bt/dev/world-sys" } # AWESOME WORLD VOCODER BINDINGS HAH HA HAH +zmq = "0.9.2" # zeromq + +[build-dependencies] +prost-build = "0.5.0" diff --git a/README.md b/README.md new file mode 100644 index 0000000..5364863 --- /dev/null +++ b/README.md @@ -0,0 +1,34 @@ +Voice Converter Sidecar +======================= +This project spins up a Rust binary to handle audio input and output, then +ships microphone audio over proto/zeromq to a Python sidecar running +CycleGAN-VC to convert it into target speech. + +### CycleGAN +I've included a pared down copy of CycleGAN for the purposes of model +evaluation. It handles all of the audio buffering / sidecar integration +pieces. [The original CycleGAN I used is here](https://github.com/leimao/Voice_Converter_CycleGAN). + +#### Note on Training +It's possible to train and evaluate at the same time using dual GPUs +(at least my dual 1080Ti setup). To run the sidecar on a particular +GPU (0-indexded), use: + +``` +CUDA_VISIBLE_DEVICES=1 ./sidecar.py +``` + +### Proto Compilation (for Python) +Codegen for Rust is built in. Codegen for Python uses, + +``` +protoc --python_out=cycle_gan protos/audio.proto +``` + +### Current Results +Currently there is 4.39 seconds of delay between speaking and generated output +with the sidecar setup on my desktop computer. This is really great and seems +promising. + +This gets up to 6.0 seconds later. Drift continues to accrue, but it's a slow +build. diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..5c0635e --- /dev/null +++ b/build.rs @@ -0,0 +1,11 @@ +extern crate prost_build; + +fn main() { + //prost_build::compile_protos(&["protos/audio.proto"], + // &["protos/"]).unwrap(); + prost_build::Config::new() + .out_dir("src/protos/") + .compile_protos(&["protos/audio.proto"], + &["protos/"]) + .unwrap(); +} \ No newline at end of file diff --git a/cycle_gan/demo_server.py b/cycle_gan/demo_server.py new file mode 100755 index 0000000..3b767ba --- /dev/null +++ b/cycle_gan/demo_server.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 + +""" +Starts a demo HTTP server to capture and transform audio +as a live demonstration of the trained model. + +Brandon Thomas 2019-07-29 +""" + +import argparse +import falcon +import io +import librosa +import numpy as np +import os +import scipy +import soundfile +import tensorflow as tf +import subprocess +import tempfile + +from falcon_multipart.middleware import MultipartMiddleware +from model import CycleGAN +from preprocess import * +from wsgiref import simple_server + +print("TensorFlow version: {}".format(tf.version.VERSION)) + +INDEX_HTML = ''' + + + +

Voice Demo

+ + + + +
+
+ +
+
+ + + +''' + + +class Converter(): + def __init__(self, model_dir, model_name): + self.num_features = 24 + self.sampling_rate = 16000 + self.frame_period = 5.0 + + self.model = CycleGAN(num_features = self.num_features, mode = 'test') + + self.model.load(filepath = os.path.join(model_dir, model_name)) + + """ + # NB: Save the graph + definition = self.model.sess.graph_def + directory = 'saved_model_2' + tf.train.write_graph(definition, directory, 'saved_model_2.pb', as_text=True) + + # https://github.com/tensorflow/models/issues/3530#issuecomment-395968881 + output_dir = './saved_model/' + builder = tf.saved_model.builder.SavedModelBuilder(output_dir) + + builder.add_meta_graph_and_variables( + self.model.sess, + [tf.saved_model.tag_constants.SERVING], + main_op=tf.tables_initializer(), + ) + + builder.save() + """ + + """ + builder.add_meta_graph_and_variables( + self.model.sess, + [tf.saved_model.tag_constants.SERVING], + signature_def_map={ + 'predict_images': + prediction_signature, + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + classification_signature, + }, + main_op=tf.tables_initializer()) + """ + + self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz')) + self.mcep_mean_A = self.mcep_normalization_params['mean_A'] + self.mcep_std_A = self.mcep_normalization_params['std_A'] + self.mcep_mean_B = self.mcep_normalization_params['mean_B'] + self.mcep_std_B = self.mcep_normalization_params['std_B'] + + self.logf0s_normalization_params = np.load(os.path.join(model_dir, 'logf0s_normalization.npz')) + self.logf0s_mean_A = self.logf0s_normalization_params['mean_A'] + self.logf0s_std_A = self.logf0s_normalization_params['std_A'] + self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] + self.logf0s_std_B = self.logf0s_normalization_params['std_B'] + + def convert(self, wav, conversion_direction='A2B'): + wav = wav_padding(wav = wav, sr = self.sampling_rate, frame_period = self.frame_period, multiple = 4) + f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = self.sampling_rate, frame_period = self.frame_period) + coded_sp = world_encode_spectral_envelop(sp = sp, fs = self.sampling_rate, dim = self.num_features) + coded_sp_transposed = coded_sp.T + + if conversion_direction == 'A2B': + f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_A, std_log_src = self.logf0s_std_A, mean_log_target = self.logf0s_mean_B, std_log_target = self.logf0s_std_B) + coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A + coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] + coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B + else: + f0_converted = pitch_conversion(f0 = f0, mean_log_src = self.logf0s_mean_B, std_log_src = self.logf0s_std_B, mean_log_target = self.logf0s_mean_A, std_log_target = self.logf0s_std_A) + coded_sp_norm = (coded_sp_transposed - self.mcep_mean_B) / self.mcep_std_B + coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), direction = conversion_direction)[0] + coded_sp_converted = coded_sp_converted_norm * self.mcep_std_A + self.mcep_mean_A + + coded_sp_converted = coded_sp_converted.T + coded_sp_converted = np.ascontiguousarray(coded_sp_converted) + decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, fs = self.sampling_rate) + wav_transformed = world_speech_synthesis(f0 = f0_converted, decoded_sp = decoded_sp_converted, ap = ap, fs = self.sampling_rate, frame_period = self.frame_period) + + # For debugging model output, uncomment the following line: + # librosa.output.write_wav('model_output.wav', wav_transformed, self.sampling_rate) + + # TODO: Perhaps ditch this. It's probably unnecessary work. + upsampled = librosa.resample(wav_transformed, self.sampling_rate, 48000) + pcm_data = upsampled.astype(np.float64) + stereo_pcm_data = np.tile(pcm_data, (2,1)).T + + buf = io.BytesIO() + scipy.io.wavfile.write(buf, 48000, stereo_pcm_data.astype(np.float32)) + return buf + +# Set up model +# This should live long in memory, so we do it up front. +model_dir_default = './model/sf1_tm1' +model_name_default = 'sf1_tm1.ckpt' +converter = Converter(model_dir_default, model_name_default) + +class IndexHandler(): + def on_get(self, request, response): + response.content_type = 'text/html' + response.body = INDEX_HTML + +class ApiHandler(): + def on_post(self, request, response): + # NB: uses middleware to pull out data. + form_data = request.params['audio_data'].file + data, samplerate = soundfile.read(form_data) + + print('samplerate', samplerate) + print('data', data) + print('data.shape', data.shape) + print('data.dtype', data.dtype) + + # For debugging browser input, uncomment the following line: + # scipy.io.wavfile.write('browser_input_audio.wav', samplerate, data) + + # NB: Convert the input stereo signal into mono. + # In the future the frontend should be responsible for sampling details. + mono = data[:, 0] + print('mono', mono) + print('mono.shape', mono.shape) + print('mono.dtype', mono.dtype) + + # NB: We must downsample to the rate that the network is trained on. + downsampled = librosa.resample(mono, samplerate, 16000) + print('downsampled', downsampled) + print('downsampled.shape', downsampled.shape) + print('downsampled.dtype', downsampled.dtype) + + # Evaluate the model + print(">>> Converting...") + results = converter.convert(downsampled, conversion_direction = 'A2B') + + print('type(results)', type(results)) + + temp_dir = tempfile.TemporaryDirectory(prefix='tmp_ml_audio') + temp_file = tempfile.NamedTemporaryFile(suffix='.wav') + + temp_file.write(results.read()) + + out_file = temp_dir.name + '/output.ogg' + + # NB: Browsers have a great deal of trouble decoding WAV files unless they are in the + # narrow slice of the WAV spec expected. None of the {librosa, scipy, soundfile} python + # tools do a good job of this, so here we shell out to ffmpeg and generate OGG. + # It's lazy and messy, but it works for now. + # See https://github.com/librosa/librosa/issues/361 for a survey of the library landscape + # See https://bugzilla.mozilla.org/show_bug.cgi?id=523837 for one of dozens of browser codec bugs + _stdout = subprocess.check_output(['ffmpeg', '-i', temp_file.name, '-acodec', 'libvorbis', out_file]) + + response.content_type = 'audio/ogg' + with open(out_file, mode='rb') as f: + response.data = f.read() + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--port', type=int, default=8000) + args = parser.parse_args() + + api = falcon.API(middleware=[MultipartMiddleware()]) + api.add_route('/', IndexHandler()) + api.add_route('/upload', ApiHandler()) + api.add_static_route('/script', os.path.abspath('./script')) + api.add_static_route('/sound', os.path.abspath('./sound')) + print('Serving on 0.0.0.0:%d' % args.port) + simple_server.make_server('0.0.0.0', args.port, api).serve_forever() + +if __name__ == '__main__': + main() + diff --git a/cycle_gan/model.py b/cycle_gan/model.py new file mode 100644 index 0000000..cf7a32c --- /dev/null +++ b/cycle_gan/model.py @@ -0,0 +1,183 @@ +import os +import tensorflow as tf +from module import discriminator, generator_gatedcnn +from utils import l1_loss, l2_loss, cross_entropy_loss +from datetime import datetime + +class CycleGAN(object): + + def __init__(self, num_features, discriminator = discriminator, generator = generator_gatedcnn, mode = 'train', log_dir = './log'): + + self.num_features = num_features + self.input_shape = [None, num_features, None] # [batch_size, num_features, num_frames] + + self.discriminator = discriminator + self.generator = generator + self.mode = mode + + self.build_model() + self.optimizer_initializer() + + self.saver = tf.train.Saver() + + #config = tf.ConfigProto(device_count = {'GPU': 1}) + #self.sess = tf.Session(config=config) + self.sess = tf.Session() + self.sess.run(tf.global_variables_initializer()) + + if self.mode == 'train': + self.train_step = 0 + now = datetime.now() + self.log_dir = os.path.join(log_dir, now.strftime('%Y%m%d-%H%M%S')) + self.writer = tf.summary.FileWriter(self.log_dir, tf.get_default_graph()) + self.generator_summaries, self.discriminator_summaries = self.summary() + + def build_model(self): + + # Placeholders for real training samples + self.input_A_real = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_real') + self.input_B_real = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_real') + # Placeholders for fake generated samples + self.input_A_fake = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_fake') + self.input_B_fake = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_fake') + # Placeholder for test samples + self.input_A_test = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_A_test') + self.input_B_test = tf.placeholder(tf.float32, shape = self.input_shape, name = 'input_B_test') + + self.generation_B = self.generator(inputs = self.input_A_real, reuse = False, scope_name = 'generator_A2B') + self.cycle_A = self.generator(inputs = self.generation_B, reuse = False, scope_name = 'generator_B2A') + + self.generation_A = self.generator(inputs = self.input_B_real, reuse = True, scope_name = 'generator_B2A') + self.cycle_B = self.generator(inputs = self.generation_A, reuse = True, scope_name = 'generator_A2B') + + self.generation_A_identity = self.generator(inputs = self.input_A_real, reuse = True, scope_name = 'generator_B2A') + self.generation_B_identity = self.generator(inputs = self.input_B_real, reuse = True, scope_name = 'generator_A2B') + + self.discrimination_A_fake = self.discriminator(inputs = self.generation_A, reuse = False, scope_name = 'discriminator_A') + self.discrimination_B_fake = self.discriminator(inputs = self.generation_B, reuse = False, scope_name = 'discriminator_B') + + # Cycle loss + self.cycle_loss = l1_loss(y = self.input_A_real, y_hat = self.cycle_A) + l1_loss(y = self.input_B_real, y_hat = self.cycle_B) + + # Identity loss + self.identity_loss = l1_loss(y = self.input_A_real, y_hat = self.generation_A_identity) + l1_loss(y = self.input_B_real, y_hat = self.generation_B_identity) + + # Place holder for lambda_cycle and lambda_identity + self.lambda_cycle = tf.placeholder(tf.float32, None, name = 'lambda_cycle') + self.lambda_identity = tf.placeholder(tf.float32, None, name = 'lambda_identity') + + # Generator loss + # Generator wants to fool discriminator + self.generator_loss_A2B = l2_loss(y = tf.ones_like(self.discrimination_B_fake), y_hat = self.discrimination_B_fake) + self.generator_loss_B2A = l2_loss(y = tf.ones_like(self.discrimination_A_fake), y_hat = self.discrimination_A_fake) + + # Merge the two generators and the cycle loss + self.generator_loss = self.generator_loss_A2B + self.generator_loss_B2A + self.lambda_cycle * self.cycle_loss + self.lambda_identity * self.identity_loss + + # Discriminator loss + self.discrimination_input_A_real = self.discriminator(inputs = self.input_A_real, reuse = True, scope_name = 'discriminator_A') + self.discrimination_input_B_real = self.discriminator(inputs = self.input_B_real, reuse = True, scope_name = 'discriminator_B') + self.discrimination_input_A_fake = self.discriminator(inputs = self.input_A_fake, reuse = True, scope_name = 'discriminator_A') + self.discrimination_input_B_fake = self.discriminator(inputs = self.input_B_fake, reuse = True, scope_name = 'discriminator_B') + + # Discriminator wants to classify real and fake correctly + self.discriminator_loss_input_A_real = l2_loss(y = tf.ones_like(self.discrimination_input_A_real), y_hat = self.discrimination_input_A_real) + self.discriminator_loss_input_A_fake = l2_loss(y = tf.zeros_like(self.discrimination_input_A_fake), y_hat = self.discrimination_input_A_fake) + self.discriminator_loss_A = (self.discriminator_loss_input_A_real + self.discriminator_loss_input_A_fake) / 2 + + self.discriminator_loss_input_B_real = l2_loss(y = tf.ones_like(self.discrimination_input_B_real), y_hat = self.discrimination_input_B_real) + self.discriminator_loss_input_B_fake = l2_loss(y = tf.zeros_like(self.discrimination_input_B_fake), y_hat = self.discrimination_input_B_fake) + self.discriminator_loss_B = (self.discriminator_loss_input_B_real + self.discriminator_loss_input_B_fake) / 2 + + # Merge the two discriminators into one + self.discriminator_loss = self.discriminator_loss_A + self.discriminator_loss_B + + # Categorize variables because we have to optimize the two sets of the variables separately + trainable_variables = tf.trainable_variables() + self.discriminator_vars = [var for var in trainable_variables if 'discriminator' in var.name] + self.generator_vars = [var for var in trainable_variables if 'generator' in var.name] + #for var in t_vars: print(var.name) + + # Reserved for test + self.generation_B_test = self.generator(inputs = self.input_A_test, reuse = True, scope_name = 'generator_A2B') + self.generation_A_test = self.generator(inputs = self.input_B_test, reuse = True, scope_name = 'generator_B2A') + + + def optimizer_initializer(self): + + self.generator_learning_rate = tf.placeholder(tf.float32, None, name = 'generator_learning_rate') + self.discriminator_learning_rate = tf.placeholder(tf.float32, None, name = 'discriminator_learning_rate') + self.discriminator_optimizer = tf.train.AdamOptimizer(learning_rate = self.discriminator_learning_rate, beta1 = 0.5).minimize(self.discriminator_loss, var_list = self.discriminator_vars) + self.generator_optimizer = tf.train.AdamOptimizer(learning_rate = self.generator_learning_rate, beta1 = 0.5).minimize(self.generator_loss, var_list = self.generator_vars) + + def train(self, input_A, input_B, lambda_cycle, lambda_identity, generator_learning_rate, discriminator_learning_rate): + + generation_A, generation_B, generator_loss, _, generator_summaries = self.sess.run( + [self.generation_A, self.generation_B, self.generator_loss, self.generator_optimizer, self.generator_summaries], \ + feed_dict = {self.lambda_cycle: lambda_cycle, self.lambda_identity: lambda_identity, self.input_A_real: input_A, self.input_B_real: input_B, self.generator_learning_rate: generator_learning_rate}) + + self.writer.add_summary(generator_summaries, self.train_step) + + discriminator_loss, _, discriminator_summaries = self.sess.run([self.discriminator_loss, self.discriminator_optimizer, self.discriminator_summaries], \ + feed_dict = {self.input_A_real: input_A, self.input_B_real: input_B, self.discriminator_learning_rate: discriminator_learning_rate, self.input_A_fake: generation_A, self.input_B_fake: generation_B}) + + self.writer.add_summary(discriminator_summaries, self.train_step) + + self.train_step += 1 + + return generator_loss, discriminator_loss + + + def test(self, inputs, direction): + #print(">>> model.test()!") + + if direction == 'A2B': + #print("generation_B_test: {}".format(self.generation_B_test)) + #print("input_A_test: {}".format(self.input_A_test)) + generation = self.sess.run(self.generation_B_test, feed_dict = {self.input_A_test: inputs}) + elif direction == 'B2A': + #print("generation_A_test: {}".format(self.generation_A_test)) + #print("input_B_test: {}".format(self.input_B_test)) + generation = self.sess.run(self.generation_A_test, feed_dict = {self.input_B_test: inputs}) + else: + raise Exception('Conversion direction must be specified.') + + return generation + + + def save(self, directory, filename): + + if not os.path.exists(directory): + os.makedirs(directory) + self.saver.save(self.sess, os.path.join(directory, filename)) + + return os.path.join(directory, filename) + + def load(self, filepath): + + self.saver.restore(self.sess, filepath) + + + def summary(self): + + with tf.name_scope('generator_summaries'): + cycle_loss_summary = tf.summary.scalar('cycle_loss', self.cycle_loss) + identity_loss_summary = tf.summary.scalar('identity_loss', self.identity_loss) + generator_loss_A2B_summary = tf.summary.scalar('generator_loss_A2B', self.generator_loss_A2B) + generator_loss_B2A_summary = tf.summary.scalar('generator_loss_B2A', self.generator_loss_B2A) + generator_loss_summary = tf.summary.scalar('generator_loss', self.generator_loss) + generator_summaries = tf.summary.merge([cycle_loss_summary, identity_loss_summary, generator_loss_A2B_summary, generator_loss_B2A_summary, generator_loss_summary]) + + with tf.name_scope('discriminator_summaries'): + discriminator_loss_A_summary = tf.summary.scalar('discriminator_loss_A', self.discriminator_loss_A) + discriminator_loss_B_summary = tf.summary.scalar('discriminator_loss_B', self.discriminator_loss_B) + discriminator_loss_summary = tf.summary.scalar('discriminator_loss', self.discriminator_loss) + discriminator_summaries = tf.summary.merge([discriminator_loss_A_summary, discriminator_loss_B_summary, discriminator_loss_summary]) + + return generator_summaries, discriminator_summaries + + +if __name__ == '__main__': + + model = CycleGAN(num_features = 24) + #print('Graph Compile Successeded.') diff --git a/cycle_gan/module.py b/cycle_gan/module.py new file mode 100644 index 0000000..4319d9a --- /dev/null +++ b/cycle_gan/module.py @@ -0,0 +1,214 @@ +import tensorflow as tf + +def gated_linear_layer(inputs, gates, name = None): + + activation = tf.multiply(x = inputs, y = tf.sigmoid(gates), name = name) + + return activation + +def instance_norm_layer( + inputs, + epsilon = 1e-06, + activation_fn = None, + name = None): + + instance_norm_layer = tf.contrib.layers.instance_norm( + inputs = inputs, + epsilon = epsilon, + activation_fn = activation_fn) + + return instance_norm_layer + +def conv1d_layer( + inputs, + filters, + kernel_size, + strides = 1, + padding = 'same', + activation = None, + kernel_initializer = None, + name = None): + + conv_layer = tf.layers.conv1d( + inputs = inputs, + filters = filters, + kernel_size = kernel_size, + strides = strides, + padding = padding, + activation = activation, + kernel_initializer = kernel_initializer, + name = name) + + return conv_layer + +def conv2d_layer( + inputs, + filters, + kernel_size, + strides, + padding = 'same', + activation = None, + kernel_initializer = None, + name = None): + + conv_layer = tf.layers.conv2d( + inputs = inputs, + filters = filters, + kernel_size = kernel_size, + strides = strides, + padding = padding, + activation = activation, + kernel_initializer = kernel_initializer, + name = name) + + return conv_layer + +def residual1d_block( + inputs, + filters = 1024, + kernel_size = 3, + strides = 1, + name_prefix = 'residule_block_'): + + h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv') + h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm') + h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates') + h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates') + h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu') + h2 = conv1d_layer(inputs = h1_glu, filters = filters // 2, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h2_conv') + h2_norm = instance_norm_layer(inputs = h2, activation_fn = None, name = name_prefix + 'h2_norm') + + h3 = inputs + h2_norm + + return h3 + +def downsample1d_block( + inputs, + filters, + kernel_size, + strides, + name_prefix = 'downsample1d_block_'): + + h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv') + h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm') + h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates') + h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates') + h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu') + + return h1_glu + +def downsample2d_block( + inputs, + filters, + kernel_size, + strides, + name_prefix = 'downsample2d_block_'): + + h1 = conv2d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv') + h1_norm = instance_norm_layer(inputs = h1, activation_fn = None, name = name_prefix + 'h1_norm') + h1_gates = conv2d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates') + h1_norm_gates = instance_norm_layer(inputs = h1_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates') + h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu') + + return h1_glu + +def upsample1d_block( + inputs, + filters, + kernel_size, + strides, + shuffle_size = 2, + name_prefix = 'upsample1d_block_'): + + h1 = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_conv') + h1_shuffle = pixel_shuffler(inputs = h1, shuffle_size = shuffle_size, name = name_prefix + 'h1_shuffle') + h1_norm = instance_norm_layer(inputs = h1_shuffle, activation_fn = None, name = name_prefix + 'h1_norm') + + h1_gates = conv1d_layer(inputs = inputs, filters = filters, kernel_size = kernel_size, strides = strides, activation = None, name = name_prefix + 'h1_gates') + h1_shuffle_gates = pixel_shuffler(inputs = h1_gates, shuffle_size = shuffle_size, name = name_prefix + 'h1_shuffle_gates') + h1_norm_gates = instance_norm_layer(inputs = h1_shuffle_gates, activation_fn = None, name = name_prefix + 'h1_norm_gates') + + h1_glu = gated_linear_layer(inputs = h1_norm, gates = h1_norm_gates, name = name_prefix + 'h1_glu') + + return h1_glu + +def pixel_shuffler(inputs, shuffle_size = 2, name = None): + + n = tf.shape(inputs)[0] + w = tf.shape(inputs)[1] + c = inputs.get_shape().as_list()[2] + + oc = c // shuffle_size + ow = w * shuffle_size + + outputs = tf.reshape(tensor = inputs, shape = [n, ow, oc], name = name) + + return outputs + +def generator_gatedcnn(inputs, reuse = False, scope_name = 'generator_gatedcnn'): + + # inputs has shape [batch_size, num_features, time] + # we need to convert it to [batch_size, time, num_features] for 1D convolution + inputs = tf.transpose(inputs, perm = [0, 2, 1], name = 'input_transpose') + + with tf.variable_scope(scope_name) as scope: + # Discriminator would be reused in CycleGAN + if reuse: + scope.reuse_variables() + else: + assert scope.reuse is False + + h1 = conv1d_layer(inputs = inputs, filters = 128, kernel_size = 15, strides = 1, activation = None, name = 'h1_conv') + h1_gates = conv1d_layer(inputs = inputs, filters = 128, kernel_size = 15, strides = 1, activation = None, name = 'h1_conv_gates') + h1_glu = gated_linear_layer(inputs = h1, gates = h1_gates, name = 'h1_glu') + + # Downsample + d1 = downsample1d_block(inputs = h1_glu, filters = 256, kernel_size = 5, strides = 2, name_prefix = 'downsample1d_block1_') + d2 = downsample1d_block(inputs = d1, filters = 512, kernel_size = 5, strides = 2, name_prefix = 'downsample1d_block2_') + + # Residual blocks + r1 = residual1d_block(inputs = d2, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block1_') + r2 = residual1d_block(inputs = r1, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block2_') + r3 = residual1d_block(inputs = r2, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block3_') + r4 = residual1d_block(inputs = r3, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block4_') + r5 = residual1d_block(inputs = r4, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block5_') + r6 = residual1d_block(inputs = r5, filters = 1024, kernel_size = 3, strides = 1, name_prefix = 'residual1d_block6_') + + # Upsample + u1 = upsample1d_block(inputs = r6, filters = 1024, kernel_size = 5, strides = 1, shuffle_size = 2, name_prefix = 'upsample1d_block1_') + u2 = upsample1d_block(inputs = u1, filters = 512, kernel_size = 5, strides = 1, shuffle_size = 2, name_prefix = 'upsample1d_block2_') + + # Output + o1 = conv1d_layer(inputs = u2, filters = 24, kernel_size = 15, strides = 1, activation = None, name = 'o1_conv') + o2 = tf.transpose(o1, perm = [0, 2, 1], name = 'output_transpose') + + return o2 + + +def discriminator(inputs, reuse = False, scope_name = 'discriminator'): + + # inputs has shape [batch_size, num_features, time] + # we need to add channel for 2D convolution [batch_size, num_features, time, 1] + inputs = tf.expand_dims(inputs, -1) + + with tf.variable_scope(scope_name) as scope: + # Discriminator would be reused in CycleGAN + if reuse: + scope.reuse_variables() + else: + assert scope.reuse is False + + h1 = conv2d_layer(inputs = inputs, filters = 128, kernel_size = [3, 3], strides = [1, 2], activation = None, name = 'h1_conv') + h1_gates = conv2d_layer(inputs = inputs, filters = 128, kernel_size = [3, 3], strides = [1, 2], activation = None, name = 'h1_conv_gates') + h1_glu = gated_linear_layer(inputs = h1, gates = h1_gates, name = 'h1_glu') + + # Downsample + d1 = downsample2d_block(inputs = h1_glu, filters = 256, kernel_size = [3, 3], strides = [2, 2], name_prefix = 'downsample2d_block1_') + d2 = downsample2d_block(inputs = d1, filters = 512, kernel_size = [3, 3], strides = [2, 2], name_prefix = 'downsample2d_block2_') + d3 = downsample2d_block(inputs = d2, filters = 1024, kernel_size = [6, 3], strides = [1, 2], name_prefix = 'downsample2d_block3_') + + # Output + o1 = tf.layers.dense(inputs = d3, units = 1, activation = tf.nn.sigmoid) + + return o1 + diff --git a/cycle_gan/preprocess.py b/cycle_gan/preprocess.py new file mode 100644 index 0000000..f073b0b --- /dev/null +++ b/cycle_gan/preprocess.py @@ -0,0 +1,238 @@ +import librosa +import numpy as np +import os +import pyworld + +def load_wavs(wav_dir, sr): + + wavs = list() + for file in os.listdir(wav_dir): + file_path = os.path.join(wav_dir, file) + wav, _ = librosa.load(file_path, sr = sr, mono = True) + #wav = wav.astype(np.float64) + wavs.append(wav) + + return wavs + +def world_decompose(wav, fs, frame_period = 5.0): + + # Decompose speech signal into f0, spectral envelope and aperiodicity using WORLD + wav = wav.astype(np.float64) + f0, timeaxis = pyworld.harvest(wav, fs, frame_period = frame_period, f0_floor = 71.0, f0_ceil = 800.0) + sp = pyworld.cheaptrick(wav, f0, timeaxis, fs) + ap = pyworld.d4c(wav, f0, timeaxis, fs) + + return f0, timeaxis, sp, ap + +def world_encode_spectral_envelop(sp, fs, dim = 24): + + # Get Mel-cepstral coefficients (MCEPs) + + #sp = sp.astype(np.float64) + coded_sp = pyworld.code_spectral_envelope(sp, fs, dim) + + return coded_sp + +def world_decode_spectral_envelop(coded_sp, fs): + + fftlen = pyworld.get_cheaptrick_fft_size(fs) + #coded_sp = coded_sp.astype(np.float32) + #coded_sp = np.ascontiguousarray(coded_sp) + decoded_sp = pyworld.decode_spectral_envelope(coded_sp, fs, fftlen) + + return decoded_sp + + +def world_encode_data(wavs, fs, frame_period = 5.0, coded_dim = 24): + + f0s = list() + timeaxes = list() + sps = list() + aps = list() + coded_sps = list() + + for wav in wavs: + f0, timeaxis, sp, ap = world_decompose(wav = wav, fs = fs, frame_period = frame_period) + coded_sp = world_encode_spectral_envelop(sp = sp, fs = fs, dim = coded_dim) + f0s.append(f0) + timeaxes.append(timeaxis) + sps.append(sp) + aps.append(ap) + coded_sps.append(coded_sp) + + return f0s, timeaxes, sps, aps, coded_sps + + +def transpose_in_list(lst): + + transposed_lst = list() + for array in lst: + transposed_lst.append(array.T) + return transposed_lst + + +def world_decode_data(coded_sps, fs): + + decoded_sps = list() + + for coded_sp in coded_sps: + decoded_sp = world_decode_spectral_envelop(coded_sp, fs) + decoded_sps.append(decoded_sp) + + return decoded_sps + + +def world_speech_synthesis(f0, decoded_sp, ap, fs, frame_period): + + #decoded_sp = decoded_sp.astype(np.float64) + wav = pyworld.synthesize(f0, decoded_sp, ap, fs, frame_period) + # Librosa could not save wav if not doing so + wav = wav.astype(np.float32) + + return wav + + +def world_synthesis_data(f0s, decoded_sps, aps, fs, frame_period): + + wavs = list() + + for f0, decoded_sp, ap in zip(f0s, decoded_sps, aps): + wav = world_speech_synthesis(f0, decoded_sp, ap, fs, frame_period) + wavs.append(wav) + + return wavs + + +def coded_sps_normalization_fit_transoform(coded_sps): + + coded_sps_concatenated = np.concatenate(coded_sps, axis = 1) + coded_sps_mean = np.mean(coded_sps_concatenated, axis = 1, keepdims = True) + coded_sps_std = np.std(coded_sps_concatenated, axis = 1, keepdims = True) + + coded_sps_normalized = list() + for coded_sp in coded_sps: + coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std) + + return coded_sps_normalized, coded_sps_mean, coded_sps_std + +def coded_sps_normalization_transoform(coded_sps, coded_sps_mean, coded_sps_std): + + coded_sps_normalized = list() + for coded_sp in coded_sps: + coded_sps_normalized.append((coded_sp - coded_sps_mean) / coded_sps_std) + + return coded_sps_normalized + +def coded_sps_normalization_inverse_transoform(normalized_coded_sps, coded_sps_mean, coded_sps_std): + + coded_sps = list() + for normalized_coded_sp in normalized_coded_sps: + coded_sps.append(normalized_coded_sp * coded_sps_std + coded_sps_mean) + + return coded_sps + +def coded_sp_padding(coded_sp, multiple = 4): + + num_features = coded_sp.shape[0] + num_frames = coded_sp.shape[1] + num_frames_padded = int(np.ceil(num_frames / multiple)) * multiple + num_frames_diff = num_frames_padded - num_frames + num_pad_left = num_frames_diff // 2 + num_pad_right = num_frames_diff - num_pad_left + coded_sp_padded = np.pad(coded_sp, ((0, 0), (num_pad_left, num_pad_right)), 'constant', constant_values = 0) + + return coded_sp_padded + +def wav_padding(wav, sr, frame_period, multiple = 4): + + assert wav.ndim == 1 + num_frames = len(wav) + num_frames_padded = int((np.ceil((np.floor(num_frames / (sr * frame_period / 1000)) + 1) / multiple + 1) * multiple - 1) * (sr * frame_period / 1000)) + num_frames_diff = num_frames_padded - num_frames + num_pad_left = num_frames_diff // 2 + num_pad_right = num_frames_diff - num_pad_left + wav_padded = np.pad(wav, (num_pad_left, num_pad_right), 'constant', constant_values = 0) + + return wav_padded + + +def logf0_statistics(f0s): + + log_f0s_concatenated = np.ma.log(np.concatenate(f0s)) + log_f0s_mean = log_f0s_concatenated.mean() + log_f0s_std = log_f0s_concatenated.std() + + return log_f0s_mean, log_f0s_std + +def pitch_conversion(f0, mean_log_src, std_log_src, mean_log_target, std_log_target): + + # Logarithm Gaussian normalization for Pitch Conversions + f0_converted = np.exp((np.log(f0) - mean_log_src) / std_log_src * std_log_target + mean_log_target) + + return f0_converted + +def wavs_to_specs(wavs, n_fft = 1024, hop_length = None): + + stfts = list() + for wav in wavs: + stft = librosa.stft(wav, n_fft = n_fft, hop_length = hop_length) + stfts.append(stft) + + return stfts + + +def wavs_to_mfccs(wavs, sr, n_fft = 1024, hop_length = None, n_mels = 128, n_mfcc = 24): + + mfccs = list() + for wav in wavs: + mfcc = librosa.feature.mfcc(y = wav, sr = sr, n_fft = n_fft, hop_length = hop_length, n_mels = n_mels, n_mfcc = n_mfcc) + mfccs.append(mfcc) + + return mfccs + + +def mfccs_normalization(mfccs): + + mfccs_concatenated = np.concatenate(mfccs, axis = 1) + mfccs_mean = np.mean(mfccs_concatenated, axis = 1, keepdims = True) + mfccs_std = np.std(mfccs_concatenated, axis = 1, keepdims = True) + + mfccs_normalized = list() + for mfcc in mfccs: + mfccs_normalized.append((mfcc - mfccs_mean) / mfccs_std) + + return mfccs_normalized, mfccs_mean, mfccs_std + + +def sample_train_data(dataset_A, dataset_B, n_frames = 128): + + num_samples = min(len(dataset_A), len(dataset_B)) + train_data_A_idx = np.arange(len(dataset_A)) + train_data_B_idx = np.arange(len(dataset_B)) + np.random.shuffle(train_data_A_idx) + np.random.shuffle(train_data_B_idx) + train_data_A_idx_subset = train_data_A_idx[:num_samples] + train_data_B_idx_subset = train_data_B_idx[:num_samples] + + train_data_A = list() + train_data_B = list() + + for idx_A, idx_B in zip(train_data_A_idx_subset, train_data_B_idx_subset): + data_A = dataset_A[idx_A] + frames_A_total = data_A.shape[1] + assert frames_A_total >= n_frames + start_A = np.random.randint(frames_A_total - n_frames + 1) + end_A = start_A + n_frames + train_data_A.append(data_A[:,start_A:end_A]) + + data_B = dataset_B[idx_B] + frames_B_total = data_B.shape[1] + assert frames_B_total >= n_frames + start_B = np.random.randint(frames_B_total - n_frames + 1) + end_B = start_B + n_frames + train_data_B.append(data_B[:,start_B:end_B]) + + train_data_A = np.array(train_data_A) + train_data_B = np.array(train_data_B) + + return train_data_A, train_data_B \ No newline at end of file diff --git a/cycle_gan/protos/audio_pb2.py b/cycle_gan/protos/audio_pb2.py new file mode 100644 index 0000000..735b5e3 --- /dev/null +++ b/cycle_gan/protos/audio_pb2.py @@ -0,0 +1,260 @@ +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: protos/audio.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='protos/audio.proto', + package='voder_audio', + syntax='proto3', + serialized_pb=_b('\n\x12protos/audio.proto\x12\x0bvoder_audio\"\xe6\x04\n\x12VocodeAudioRequest\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x1c\n\x14request_batch_number\x18\x02 \x01(\x03\x12\x43\n\rvocode_params\x18\n \x01(\x0b\x32,.voder_audio.VocodeAudioRequest.VocodeParams\x12\x13\n\x0bskip_vocode\x18\x04 \x01(\x08\x12\x1b\n\x13\x62uffer_size_minimum\x18\x06 \x01(\x05\x12\x1d\n\x15\x64iscard_vocoded_audio\x18\x08 \x01(\x08\x1a\x86\x03\n\x0cVocodeParams\x12\x1b\n\x13initial_sample_rate\x18\x01 \x01(\x05\x12\x1c\n\x14original_source_rate\x18\t \x01(\x05\x12!\n\x19original_source_save_file\x18\n \x01(\x08\x12\x1c\n\x14pre_convert_resample\x18\x02 \x01(\x08\x12!\n\x19pre_convert_resample_rate\x18\x03 \x01(\x05\x12&\n\x1epre_convert_resample_save_file\x18\x04 \x01(\x08\x12*\n\"model_hyperparameter_sampling_rate\x18\x05 \x01(\x05\x12\x17\n\x0fmodel_save_file\x18\x0b \x01(\x08\x12\x1d\n\x15post_convert_resample\x18\x06 \x01(\x08\x12\"\n\x1apost_convert_resample_rate\x18\x07 \x01(\x05\x12\'\n\x1fpost_convert_resample_save_file\x18\x08 \x01(\x08\"I\n\x13VocodeAudioResponse\x12\x13\n\x0b\x66loat_audio\x18\x01 \x03(\x02\x12\x1d\n\x15response_batch_number\x18\x02 \x01(\x03\x62\x06proto3') +) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + +_VOCODEAUDIOREQUEST_VOCODEPARAMS = _descriptor.Descriptor( + name='VocodeParams', + full_name='voder_audio.VocodeAudioRequest.VocodeParams', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='initial_sample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.initial_sample_rate', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='original_source_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_rate', index=1, + number=9, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='original_source_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.original_source_save_file', index=2, + number=10, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pre_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample', index=3, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pre_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_rate', index=4, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='pre_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.pre_convert_resample_save_file', index=5, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='model_hyperparameter_sampling_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_hyperparameter_sampling_rate', index=6, + number=5, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='model_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.model_save_file', index=7, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_convert_resample', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample', index=8, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_convert_resample_rate', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_rate', index=9, + number=7, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='post_convert_resample_save_file', full_name='voder_audio.VocodeAudioRequest.VocodeParams.post_convert_resample_save_file', index=10, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=260, + serialized_end=650, +) + +_VOCODEAUDIOREQUEST = _descriptor.Descriptor( + name='VocodeAudioRequest', + full_name='voder_audio.VocodeAudioRequest', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='float_audio', full_name='voder_audio.VocodeAudioRequest.float_audio', index=0, + number=1, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='request_batch_number', full_name='voder_audio.VocodeAudioRequest.request_batch_number', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='vocode_params', full_name='voder_audio.VocodeAudioRequest.vocode_params', index=2, + number=10, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='skip_vocode', full_name='voder_audio.VocodeAudioRequest.skip_vocode', index=3, + number=4, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='buffer_size_minimum', full_name='voder_audio.VocodeAudioRequest.buffer_size_minimum', index=4, + number=6, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='discard_vocoded_audio', full_name='voder_audio.VocodeAudioRequest.discard_vocoded_audio', index=5, + number=8, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[_VOCODEAUDIOREQUEST_VOCODEPARAMS, ], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=36, + serialized_end=650, +) + + +_VOCODEAUDIORESPONSE = _descriptor.Descriptor( + name='VocodeAudioResponse', + full_name='voder_audio.VocodeAudioResponse', + filename=None, + file=DESCRIPTOR, + containing_type=None, + fields=[ + _descriptor.FieldDescriptor( + name='float_audio', full_name='voder_audio.VocodeAudioResponse.float_audio', index=0, + number=1, type=2, cpp_type=6, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + _descriptor.FieldDescriptor( + name='response_batch_number', full_name='voder_audio.VocodeAudioResponse.response_batch_number', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + options=None), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + options=None, + is_extendable=False, + syntax='proto3', + extension_ranges=[], + oneofs=[ + ], + serialized_start=652, + serialized_end=725, +) + +_VOCODEAUDIOREQUEST_VOCODEPARAMS.containing_type = _VOCODEAUDIOREQUEST +_VOCODEAUDIOREQUEST.fields_by_name['vocode_params'].message_type = _VOCODEAUDIOREQUEST_VOCODEPARAMS +DESCRIPTOR.message_types_by_name['VocodeAudioRequest'] = _VOCODEAUDIOREQUEST +DESCRIPTOR.message_types_by_name['VocodeAudioResponse'] = _VOCODEAUDIORESPONSE + +VocodeAudioRequest = _reflection.GeneratedProtocolMessageType('VocodeAudioRequest', (_message.Message,), dict( + + VocodeParams = _reflection.GeneratedProtocolMessageType('VocodeParams', (_message.Message,), dict( + DESCRIPTOR = _VOCODEAUDIOREQUEST_VOCODEPARAMS, + __module__ = 'protos.audio_pb2' + # @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest.VocodeParams) + )) + , + DESCRIPTOR = _VOCODEAUDIOREQUEST, + __module__ = 'protos.audio_pb2' + # @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioRequest) + )) +_sym_db.RegisterMessage(VocodeAudioRequest) +_sym_db.RegisterMessage(VocodeAudioRequest.VocodeParams) + +VocodeAudioResponse = _reflection.GeneratedProtocolMessageType('VocodeAudioResponse', (_message.Message,), dict( + DESCRIPTOR = _VOCODEAUDIORESPONSE, + __module__ = 'protos.audio_pb2' + # @@protoc_insertion_point(class_scope:voder_audio.VocodeAudioResponse) + )) +_sym_db.RegisterMessage(VocodeAudioResponse) + + +# @@protoc_insertion_point(module_scope) diff --git a/cycle_gan/requirements.txt b/cycle_gan/requirements.txt new file mode 100644 index 0000000..143c687 --- /dev/null +++ b/cycle_gan/requirements.txt @@ -0,0 +1,36 @@ +absl-py==0.7.1 +astor==0.8.0 +audioread==2.1.8 +cffi==1.12.3 +Cython==0.29.12 +decorator==4.4.0 +falcon==2.0.0 +falcon-multipart==0.2.0 +gast==0.2.2 +google-pasta==0.1.7 +grpcio==1.22.0 +h5py==2.9.0 +joblib==0.13.2 +Keras-Applications==1.0.8 +Keras-Preprocessing==1.1.0 +librosa==0.7.0 +llvmlite==0.29.0 +Markdown==3.1.1 +numba==0.45.0 +numpy==1.16.4 +protobuf==3.9.0 +pycparser==2.19 +pyworld==0.2.8 +pyzmq==18.1.0 +resampy==0.2.1 +scikit-learn==0.21.2 +scipy==1.3.0 +six==1.12.0 +SoundFile==0.10.2 +tensorboard==1.14.0 +tensorflow-estimator==1.14.0 +tensorflow-gpu==1.14.0 +termcolor==1.1.0 +Werkzeug==0.15.5 +wrapt==1.11.2 +zmq==0.0.0 diff --git a/cycle_gan/sidecar.py b/cycle_gan/sidecar.py new file mode 100755 index 0000000..9af7fb9 --- /dev/null +++ b/cycle_gan/sidecar.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 + +""" +Starts a demo HTTP server to capture and transform audio +as a live demonstration of the trained model. + +Brandon Thomas 2019-07-29 +""" + +# noinspection PyInterpreter +import argparse +import io +import librosa +import numpy as np +import os +import pathlib +import scipy +import soundfile +import struct +import subprocess +import tempfile +import tensorflow as tf +import zmq + +from model import CycleGAN +from preprocess import * +from protos.audio_pb2 import VocodeAudioRequest +from protos.audio_pb2 import VocodeAudioResponse + +print("TensorFlow version: {}".format(tf.version.VERSION)) + + +class Converter(): + def __init__(self, model_dir, model_name): + self.num_features = 24 + self.frame_period = 5.0 + + self.model = CycleGAN(num_features = self.num_features, mode = 'test') + + self.model.load(filepath = os.path.join(model_dir, model_name)) + + self.mcep_normalization_params = np.load(os.path.join(model_dir, 'mcep_normalization.npz')) + self.mcep_mean_A = self.mcep_normalization_params['mean_A'] + self.mcep_std_A = self.mcep_normalization_params['std_A'] + self.mcep_mean_B = self.mcep_normalization_params['mean_B'] + self.mcep_std_B = self.mcep_normalization_params['std_B'] + + self.logf0s_normalization_params = np.load(os.path.join(model_dir, + 'logf0s_normalization.npz')) + self.logf0s_mean_A = self.logf0s_normalization_params['mean_A'] + self.logf0s_std_A = self.logf0s_normalization_params['std_A'] + self.logf0s_mean_B = self.logf0s_normalization_params['mean_B'] + self.logf0s_std_B = self.logf0s_normalization_params['std_B'] + + def convert_partial(self, wav, conversion_direction='A2B', model_sampling_rate=16000): + wav = wav_padding(wav = wav, + sr = model_sampling_rate, + frame_period = self.frame_period, + multiple = 4) + + #librosa.output.write_wav('wav_padding.wav', wav, model_sampling_rate) + + f0, timeaxis, sp, ap = world_decompose(wav = wav, + fs = model_sampling_rate, + frame_period = self.frame_period) + coded_sp = world_encode_spectral_envelop(sp = sp, + fs = model_sampling_rate, + dim = self.num_features) + coded_sp_transposed = coded_sp.T + + f0_converted = pitch_conversion(f0 = f0, + mean_log_src = self.logf0s_mean_A, + std_log_src = self.logf0s_std_A, + mean_log_target = self.logf0s_mean_B, + std_log_target = self.logf0s_std_B) + coded_sp_norm = (coded_sp_transposed - self.mcep_mean_A) / self.mcep_std_A + + coded_sp_converted_norm = self.model.test(inputs = np.array([coded_sp_norm]), + direction = conversion_direction)[0] + coded_sp_converted = coded_sp_converted_norm * self.mcep_std_B + self.mcep_mean_B + + coded_sp_converted = coded_sp_converted.T + coded_sp_converted = np.ascontiguousarray(coded_sp_converted) + decoded_sp_converted = world_decode_spectral_envelop(coded_sp = coded_sp_converted, + fs = model_sampling_rate) + wav_transformed = world_speech_synthesis(f0 = f0_converted, + decoded_sp = decoded_sp_converted, + ap = ap, + fs = model_sampling_rate, + frame_period = self.frame_period) + + # For debugging model output, uncomment the following line: + #librosa.output.write_wav('model_output.wav', wav_transformed, model_sampling_rate) + + # TODO: Perhaps ditch this. It's probably unnecessary work. + #upsampled = librosa.resample(wav_transformed, model_sampling_rate, 48000) + #pcm_data = upsampled.astype(np.float64) + #stereo_pcm_data = np.tile(pcm_data, (2,1)).T + #return stereo_pcm_data.astype(np.float32) + #librosa.output.write_wav('model_output.wav', pcm_data, 48000) + + #return wav + return wav_transformed + + #def convert(self, wav, conversion_direction='A2B'): + # pcm_data = self.convert_partial(wav, conversion_direction=conversion_direction) + # buf = io.BytesIO() + # # pcm_data: A 1-D or 2-D numpy array of either integer or float data-type. + # # To write multiple-channels, use a 2-D array of shape (Nsamples, Nchannels). + # scipy.io.wavfile.write(buf, 48000, pcm_data) + # return buf + +# Set up model +# This should live long in memory, so we do it up front. +model_dir_default = './model/sf1_tm1' +model_name_default = 'sf1_tm1.ckpt' + +# TODO: UNCOMMENT +converter = Converter(model_dir_default, model_name_default) + +TEMP_DIR = tempfile.TemporaryDirectory(prefix='queue_audio') + +def temp_file_name(suffix='.wav'): + # NB: Not actually using the tempfile. Just the random name. + temp_file = tempfile.NamedTemporaryFile(suffix=suffix) + name = os.path.basename(temp_file.name) + return os.path.join(TEMP_DIR.name, name) + +def convert(audio, + vocode_params=None, + request_batch_number=0, + skip_vocode=False, + save_files=False, + discard_vocoded_audio=False): + #audio = np.array(audio, dtype=np.int16) + #data, samplerate = soundfile.read(audio) + #print('samplerate', samplerate) + """ + samplerate 44100 + data.shape (77824, 2) + data.dtype float64 + mono [0.0050354 0.00518799 0.0050354 ... 0.11651611 0.11935425 0.1164856 ] + mono.shape (77824,) + mono.dtype float64 + downsampled [0.00329925 0.00577342 0.00474898 ... 0.0851728 0.1166483 0. ] + downsampled.shape (28236,) + downsampled.dtype float64 + + 1) Data should be float64 + 2) Output is BYTES!! Not floats. + 3) Result is still somehow mono!? + -------------------------------------------------- + + demo_server.py + + capture sample rate = 44100 + downsample for algorithm = 16000 + upsample after done = 48000 + + """ + + audio = np.array(audio, dtype=np.float64) + print('audio.shape', audio.shape) + print('audio.type', audio.dtype) + + #source_rate = 88000 # Experimentally determined for Rust library 'CPAL' + #source_rate = 44100 + + if vocode_params.original_source_save_file: + #filename = temp_file_name('.wav') + filename = 'debug/{}_original_source.wav'.format(request_batch_number) + print('----- Original wav file out: {}'.format(filename)) + scipy.io.wavfile.write(filename, vocode_params.original_source_rate, audio) + + # NB: Convert the input stereo signal into mono. + # In the future the frontend should be responsible for sampling details. + #audio = audio[:, 0] + + if vocode_params.pre_convert_resample: + print("Resampling audio from {} Hz to {} Hz".format(vocode_params.original_source_rate, + vocode_params.pre_convert_resample_rate)) + audio = librosa.resample(audio, vocode_params.original_source_rate, vocode_params.pre_convert_resample_rate) + print('resampled_audio.shape', audio.shape) + print('resampled_audio.type', audio.dtype) + if vocode_params.pre_convert_resample_save_file: + #filename = temp_file_name('.wav') + filename = 'debug/{}_pre_convert_resample.wav'.format(request_batch_number) + print('----- Pre-downsampled file out: {}'.format(filename)) + scipy.io.wavfile.write(filename, vocode_params.pre_convert_resample_rate, audio) + + if skip_vocode: + return audio + + results = converter.convert_partial(audio, + conversion_direction='A2B', + model_sampling_rate=vocode_params.model_hyperparameter_sampling_rate) + + if vocode_params.model_save_file: + #filename = temp_file_name('.wav') + filename = 'debug/{}_model_output.wav'.format(request_batch_number) + print('----- Model file out: {}'.format(filename)) + scipy.io.wavfile.write(filename, vocode_params.model_hyperparameter_sampling_rate, results) + + print('results.type', type(results)) + print('results.len', len(results)) + + if vocode_params.post_convert_resample: + print("Resampling output audio from {} Hz to {} Hz".format(vocode_params.model_hyperparameter_sampling_rate, + vocode_params.post_convert_resample_rate)) + #consume_rate = 68000 # Experimentally determined for Rust lib 'CPAL' + results = librosa.resample(results, + vocode_params.model_hyperparameter_sampling_rate, + vocode_params.post_convert_resample_rate) + if vocode_params.post_convert_resample_save_file: + #filename = temp_file_name('.wav') + filename = 'debug/{}_post_convert_resample.wav'.format(request_batch_number) + print('----- Upsampled (transformed) file out: {}'.format(filename)) + scipy.io.wavfile.write(filename, vocode_params.post_convert_resample_rate, results) + + if discard_vocoded_audio: + return audio + else: + return results + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--port', type=int, default=5555) + args = parser.parse_args() + + context = zmq.Context() + socket = context.socket(zmq.REP) + socket.bind("tcp://*:{}".format(args.port)) + print('Running server...') + + queue = [] + while True: + # Wait for next request from client + message = socket.recv() + + vocode_request = VocodeAudioRequest.FromString(message) + vocode_params = vocode_request.vocode_params + + queue.extend(vocode_request.float_audio) + + if len(queue) >= vocode_request.buffer_size_minimum: + #results = queue[:] + results = convert(queue, + vocode_params=vocode_params, + request_batch_number=vocode_request.request_batch_number, + skip_vocode=vocode_request.skip_vocode, + discard_vocoded_audio=vocode_request.discard_vocoded_audio) + queue = [] + + vocode_response = VocodeAudioResponse() + vocode_response.float_audio[:] = results + + socket.send(vocode_response.SerializeToString()) + else: + # Must send reply back to client + socket.send(b"OK") + +if __name__ == '__main__': + main() + diff --git a/cycle_gan/utils.py b/cycle_gan/utils.py new file mode 100644 index 0000000..858af12 --- /dev/null +++ b/cycle_gan/utils.py @@ -0,0 +1,17 @@ +import tensorflow as tf +import os +import random +import numpy as np + +def l1_loss(y, y_hat): + + return tf.reduce_mean(tf.abs(y - y_hat)) + +def l2_loss(y, y_hat): + + return tf.reduce_mean(tf.square(y - y_hat)) + +def cross_entropy_loss(logits, labels): + return tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits = logits, labels = labels)) + + diff --git a/protos/audio.proto b/protos/audio.proto new file mode 100644 index 0000000..46e26b6 --- /dev/null +++ b/protos/audio.proto @@ -0,0 +1,64 @@ +syntax = "proto3"; +package voder_audio; + +message VocodeAudioRequest { + repeated float float_audio = 1; + + // An autoincrement sent with each request. + int64 request_batch_number = 2; + + message VocodeParams { + // The initial sample rate coming from the microphone. + int32 initial_sample_rate = 1; + + // Original sample + int32 original_source_rate = 9; + bool original_source_save_file = 10; + + // Resample before passing to the algorithm? + bool pre_convert_resample = 2; + int32 pre_convert_resample_rate = 3; + bool pre_convert_resample_save_file = 4; + + // The hard-coded hyperparameter of the model. + // Probably don't want to change from 16000. + int32 model_hyperparameter_sampling_rate = 5; + bool model_save_file = 11; + + // Resample after passing to the algorithm? + bool post_convert_resample = 6; + int32 post_convert_resample_rate = 7; + bool post_convert_resample_save_file = 8; + } + + VocodeParams vocode_params = 10; + + // We'll resend these params for every batch, but will only use + // the last batch's settings. Kind of a dumb API, but it'll work. + //int32 sample_rate = 2; + + //bool skip_resample = 3; + bool skip_vocode = 4; + + // Debugging + //bool save_files = 5; + + // How big we let the buffer grow before running 'convert'. + int32 buffer_size_minimum = 6; + + // Requested output rate + //int32 output_rate = 7; + + // Discard the vocoded audio and return the original + bool discard_vocoded_audio = 8; + + // Rate param for the model + //int32 model_sampling_rate = 9; +} + +message VocodeAudioResponse { + repeated float float_audio = 1; + + // A sidecar-side autoincrement sent with each request. + int64 response_batch_number = 2; +} diff --git a/src/audio_hardware.rs b/src/audio_hardware.rs new file mode 100644 index 0000000..32f31fc --- /dev/null +++ b/src/audio_hardware.rs @@ -0,0 +1,84 @@ +use cpal::traits::{DeviceTrait, EventLoopTrait, HostTrait}; +use cpal; +use audio_queue::AudioQueue; +use std::sync::Arc; + +/// CPAL Audio Device +/// Access audio input and output streams. +pub struct AudioHardware { + host: cpal::Host, + event_loop: cpal::EventLoop, + microphone_queue: Arc, + speaker_queue: Arc, +} + +impl AudioHardware { + pub fn new(microphone_queue: Arc, speaker_queue: Arc) -> Result { + let host = cpal::default_host(); + let event_loop = host.event_loop(); + + let input_device = host.default_input_device().expect("failed to get default input device"); + let output_device = host.default_output_device().expect("failed to get default output device"); + + println!("Using default input device: \"{}\"", input_device.name()?); + println!("Using default output device: \"{}\"", output_device.name()?); + + // We'll try and use the same format between streams to keep it simple + let mut format = input_device.default_input_format()?; + format.data_type = cpal::SampleFormat::F32; + + // Build streams. + println!("Attempting to build both streams with `{:?}`.", format); + let input_stream_id = event_loop.build_input_stream(&input_device, &format)?; + let output_stream_id = event_loop.build_output_stream(&output_device, &format)?; + println!("Successfully built streams."); + + Ok(AudioHardware { + host, + event_loop, + microphone_queue, + speaker_queue, + }) + } + + pub fn run(&mut self) { + self.event_loop.run(|id, result| { + let data = match result { + Ok(data) => data, + Err(err) => { + eprintln!("An error occurred on stream {:?}: {}", id, err); + return; + } + }; + + match data { + cpal::StreamData::Input { buffer: cpal::UnknownTypeInputBuffer::F32(buffer) } => { + //assert_eq!(id, input_stream_id); + let mut output_fell_behind = false; + for &sample in buffer.iter() { + self.microphone_queue.push_back(sample); + } + }, + cpal::StreamData::Output { buffer: cpal::UnknownTypeOutputBuffer::F32(mut buffer) } => { + //println!("Audio out buffer len: {}", self.speaker_queue.len()); + let request_size = buffer.len(); + let mut drained = self.speaker_queue.drain_size((request_size)); + match drained { + None => { + for sample in buffer.iter_mut() { + *sample = 0.0; + } + }, + Some(mut drained) => { + for (i, sample) in buffer.iter_mut().enumerate() { + *sample = drained.get(i).copied().unwrap(); + //*sample = 0.0; + } + }, + } + }, + _ => panic!("We're expecting f32 data."), + } + }) + } +} diff --git a/src/audio_queue.rs b/src/audio_queue.rs new file mode 100644 index 0000000..f8314aa --- /dev/null +++ b/src/audio_queue.rs @@ -0,0 +1,78 @@ +use zmq; +use zmq::{Error, Socket}; +use std::collections::VecDeque; +use std::sync::{RwLock, PoisonError}; + +pub struct AudioQueue { + queue: RwLock>, +} + +impl AudioQueue { + pub fn new() -> Self { + Self { + queue: RwLock::new(VecDeque::new()), + } + } + + pub fn push_back(&self, val: f32) { + match self.queue.write() { + Ok(mut queue) => { + queue.push_back(val); + }, + Err(_) => { + unreachable!("This shouldn't happen (push_back)"); + }, + } + } + + pub fn extend(&self, buf: Vec) { + match self.queue.write() { + Ok(mut queue) => { + queue.extend(buf); + }, + Err(_) => { + unreachable!("This shouldn't happen (extend)"); + }, + } + } + + pub fn drain(&self) -> Vec { + match self.queue.write() { + Ok(mut queue) => { + return queue.drain(..) + .collect::>(); + }, + Err(_) => { + unreachable!("This shouldn't happen (drain)"); + }, + } + } + + pub fn drain_size(&self, size: usize) -> Option> { + match self.queue.write() { + Ok(mut queue) => { + if queue.len() < size { + return None; + } + let vec = queue.drain(0..size) + .collect::>(); + + return Some(vec); + }, + Err(_) => { + unreachable!("This shouldn't happen (drain_size)"); + }, + } + } + + pub fn len(&self) -> usize { + match self.queue.read() { + Ok(mut queue) => { + return queue.len(); + }, + Err(_) => { + unreachable!("This shouldn't happen (len)"); + }, + } + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..80f840b --- /dev/null +++ b/src/main.rs @@ -0,0 +1,72 @@ +//! +//! This is a process that captures microphone input, ships proto-over-zeromq to a python +//! CycleGAN-VC (tensorflow) sidecar server, collects the results, then plays them over +//! the speaker. +//! + +extern crate bytes; +extern crate byteorder; +extern crate cpal; +extern crate failure; +extern crate prost; +extern crate prost_types; +extern crate wavy; +extern crate world_sys; +extern crate zmq; + +//include!(concat!(env!("OUT_DIR"), "/voder.audio.rs")); + +pub mod audio_hardware; +pub mod audio_queue; +//pub mod model; +pub mod protos; +pub mod sidecar; +pub mod synthesis; + +use audio_hardware::AudioHardware; +use protos::voder_audio::VocodeAudioRequest; +use protos::voder_audio::VocodeAudioResponse; +use sidecar::Sidecar; + +use wavy::*; + +use byteorder::{ByteOrder, BigEndian, LittleEndian, ReadBytesExt}; +use failure::_core::time::Duration; +use audio_queue::AudioQueue; +use prost::Message; +//use model::load_model; +//use model::print_version; +use std::collections::VecDeque; +use std::fs::File; +use std::io::{Read, Cursor}; +use std::path::Path; +use std::process::exit; +use std::sync::Arc; +use std::thread; +use zmq::{Error, Socket}; +use protos::voder_audio::vocode_audio_request::VocodeParams; + +fn main() { + //print_version(); + //load_model(); // TODO: This works. Temporarily commented out + run_audio().expect("Should work"); +} + +const LATENCY_MS: f32 = 50.0; + +fn run_audio() -> Result<(), failure::Error> { + let mut microphone_queue = Arc::new(AudioQueue::new()); + let mut microphone_queue_2 = microphone_queue.clone(); + + let mut post_process_queue = Arc::new(AudioQueue::new()); + let mut post_process_queue_2 = post_process_queue.clone(); + let mut sidecar = Sidecar::new("tcp://127.0.0.1:5555", microphone_queue_2, post_process_queue); + + thread::spawn(move || { + sidecar.run(); + }); + + let mut audio_hardware = AudioHardware::new(microphone_queue, post_process_queue_2).expect(""); + audio_hardware.run(); + Ok(()) +} diff --git a/src/protos/mod.rs b/src/protos/mod.rs new file mode 100644 index 0000000..5e3380f --- /dev/null +++ b/src/protos/mod.rs @@ -0,0 +1 @@ +pub mod voder_audio; \ No newline at end of file diff --git a/src/protos/voder_audio.rs b/src/protos/voder_audio.rs new file mode 100644 index 0000000..4dde4f4 --- /dev/null +++ b/src/protos/voder_audio.rs @@ -0,0 +1,70 @@ +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct VocodeAudioRequest { + #[prost(float, repeated, tag="1")] + pub float_audio: ::std::vec::Vec, + /// An autoincrement sent with each request. + #[prost(int64, tag="2")] + pub request_batch_number: i64, + #[prost(message, optional, tag="10")] + pub vocode_params: ::std::option::Option, + // We'll resend these params for every batch, but will only use + // the last batch's settings. Kind of a dumb API, but it'll work. + //int32 sample_rate = 2; + + ///bool skip_resample = 3; + #[prost(bool, tag="4")] + pub skip_vocode: bool, + // Debugging + //bool save_files = 5; + + /// How big we let the buffer grow before running 'convert'. + #[prost(int32, tag="6")] + pub buffer_size_minimum: i32, + // Requested output rate + //int32 output_rate = 7; + + /// Discard the vocoded audio and return the original + #[prost(bool, tag="8")] + pub discard_vocoded_audio: bool, +} +pub mod vocode_audio_request { + #[derive(Clone, PartialEq, ::prost::Message)] + pub struct VocodeParams { + /// The initial sample rate coming from the microphone. + #[prost(int32, tag="1")] + pub initial_sample_rate: i32, + /// Original sample + #[prost(int32, tag="9")] + pub original_source_rate: i32, + #[prost(bool, tag="10")] + pub original_source_save_file: bool, + /// Resample before passing to the algorithm? + #[prost(bool, tag="2")] + pub pre_convert_resample: bool, + #[prost(int32, tag="3")] + pub pre_convert_resample_rate: i32, + #[prost(bool, tag="4")] + pub pre_convert_resample_save_file: bool, + /// The hard-coded hyperparameter of the model. + /// Probably don't want to change from 16000. + #[prost(int32, tag="5")] + pub model_hyperparameter_sampling_rate: i32, + #[prost(bool, tag="11")] + pub model_save_file: bool, + /// Resample after passing to the algorithm? + #[prost(bool, tag="6")] + pub post_convert_resample: bool, + #[prost(int32, tag="7")] + pub post_convert_resample_rate: i32, + #[prost(bool, tag="8")] + pub post_convert_resample_save_file: bool, + } +} +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct VocodeAudioResponse { + #[prost(float, repeated, tag="1")] + pub float_audio: ::std::vec::Vec, + /// A sidecar-side autoincrement sent with each request. + #[prost(int64, tag="2")] + pub response_batch_number: i64, +} diff --git a/src/sidecar.rs b/src/sidecar.rs new file mode 100644 index 0000000..600ee00 --- /dev/null +++ b/src/sidecar.rs @@ -0,0 +1,217 @@ +use audio_queue::AudioQueue; +use prost::Message; +use protos::voder_audio::VocodeAudioRequest; +use protos::voder_audio::VocodeAudioResponse; +use protos::voder_audio::vocode_audio_request::VocodeParams; +use std::collections::VecDeque; +use std::sync::Arc; +use zmq::{Error, Socket}; +use zmq; + +/// Talks a custom protobuf over ZeroMQ protocol to a Python TensorFlow sidecar process +/// that evaluates the model and ships back audio byte results. +pub struct Sidecar { + socket_endpoint: String, + microphone_queue: Arc, + processed_queue: Arc, +} + +impl Sidecar { + pub fn new(socket_endpoint: &str, + microphone_queue: Arc, + processed_queue: Arc) -> Self { + Self { + socket_endpoint: socket_endpoint.into(), + microphone_queue, + processed_queue, + } + } + + /// Heuristic to determine if a speaker is speaking. + /// Basically just checks audio loudness. + fn is_speaking(audio: &Vec) -> bool { + let mut avg_pos = 0.0f32; + let mut num_pos = 0; + + for x in audio { + if x > &0.0f32 { + avg_pos += x; + num_pos += 1; + } + } + + avg_pos = avg_pos / num_pos as f32; + avg_pos > 0.006f32 + } + + pub fn run(&mut self) { + let mut context = zmq::Context::new(); + let mut socket = context.socket(zmq::REQ).unwrap(); + + socket.connect(&self.socket_endpoint).unwrap(); + + let mut reconnect = false; + let mut fail_count = 0; + + const SEND_SIZE : usize = 5000; + + let mut request_batch_number = 0i64; + + let mut activated = false; + let mut frames_activated = 0; + let mut frames_deactivated = 0; + + let mut ring_buffer = VecDeque::with_capacity(5); + + loop { + let mut drained = match self.microphone_queue.drain_size(SEND_SIZE) { + None => { continue; }, + Some(d) => d, + }; + + /* + Heuristic for audio. Don't record when silent (using threshold). + + mic: act act act dec act act act act dec dec dec dec dec act dec + sent: | ok ok ok ok ok ok ok ok ok ok ok| + */ + + + // NB: We don't want to lose audio right at the activating edge. + // Save some bounded history to replay once activated. + ring_buffer.push_back(drained.clone()); + if ring_buffer.len() > 5 { + ring_buffer.pop_front(); + } + + if activated { + if !Self::is_speaking(&drained) { + frames_deactivated += 1; + } else { + frames_deactivated = 0; + } + + if frames_deactivated > 20 { + println!("Deactivating edge"); + activated = false; + frames_activated = 0; + frames_deactivated = 0; + } + + } else { + if Self::is_speaking(&drained) { + frames_activated += 1; + } else { + frames_activated = 0; + } + + if frames_activated > 3 { + println!("Activating edge"); + activated = true; + frames_activated = 0; + frames_deactivated = 0; + } + } + + if !activated { + continue; + } + + let drained = ring_buffer.drain(..) + .flat_map(|sample| sample) + .collect::>(); + + request_batch_number += 1; + + let mut vocode_request = VocodeAudioRequest::default(); + //vocode_request.sample_rate = 16000; + //vocode_request.output_rate = 16000; + //vocode_request.model_sampling_rate = 88000; // This is close! + //vocode_request.model_sampling_rate = 88000; + //vocode_request.skip_resample = false; + vocode_request.skip_vocode = false; + vocode_request.discard_vocoded_audio = false; + //vocode_request.buffer_size_minimum = 5000; // AWFUL. SO CHOPPY. + //vocode_request.buffer_size_minimum = 50000; // Practically real time, but lots more phase distortion. + //vocode_request.buffer_size_minimum = 30000; // Hm, hmm... not bad + vocode_request.buffer_size_minimum = 70000; // Sounds pretty good. + //vocode_request.buffer_size_minimum = 100000; // This sounds good! A bit slow ~3seconds. + //vocode_request.buffer_size_minimum = 200000; + vocode_request.float_audio = drained.clone(); + vocode_request.request_batch_number = request_batch_number; + + let mut vocode_params = VocodeParams::default(); + vocode_params.original_source_rate = 88000; // This is correct for saving + + vocode_params.pre_convert_resample = true; + vocode_params.pre_convert_resample_rate = 16000; + + vocode_params.model_hyperparameter_sampling_rate = 16000; + + vocode_params.post_convert_resample = true; + vocode_params.post_convert_resample_rate = 88000; + + vocode_params.original_source_save_file = false; + vocode_params.pre_convert_resample_save_file = false; + vocode_params.model_save_file = false; + vocode_params.post_convert_resample_save_file = false; + + vocode_request.vocode_params = Some(vocode_params); + + let mut encoded_bytes = Vec::with_capacity(vocode_request.encoded_len()); + vocode_request.encode(&mut encoded_bytes).unwrap(); + + if reconnect { + reconnect = false; + println!("RECONNECT"); + + //thread::sleep(Duration::from_millis(200)); + + socket = match context.socket(zmq::REQ) { + Ok(s) => s, + Err(e) => { + println!("Error creating socket: {:?}", e); + continue + }, + }; + + match socket.connect("tcp://127.0.0.1:5555") { + Ok(_) => {}, + Err(err) => { + println!("Err B: {:?}", err); + }, + } + } + + match socket.send(&encoded_bytes, 0) { + Ok(_) => { + //println!("Sent len: {}", bytes.len()); + }, + Err(e) => { + println!("send err {}: {:?}", e.to_raw(), e); + fail_count += 1; + }, + } + + match socket.recv_bytes(0) { + Ok(buf) => { + //println!("---> Buf len: {}", buf.len()); + if buf.len() > 2 { + // Receive data condition. + let vocode_response = VocodeAudioResponse::decode(buf).unwrap(); + self.processed_queue.extend(vocode_response.float_audio); + } + }, + Err(e) => { + fail_count += 1; + println!("recv err: {:?}", e); + }, + } + + if fail_count > 5 { + fail_count = 0; + reconnect = true; + } + } + } +} \ No newline at end of file diff --git a/src/synthesis.rs b/src/synthesis.rs new file mode 100644 index 0000000..b7c926b --- /dev/null +++ b/src/synthesis.rs @@ -0,0 +1,38 @@ +use world_sys::core::synthesize::SynthesizeError; +use world_sys::core::synthesize::SynthesizeResult; +use world_sys::core::synthesize::synthesize; + +use world_sys::core::spectral::code_spectral_envelope; + +/* +TODO: ALGORITHM TO GET VOICE TRANSFORMATION: + +- [mono] +- [librosa.resample (downsample)] +- core convert() + - wav_padding() + - [maths] + - [np.pad] + - world_decompose() + - pyworld.harvest + - pyworld.cheaptrick + - pyworld.d4c + - world_encode_spectral_envelope() + - pyworld.code_spectral_envelope + - [matrix transpose] + - pitch_conversion() + - [maths] + - [maths : coded_sp_norm] + - << evaluate ML model >> + - [maths : coded_sp_converted] + - [matrix transpose] + - [np.ascontiguousarray] + - world_decode_spectral_envelop + - pyworld.get_cheaptrick_fft_size + - pyworld.decode_spectral_envelope + - world_speech_synthesis + - pyworld.synthesize + - [librosa.resample (upsample)] + - [wav things] +*/ +