From 8e868894cbcaf421c263f3555af4d29bef653420 Mon Sep 17 00:00:00 2001 From: salmonik Date: Sat, 5 Sep 2026 19:35:21 +0200 Subject: feat: piper now not a subprocess, cache system for TTS --- AGENTS.md | 6 +- CONTEXT.md | 15 ++ Cargo.lock | 359 +++++++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 2 + build.rs | 52 +++++++ src/commands/ai.rs | 5 +- src/commands/tts.rs | 14 +- src/events/mod.rs | 34 ++--- src/lang.rs | 7 - src/main.rs | 5 + src/tts.rs | 384 ++++++++++++++++++++++++++++++++++++---------------- tests/tts_smoke.rs | 54 ++++++++ 12 files changed, 762 insertions(+), 175 deletions(-) create mode 100644 build.rs create mode 100644 tests/tts_smoke.rs diff --git a/AGENTS.md b/AGENTS.md index a074a43..8352e0e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -23,7 +23,7 @@ src/ ├── events/mod.rs — handle_ready, handle_voice_state_update, handle_message ├── openrouter/mod.rs — OpenRouterClient (chat, chat_with_history) ├── tasks/mod.rs — cron_scheduler_loop (30s polling, stub exec bodies) -├── tts.rs — synthesize(): Piper subprocess → Google TTS fallback +├── tts.rs — TtsState (piper-rs in-process, DashMap cache, pre-gen, resampler, WAV encoder) └── voice/mod.rs — VoiceManager (connect, disconnect, get_call), mp3_to_input ``` @@ -35,6 +35,10 @@ src/ - **Shared state**: `Data` struct in `main.rs` — passed as poise `user_data` - **Commands**: each submodule exports `pub fn commands() -> Vec>` - **Temp path**: always use `crate::config::TEMP_PATH` (resolves to `/tmp/senator_sperma`) +- **TTS**: piper-rs in-process (no subprocess). `TtsState` in `Data` holds `OnceLock>` + `DashMap` cache. Synthesize via `data.tts.synthesize(text)`. Pre-generates 6 event phrases per user on first voice event. No Google TTS fallback. + +## Build +- `build.rs` — compiles C stubs for espeak-rs-sys missing sonic/audio_object symbols ## Stubs to Fill (see COMPARISON.md) - `commands/music.rs` — `/play` is placeholder diff --git a/CONTEXT.md b/CONTEXT.md index e272a7a..bdd2790 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -32,3 +32,18 @@ holds no implementation decisions or specs (those live in `docs/adr/`). afk). Paged at 10 entries, descending by default. - **Tracked guild** — the guild containing `status_channel_id`. Message counts are only tracked in this guild. + +## TTS + +- **Synthesize** — convert text to audio via local Piper TTS (ONNX model + loaded in-process via `piper-rs`). No external fallback. Output is f32 PCM + at the model's native sample rate (typically 22050 Hz mono), resampled to + 48 kHz stereo via inline linear interpolation before playback. Songbird does + not resample — playing un-resampled audio results in wrong playback speed. +- **TtsState** — struct held in poise `Data`: `OnceCell>` for the + model instance, `DashMap, Instant)>` for the text→audio cache. + Piper loaded lazily on first TTS call. Cache keyed by full phrase string, + evicted by TTL. No Google TTS fallback. +- **Pre-generation** — on first voice event for a user, synthesize all 6 event + phrases (join/leave/deafen/undeafen/mute/unmute) with their display name and + cache them. ~120ms first time, instant after. Lazy — no startup cost. diff --git a/Cargo.lock b/Cargo.lock index caf04ae..7ab8ebc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -174,7 +174,7 @@ dependencies = [ "anyhow", "arrayvec", "log", - "nom", + "nom 8.0.0", "num-rational", "v_frame", ] @@ -222,12 +222,41 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "base64ct" version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bindgen" +version = "0.69.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" +dependencies = [ + "bitflags 2.13.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.118", + "which", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -370,7 +399,16 @@ dependencies = [ "find-msvc-tools", "jobserver", "libc", - "shlex", + "shlex 2.0.1", +] + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom 7.1.3", ] [[package]] @@ -465,6 +503,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "clang-sys" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cmake" version = "0.1.58" @@ -926,7 +975,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ "const-oid", - "pem-rfc7468", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "pem-rfc7468 1.0.0", "zeroize", ] @@ -1086,7 +1145,7 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", "elliptic-curve", "rfc6979", @@ -1147,7 +1206,7 @@ dependencies = [ "generic-array", "group", "hkdf 0.12.4", - "pem-rfc7468", + "pem-rfc7468 0.7.0", "pkcs8", "rand_core 0.6.4", "sec1", @@ -1229,6 +1288,26 @@ dependencies = [ "version_check", ] +[[package]] +name = "espeak-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89534f20ce914016aae6cc63e9476aad41cee3e96b48e712e1702caf24c0c538" +dependencies = [ + "espeak-rs-sys", +] + +[[package]] +name = "espeak-rs-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d45d148019084e930df6cc3964a58c4c211342451ec5d3c328d8a6cc6b3464d" +dependencies = [ + "bindgen", + "cmake", + "glob", +] + [[package]] name = "etcetera" version = "0.11.0" @@ -1826,6 +1905,21 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "hmac-sha256" +version = "1.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f" + +[[package]] +name = "home" +version = "0.5.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "hpke-rs" version = "0.6.1" @@ -2012,7 +2106,7 @@ version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -2322,6 +2416,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "lebe" version = "0.5.3" @@ -2566,6 +2666,16 @@ dependencies = [ "cc", ] +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + [[package]] name = "libm" version = "0.2.16" @@ -2594,6 +2704,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.12.1" @@ -2651,6 +2767,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" +[[package]] +name = "lzma-rust2" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619" + [[package]] name = "markup5ever" version = "0.39.0" @@ -2738,6 +2860,12 @@ dependencies = [ "triomphe", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.8.9" @@ -2819,6 +2947,36 @@ dependencies = [ "tempfile", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "ndarray" +version = "0.17.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2837,6 +2995,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nom" version = "8.0.0" @@ -3121,6 +3289,30 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ort" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133" +dependencies = [ + "ndarray 0.17.2", + "ort-sys", + "smallvec", + "tracing", + "ureq", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90" +dependencies = [ + "hmac-sha256", + "lzma-rust2", + "ureq", +] + [[package]] name = "owned_ttf_parser" version = "0.25.1" @@ -3246,6 +3438,15 @@ dependencies = [ "base64ct", ] +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -3397,13 +3598,26 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "piper-rs" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b53db829904e6c2959ee6ff4922401a9eccd23c7bd6582bdd8ebec627a953c9b" +dependencies = [ + "espeak-rs", + "ndarray 0.16.1", + "ort", + "serde", + "serde_json", +] + [[package]] name = "pkcs8" version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", + "der 0.7.10", "spki", ] @@ -3543,6 +3757,16 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.118", +] + [[package]] name = "primal-check" version = "0.3.4" @@ -3663,7 +3887,7 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash", + "rustc-hash 2.1.2", "rustls 0.23.40", "socket2 0.6.4", "thiserror 2.0.18", @@ -3683,7 +3907,7 @@ dependencies = [ "lru-slab", "rand 0.9.4", "ring", - "rustc-hash", + "rustc-hash 2.1.2", "rustls 0.23.40", "rustls-pki-types", "slab", @@ -3968,7 +4192,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", "futures-core", @@ -4063,6 +4287,12 @@ dependencies = [ "realfft", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-hash" version = "2.1.2" @@ -4092,6 +4322,19 @@ dependencies = [ "transpose", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.4" @@ -4101,7 +4344,7 @@ dependencies = [ "bitflags 2.13.0", "errno", "libc", - "linux-raw-sys", + "linux-raw-sys 0.12.1", "windows-sys 0.61.2", ] @@ -4320,7 +4563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct", - "der", + "der 0.7.10", "generic-array", "pkcs8", "subtle", @@ -4374,7 +4617,7 @@ dependencies = [ "phf 0.13.1", "phf_codegen", "precomputed-hash", - "rustc-hash", + "rustc-hash 2.1.2", "servo_arc", "smallvec", ] @@ -4395,10 +4638,12 @@ version = "0.4.1" dependencies = [ "chrono", "cron", + "dashmap 6.2.1", "dotenvy", "futures-util", "glob", "image", + "piper-rs", "poise", "qrcode", "rand 0.10.1", @@ -4544,7 +4789,7 @@ checksum = "9bde37f42765dfdc34e2a039e0c84afbf79a3101c1941763b0beb816c2f17541" dependencies = [ "arrayvec", "async-trait", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "bytes", "chrono", @@ -4557,7 +4802,7 @@ dependencies = [ "parking_lot 0.12.5", "percent-encoding", "reqwest", - "rustc-hash", + "rustc-hash 2.1.2", "secrecy", "serde", "serde_cow", @@ -4666,6 +4911,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "shlex" version = "2.0.1" @@ -4794,6 +5045,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "songbird" version = "0.6.0" @@ -4863,7 +5125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", ] [[package]] @@ -4885,7 +5147,7 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "cfg-if", "chrono", @@ -4987,7 +5249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags 2.13.0", "byteorder", "chrono", @@ -5546,7 +5808,7 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix", + "rustix 1.1.4", "windows-sys 0.61.2", ] @@ -5820,7 +6082,7 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fcaf159b4e7a376b05b5bfd77bfd38f3324f5fce751b4213bfc7eaa47affb4e" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "fastrand", "futures-core", @@ -6207,6 +6469,36 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d" +dependencies = [ + "base64 0.23.1", + "der 0.8.2", + "log", + "native-tls", + "percent-encoding", + "rustls-pki-types", + "socks", + "ureq-proto", + "utf8-zero", + "webpki-root-certs", +] + +[[package]] +name = "ureq-proto" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613" +dependencies = [ + "base64 0.23.1", + "http", + "httparse", + "log", +] + [[package]] name = "uriparse" version = "0.6.4" @@ -6242,6 +6534,12 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" +[[package]] +name = "utf8-zero" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -6434,6 +6732,15 @@ dependencies = [ "string_cache_codegen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "webpki-roots" version = "0.26.11" @@ -6458,6 +6765,18 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "2.1.2" diff --git a/Cargo.toml b/Cargo.toml index 9049eef..a1d8b5b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -76,3 +76,5 @@ uuid = { version = "1.23.3", features = ["v4"] } glob = "0.3.3" tempfile = "3.27" futures-util = "0.3" +piper-rs = "0.2.0" +dashmap = "6.2.1" diff --git a/build.rs b/build.rs new file mode 100644 index 0000000..d152090 --- /dev/null +++ b/build.rs @@ -0,0 +1,52 @@ +fn main() { + // espeak-rs-sys builds espeak-ng from source but misses linking sonic + // and the speechPlayer audio_object symbols. piper-rs only uses + // espeak-ng for phonemization (never audio output), so stub implementations + // satisfy the linker without providing real functionality. + + let out_dir = std::env::var("OUT_DIR").unwrap(); + let stubs_c = format!("{}/tts_stubs.c", out_dir); + let stubs_o = format!("{}/tts_stubs.o", out_dir); + let stubs_a = format!("{}/libtts_stubs.a", out_dir); + + std::fs::write( + &stubs_c, + r#" +#include +typedef void* sonicStream; +sonicStream sonicCreateStream(int sampleRate, int channels) { return NULL; } +void sonicDestroyStream(sonicStream s) {} +void sonicSetSpeed(sonicStream s, float speed) {} +float sonicGetSpeed(sonicStream s) { return 1.0f; } +void sonicWriteShortToStream(sonicStream s, const short *samples, int n) {} +int sonicReadShortFromStream(sonicStream s, short *samples, int n) { return 0; } +void sonicFlushStream(sonicStream s) {} +typedef void* audio_object_t; +int audio_object_open(audio_object_t o) { return 0; } +int audio_object_write(audio_object_t o, const void *buf, size_t n) { return (int)n; } +int audio_object_close(audio_object_t o) { return 0; } +int audio_object_flush(audio_object_t o) { return 0; } +int audio_object_drain(audio_object_t o) { return 0; } +const char* audio_object_strerror(audio_object_t o, int err) { return "stub"; } +audio_object_t create_audio_device_object(void* d, const char* device) { return NULL; } +"#, + ) + .expect("write stubs"); + + // cc -c + let status = std::process::Command::new("cc") + .args(["-c", &stubs_c, "-o", &stubs_o]) + .status() + .expect("run cc"); + assert!(status.success(), "cc -c failed"); + + // ar rcs + let status = std::process::Command::new("ar") + .args(["rcs", &stubs_a, &stubs_o]) + .status() + .expect("run ar"); + assert!(status.success(), "ar rcs failed"); + + println!("cargo:rustc-link-search=native={}", out_dir); + println!("cargo:rustc-link-lib=static=tts_stubs"); +} diff --git a/src/commands/ai.rs b/src/commands/ai.rs index 61fb471..5dd98cb 100644 --- a/src/commands/ai.rs +++ b/src/commands/ai.rs @@ -30,15 +30,14 @@ pub async fn senator( }) .ok_or_else(lang::err_not_in_voice)?; - let (audio_data, _) = - crate::tts::synthesize(&ctx.data().config, &ctx.data().http_client, &response).await?; + let audio_data = ctx.data().tts.synthesize(&response).await?; let call = ctx .data() .voice_manager .connect(guild_id, channel_id) .await?; - let input = crate::voice::decode_audio(&audio_data)?; + let input = crate::tts::pcm_bytes_to_input(audio_data); let mut call_lock = call.lock().await; if call_lock.current_channel().is_some() { call_lock.play_input(input); diff --git a/src/commands/tts.rs b/src/commands/tts.rs index 8d20179..f45d2e8 100644 --- a/src/commands/tts.rs +++ b/src/commands/tts.rs @@ -15,7 +15,7 @@ pub enum OutputMode { Vc, } -/// Generate TTS (Piper locally, Google as fallback) and output to file or voice channel +/// Generate TTS (local Piper) and output to file or voice channel #[poise::command(slash_command)] pub async fn tts( ctx: Context<'_>, @@ -25,21 +25,21 @@ pub async fn tts( ctx.defer().await?; let output = output.unwrap_or(OutputMode::Vc); - let (audio_data, ext) = - crate::tts::synthesize(&ctx.data().config, &ctx.data().http_client, &text).await?; + let audio_data = ctx.data().tts.synthesize(&text).await?; match output { OutputMode::File => { + let wav = crate::tts::encode_wav(&audio_data, 48000, 2); + tokio::fs::create_dir_all(crate::config::TEMP_PATH) .await .ok(); let path = format!( - "{}/tts_{}.{}", + "{}/tts_{}.wav", crate::config::TEMP_PATH, uuid::Uuid::new_v4(), - ext ); - tokio::fs::write(&path, &audio_data).await?; + tokio::fs::write(&path, &wav).await?; if let Err(e) = ctx .send( poise::CreateReply::default() @@ -69,7 +69,7 @@ pub async fn tts( .voice_manager .connect(guild_id, channel_id) .await?; - let input = crate::voice::decode_audio(&audio_data)?; + let input = crate::tts::pcm_bytes_to_input(audio_data); let mut call_lock = call.lock().await; if call_lock.current_channel().is_some() { call_lock.play_input(input); diff --git a/src/events/mod.rs b/src/events/mod.rs index e4f1b22..87333c8 100644 --- a/src/events/mod.rs +++ b/src/events/mod.rs @@ -28,15 +28,14 @@ fn format_duration(seconds: f64) -> String { } async fn play_tts_in_channel( - config: crate::config::Config, - http_client: reqwest::Client, + tts: std::sync::Arc, voice_manager: crate::voice::VoiceManager, guild_id: serenity::GuildId, channel_id: serenity::ChannelId, text: String, ) { - let audio_data = match crate::tts::synthesize(&config, &http_client, &text).await { - Ok((bytes, _)) => bytes, + let audio_data = match tts.synthesize(&text).await { + Ok(bytes) => bytes, Err(e) => { warn!("Failed to synthesize TTS: {e}"); return; @@ -51,13 +50,7 @@ async fn play_tts_in_channel( } }; - let input = match crate::voice::decode_audio(&audio_data) { - Ok(i) => i, - Err(e) => { - warn!("Failed to decode TTS audio: {e}"); - return; - } - }; + let input = crate::tts::pcm_bytes_to_input(audio_data); let mut call_lock = call.lock().await; if call_lock.current_channel().is_some() { @@ -97,6 +90,16 @@ pub async fn handle_voice_state_update( .map(|m| m.display_name().to_string()) .unwrap_or_default(); + // Pre-generate all 6 event phrases for this user (background, non-blocking). + // First event per user pays ~120ms; subsequent events are instant from cache. + { + let tts = data.tts.clone(); + let name = display_name.clone(); + tokio::spawn(async move { + tts.pre_generate(&name).await; + }); + } + let now = current_ts as f64; let user_id_str = user_id.to_string(); @@ -293,8 +296,7 @@ pub async fn handle_voice_state_update( && let Some(cid) = tts_channel.or(new_channel) { tokio::spawn(play_tts_in_channel( - data.config.clone(), - data.http_client.clone(), + data.tts.clone(), data.voice_manager.clone(), gid, cid, @@ -314,8 +316,7 @@ pub async fn handle_voice_state_update( && let Some(cid) = tts_channel.or(new_channel) { tokio::spawn(play_tts_in_channel( - data.config.clone(), - data.http_client.clone(), + data.tts.clone(), data.voice_manager.clone(), gid, cid, @@ -330,8 +331,7 @@ pub async fn handle_voice_state_update( && let Some(gid) = guild_id { tokio::spawn(play_tts_in_channel( - data.config.clone(), - data.http_client.clone(), + data.tts.clone(), data.voice_manager.clone(), gid, cid, diff --git a/src/lang.rs b/src/lang.rs index de0c7d7..53888eb 100644 --- a/src/lang.rs +++ b/src/lang.rs @@ -114,13 +114,6 @@ pub fn fmt_voice_join_failed(e: &dyn std::fmt::Display) -> String { pub const L_TTS_SAVED: &str = "TTS zapisany"; pub const L_TTS_PLAYING: &str = "Grom TTS na kanale głosowym"; -pub fn fmt_tts_piper_launch(e: &str) -> String { - format!("Piper sie nie uruchomił w czas: `{e}`") -} -pub fn fmt_tts_piper_failed(e: &str) -> String { - format!("Piper TTS sie nie powiodło: `{e}`") -} - // ── Trigger command ─────────────────────────────────────────────────── pub const L_TRIGGER_UPDATED: &str = "Trigger zaktualizowany"; diff --git a/src/main.rs b/src/main.rs index 7f57421..5ab33ab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -50,6 +50,8 @@ pub struct Data { pub ac_search_cache: AcSearchCache, /// Active music track per guild (only one music source at a time). pub music_tracks: Arc>>, + /// TTS engine: piper model + audio cache. + pub tts: Arc, } impl Data { @@ -187,7 +189,10 @@ async fn main() -> Result<(), Box> { let music_tracks: Arc>> = Arc::new(tokio::sync::Mutex::new(HashMap::new())); + let tts = Arc::new(tts::TtsState::new(config.piper_model_path())); + let data = Data { + tts, config, db_pool, voice_manager: vm, diff --git a/src/tts.rs b/src/tts.rs index ded76ad..3ae5140 100644 --- a/src/tts.rs +++ b/src/tts.rs @@ -1,147 +1,291 @@ +use std::io::Cursor; use std::path::PathBuf; -use std::process::Stdio; -use std::time::Instant; +use std::sync::OnceLock; +use std::time::{Duration, Instant}; -use tokio::io::AsyncWriteExt; +use dashmap::DashMap; +use piper_rs::Piper; +use tokio::sync::Mutex; use tracing::{debug, info, warn}; -use crate::config; -use crate::config::TEMP_PATH; use crate::error::{Error, Result}; -/// Synthesize speech for `text`. -/// -/// Primary engine is local Piper TTS (via the `piper` CLI). If Piper is not -/// installed, fails, or `TTS_MODEL_PATH` is unset, we fall back to Google -/// Translate TTS over HTTP. +// ── Constants ────────────────────────────────────────────────────────── + +/// Target sample rate for songbird playback. +const TARGET_SAMPLE_RATE: u32 = 48_000; + +/// Cache entries older than this are evicted on access. +const CACHE_TTL: Duration = Duration::from_secs(3600); + +/// All voice-state TTS event types. +const EVENT_TYPES: &[&str] = &[ + "joined", + "left", + "deafened", + "undeafened", + "muted", + "unmuted", +]; + +// ── TtsState ─────────────────────────────────────────────────────────── + +/// Shared TTS state: piper model instance + audio cache. /// -/// Returns the encoded audio bytes plus the file extension to use when saving -/// (`"wav"` for Piper, `"mp3"` for Google). -pub async fn synthesize( - config: &config::Config, - http_client: &reqwest::Client, - text: &str, -) -> Result<(Vec, &'static str)> { - match config.piper_model_path() { - Some(model_path) => { - let started = Instant::now(); - match piper_synthesize(&model_path, text).await { - Ok(bytes) => { - info!( - model = %model_path.display(), - text_len = text.len(), - bytes = bytes.len(), - elapsed_ms = started.elapsed().as_millis() as u64, - "Piper TTS synthesized" - ); - return Ok((bytes, "wav")); - } - Err(e) => { - warn!( - model = %model_path.display(), - text_len = text.len(), - elapsed_ms = started.elapsed().as_millis() as u64, - "Piper TTS failed, falling back to Google TTS: {e}" - ); - } - } +/// Held in poise `Data`. Piper is loaded lazily on first synthesis call. +/// Cache maps full text string → resampled 48 kHz stereo PCM bytes. +pub struct TtsState { + piper: OnceLock>, + cache: DashMap, Instant)>, + model_path: Option, +} + +impl TtsState { + pub fn new(model_path: Option) -> Self { + Self { + piper: OnceLock::new(), + cache: DashMap::new(), + model_path, } - None => { - debug!("TTS_MODEL_PATH is not set; using Google TTS fallback"); + } + + /// Get or initialize the Piper model. Returns error if model path is + /// unset or model fails to load. + async fn get_piper(&self) -> Result<&Mutex> { + if let Some(piper) = self.piper.get() { + return Ok(piper); } + + let path = self + .model_path + .as_ref() + .ok_or_else(|| Error::Custom("TTS_MODEL_PATH is not set".into()))?; + + // Resolve .onnx.json config path from model path + let onnx_path = path.clone(); + let config_path = { + let s = path.to_string_lossy(); + if s.ends_with(".onnx") { + PathBuf::from(format!("{}.json", s)) + } else { + path.clone() + } + }; + + debug!( + onnx = %onnx_path.display(), + config = %config_path.display(), + "loading piper model" + ); + let piper = Piper::new(&onnx_path, &config_path).map_err(|e| { + Error::Custom(format!("Failed to load piper model: {e}")) + })?; + info!("piper model loaded successfully"); + + // set() may fail if another task initialized first — that's fine + let _ = self.piper.set(Mutex::new(piper)); + Ok(self.piper.get().unwrap()) } - let started = Instant::now(); - let bytes = google_synthesize(http_client, text).await?; - info!( - text_len = text.len(), - bytes = bytes.len(), - elapsed_ms = started.elapsed().as_millis() as u64, - "Google TTS synthesized" - ); - Ok((bytes, "mp3")) -} + /// Look up cached audio for a text string. Returns `None` if uncached + /// or expired. + fn cache_lookup(&self, text: &str) -> Option> { + let entry = self.cache.get(text)?; + if entry.1.elapsed() > CACHE_TTL { + drop(entry); + self.cache.remove(text); + None + } else { + Some(entry.0.clone()) + } + } + + /// Store audio bytes in the cache. + fn cache_store(&self, text: String, audio: Vec) { + self.cache.insert(text, (audio, Instant::now())); + } -/// Run `piper` as a subprocess, feeding `text` on stdin and collecting the -/// generated WAV from `--output_file`. -async fn piper_synthesize(model_path: &PathBuf, text: &str) -> Result> { - tokio::fs::create_dir_all(TEMP_PATH).await.ok(); - - let out_path = format!("{}/tts_piper_{}.wav", TEMP_PATH, uuid::Uuid::new_v4()); - let model = model_path.to_string_lossy().to_string(); - - debug!(model = %model, "spawning piper subprocess"); - let started = Instant::now(); - - let mut child = tokio::process::Command::new("piper") - .arg("--model") - .arg(&model) - .arg("--output_file") - .arg(&out_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .map_err(|e| Error::Custom(crate::lang::fmt_tts_piper_launch(&e.to_string())))?; - - { - let mut stdin = child - .stdin - .take() - .ok_or_else(|| Error::Custom("Piper stdin unavailable".into()))?; - stdin.write_all(text.as_bytes()).await?; - stdin.write_all(b"\n").await?; + /// Evict all expired cache entries. + fn cache_evict(&self) { + self.cache + .retain(|_, (_, inserted_at)| inserted_at.elapsed() <= CACHE_TTL); } - let output = child.wait_with_output().await?; - let elapsed_ms = started.elapsed().as_millis() as u64; + /// Synthesize text → resampled 48 kHz stereo PCM bytes. + /// + /// Checks cache first. On miss, runs piper inference, resamples, caches, + /// and returns. + pub async fn synthesize(&self, text: &str) -> Result> { + // Cache hit + if let Some(cached) = self.cache_lookup(text) { + debug!(text_len = text.len(), "tts cache hit"); + return Ok(cached); + } + + // Cache miss — synthesize + let piper = self.get_piper().await?; + let started = Instant::now(); - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - warn!( - model = %model, + let (samples, sample_rate) = { + let mut lock = piper.lock().await; + lock.create(text, false, None, None, None, None) + .map_err(|e| Error::Custom(format!("piper synthesis failed: {e}")))? + }; + + let elapsed_ms = started.elapsed().as_millis() as u64; + debug!( + text_len = text.len(), + samples = samples.len(), + sample_rate, elapsed_ms, - "piper exited with error: {}", - stderr.trim_end() + "piper synthesized" ); - let _ = tokio::fs::remove_file(&out_path).await; - return Err(Error::Custom(crate::lang::fmt_tts_piper_failed( - stderr.trim_end(), - ))); + + // Resample mono → stereo at target sample rate + let pcm_bytes = resample_to_stereo_bytes(&samples, sample_rate, TARGET_SAMPLE_RATE); + + self.cache_store(text.to_string(), pcm_bytes.clone()); + Ok(pcm_bytes) } - let bytes = tokio::fs::read(&out_path).await?; - debug!( - model = %model, - bytes = bytes.len(), - elapsed_ms, - "piper succeeded" - ); - let _ = tokio::fs::remove_file(&out_path).await; - Ok(bytes) + /// Pre-generate all 6 event phrases for a display name. + /// + /// Called on first voice event for a user. Synthesizes all phrases and + /// caches them so subsequent events are instant. + pub async fn pre_generate(&self, display_name: &str) { + let phrases: Vec = EVENT_TYPES + .iter() + .map(|evt| make_event_phrase(display_name, evt)) + .collect(); + + for phrase in &phrases { + if self.cache_lookup(phrase).is_some() { + continue; + } + if let Err(e) = self.synthesize(phrase).await { + warn!( + display_name, + phrase, "pre-generation failed: {e}" + ); + } + } + + // Evict expired entries while we're at it + self.cache_evict(); + } } -/// Fetch Google Translate TTS (Polish) over HTTP. -async fn google_synthesize(client: &reqwest::Client, text: &str) -> Result> { - let tts_url = format!( - "https://translate.google.com/translate_tts?ie=UTF-8&q={}&tl=pl&client=tw-ob", - urlencoding(text) - ); - debug!(url_len = tts_url.len(), "fetching Google TTS"); - let bytes = client.get(&tts_url).send().await?.bytes().await?; - Ok(bytes.to_vec()) +// ── Phrase construction ──────────────────────────────────────────────── + +/// Build the TTS phrase for a voice event + display name. +pub fn make_event_phrase(display_name: &str, event: &str) -> String { + let suffix = match event { + "joined" => "sie zespawnił", + "left" => "jest chujem", + "deafened" => "jest bity przez ojca", + "undeafened" => "uciekł od ojca", + "muted" => "idzie robić jajecznice", + "unmuted" => "zrobił jajecznice", + _ => return format!("{display_name} {event}"), + }; + format!("{display_name} {suffix}") } -pub fn urlencoding(s: &str) -> String { - let mut out = String::with_capacity(s.len()); - for &byte in s.as_bytes() { - match byte { - b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'-' | b'.' | b'~' => { - out.push(byte as char) - } - b' ' => out.push('+'), - _ => out.push_str(&format!("%{:02X}", byte)), +// ── Resampling ───────────────────────────────────────────────────────── + +/// Resample mono f32 PCM from `src_rate` to `dst_rate`, duplicate to stereo, +/// and return as little-endian bytes. +fn resample_to_stereo_bytes(samples: &[f32], src_rate: u32, dst_rate: u32) -> Vec { + if src_rate == dst_rate { + // Just duplicate mono → stereo, no resampling needed + let mut out = Vec::with_capacity(samples.len() * 2 * 4); + for &s in samples { + out.extend_from_slice(&s.to_le_bytes()); + out.extend_from_slice(&s.to_le_bytes()); } + return out; } + + let ratio = src_rate as f64 / dst_rate as f64; + let out_len = (samples.len() as f64 / ratio).ceil() as usize; + let mut out = Vec::with_capacity(out_len * 2 * 4); + + for i in 0..out_len { + let src_idx = i as f64 * ratio; + let idx = src_idx as usize; + let frac = src_idx - idx as f64; + + let sample = if idx + 1 < samples.len() { + // Linear interpolation + samples[idx] + (samples[idx + 1] - samples[idx]) * frac as f32 + } else if idx < samples.len() { + samples[idx] + } else { + 0.0 + }; + + // Mono → stereo duplication + out.extend_from_slice(&sample.to_le_bytes()); + out.extend_from_slice(&sample.to_le_bytes()); + } + out } + +// ── WAV encoding ─────────────────────────────────────────────────────── + +/// Encode raw f32 stereo PCM bytes into a WAV file (16-bit PCM). +/// +/// `pcm_f32_bytes` is the raw little-endian f32 interleaved stereo data +/// (e.g. from `resample_to_stereo_bytes`). +pub fn encode_wav(pcm_f32_bytes: &[u8], sample_rate: u32, channels: u16) -> Vec { + let bits_per_sample: u16 = 16; + let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8; + let block_align = channels * bits_per_sample / 8; + + // Convert f32 samples to i16 + let sample_count = pcm_f32_bytes.len() / 4; // 4 bytes per f32 + let mut pcm_i16 = Vec::with_capacity(sample_count * 2); + for chunk in pcm_f32_bytes.chunks_exact(4) { + let f = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + let i = (f * 32767.0).clamp(-32768.0, 32767.0) as i16; + pcm_i16.extend_from_slice(&i.to_le_bytes()); + } + + let data_size = pcm_i16.len() as u32; + let file_size = 36 + data_size; + + let mut wav = Vec::with_capacity(44 + pcm_i16.len()); + + // RIFF header + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&file_size.to_le_bytes()); + wav.extend_from_slice(b"WAVE"); + + // fmt chunk + wav.extend_from_slice(b"fmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); // chunk size + wav.extend_from_slice(&1u16.to_le_bytes()); // PCM format + wav.extend_from_slice(&channels.to_le_bytes()); + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&byte_rate.to_le_bytes()); + wav.extend_from_slice(&block_align.to_le_bytes()); + wav.extend_from_slice(&bits_per_sample.to_le_bytes()); + + // data chunk + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + wav.extend_from_slice(&pcm_i16); + + wav +} + +// ── Songbird Input helper ────────────────────────────────────────────── + +/// Create a songbird `Input` from resampled f32 stereo PCM bytes. +pub fn pcm_bytes_to_input(pcm_f32_bytes: Vec) -> songbird::input::Input { + use songbird::input::RawAdapter; + let cursor = Cursor::new(pcm_f32_bytes); + // 2 channels (stereo), 48 kHz — matches TARGET_SAMPLE_RATE + RawAdapter::new(cursor, TARGET_SAMPLE_RATE, 2).into() +} diff --git a/tests/tts_smoke.rs b/tests/tts_smoke.rs new file mode 100644 index 0000000..e98d56d --- /dev/null +++ b/tests/tts_smoke.rs @@ -0,0 +1,54 @@ +//! Smoke test: verify piper-rs loads the Polish model and synthesizes audio. + +use std::path::Path; + +#[test] +fn piper_load_and_synthesize() { + let model_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("piper"); + let onnx = model_dir.join("pl_PL-bass-high.onnx"); + let json = model_dir.join("pl_PL-bass-high.onnx.json"); + + assert!(onnx.exists(), "model file missing: {}", onnx.display()); + assert!(json.exists(), "config file missing: {}", json.display()); + + let mut piper = + piper_rs::Piper::new(&onnx, &json).expect("failed to load piper model"); + + let text = "Kamil sie zespawnił"; + let (samples, sample_rate) = piper + .create(text, false, None, None, None, None) + .expect("synthesis failed"); + + assert!(!samples.is_empty(), "no samples produced"); + assert!(sample_rate > 0, "invalid sample rate: {sample_rate}"); + assert!( + samples.len() as u32 > sample_rate / 10, + "suspiciously short output: {} samples at {} Hz", + samples.len(), + sample_rate + ); + + let duration = samples.len() as f64 / sample_rate as f64; + println!( + "OK: '{}' → {} samples @ {} Hz ({:.2}s)", + text, + samples.len(), + sample_rate, + duration, + ); + + // Second call measures cached-model synthesis speed + let start = std::time::Instant::now(); + let (samples2, sr2) = piper + .create("Kamil jest chujem", false, None, None, None, None) + .expect("second synthesis failed"); + let elapsed = start.elapsed(); + println!( + "Cached: '{}' → {} samples @ {} Hz in {:?}", + "Kamil jest chujem", + samples2.len(), + sr2, + elapsed, + ); + assert!(!samples2.is_empty()); +} -- cgit v1.2.3