aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorsalmonik <salmonik@tuta.io>2026-09-05 19:35:21 +0200
committersalmonik <salmonik@tuta.io>2026-09-05 19:35:21 +0200
commit8e868894cbcaf421c263f3555af4d29bef653420 (patch)
treea6b6f1de5caeef59367e52a8b7cfa6c2cd9f201d
parentremove ` from TTS and make events use lang.rs (diff)
downloadsenator-sperma-8e868894cbcaf421c263f3555af4d29bef653420.tar.gz
senator-sperma-8e868894cbcaf421c263f3555af4d29bef653420.zip
feat: piper now not a subprocess, cache system for TTS
Diffstat (limited to '')
-rw-r--r--AGENTS.md6
-rw-r--r--CONTEXT.md15
-rw-r--r--Cargo.lock359
-rw-r--r--Cargo.toml2
-rw-r--r--build.rs52
-rw-r--r--src/commands/ai.rs5
-rw-r--r--src/commands/tts.rs14
-rw-r--r--src/events/mod.rs34
-rw-r--r--src/lang.rs7
-rw-r--r--src/main.rs5
-rw-r--r--src/tts.rs384
-rw-r--r--tests/tts_smoke.rs54
12 files changed, 762 insertions, 175 deletions
diff --git a/AGENTS.md b/AGENTS.md
index a074a43..8352e0e 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -23,7 +23,7 @@ src/
23├── events/mod.rs — handle_ready, handle_voice_state_update, handle_message 23├── events/mod.rs — handle_ready, handle_voice_state_update, handle_message
24├── openrouter/mod.rs — OpenRouterClient (chat, chat_with_history) 24├── openrouter/mod.rs — OpenRouterClient (chat, chat_with_history)
25├── tasks/mod.rs — cron_scheduler_loop (30s polling, stub exec bodies) 25├── tasks/mod.rs — cron_scheduler_loop (30s polling, stub exec bodies)
26├── tts.rs — synthesize(): Piper subprocess → Google TTS fallback 26├── tts.rs — TtsState (piper-rs in-process, DashMap cache, pre-gen, resampler, WAV encoder)
27└── voice/mod.rs — VoiceManager (connect, disconnect, get_call), mp3_to_input 27└── voice/mod.rs — VoiceManager (connect, disconnect, get_call), mp3_to_input
28``` 28```
29 29
@@ -35,6 +35,10 @@ src/
35- **Shared state**: `Data` struct in `main.rs` — passed as poise `user_data` 35- **Shared state**: `Data` struct in `main.rs` — passed as poise `user_data`
36- **Commands**: each submodule exports `pub fn commands() -> Vec<poise::Command<Data, Error>>` 36- **Commands**: each submodule exports `pub fn commands() -> Vec<poise::Command<Data, Error>>`
37- **Temp path**: always use `crate::config::TEMP_PATH` (resolves to `/tmp/senator_sperma`) 37- **Temp path**: always use `crate::config::TEMP_PATH` (resolves to `/tmp/senator_sperma`)
38- **TTS**: piper-rs in-process (no subprocess). `TtsState` in `Data` holds `OnceLock<Mutex<Piper>>` + `DashMap` cache. Synthesize via `data.tts.synthesize(text)`. Pre-generates 6 event phrases per user on first voice event. No Google TTS fallback.
39
40## Build
41- `build.rs` — compiles C stubs for espeak-rs-sys missing sonic/audio_object symbols
38 42
39## Stubs to Fill (see COMPARISON.md) 43## Stubs to Fill (see COMPARISON.md)
40- `commands/music.rs` — `/play` is placeholder 44- `commands/music.rs` — `/play` is placeholder
diff --git a/CONTEXT.md b/CONTEXT.md
index e272a7a..bdd2790 100644
--- a/CONTEXT.md
+++ b/CONTEXT.md
@@ -32,3 +32,18 @@ holds no implementation decisions or specs (those live in `docs/adr/`).
32 afk). Paged at 10 entries, descending by default. 32 afk). Paged at 10 entries, descending by default.
33- **Tracked guild** — the guild containing `status_channel_id`. Message counts 33- **Tracked guild** — the guild containing `status_channel_id`. Message counts
34 are only tracked in this guild. 34 are only tracked in this guild.
35
36## TTS
37
38- **Synthesize** — convert text to audio via local Piper TTS (ONNX model
39 loaded in-process via `piper-rs`). No external fallback. Output is f32 PCM
40 at the model's native sample rate (typically 22050 Hz mono), resampled to
41 48 kHz stereo via inline linear interpolation before playback. Songbird does
42 not resample — playing un-resampled audio results in wrong playback speed.
43- **TtsState** — struct held in poise `Data`: `OnceCell<Mutex<Piper>>` for the
44 model instance, `DashMap<String, (Vec<u8>, Instant)>` for the text→audio cache.
45 Piper loaded lazily on first TTS call. Cache keyed by full phrase string,
46 evicted by TTL. No Google TTS fallback.
47- **Pre-generation** — on first voice event for a user, synthesize all 6 event
48 phrases (join/leave/deafen/undeafen/mute/unmute) with their display name and
49 cache them. ~120ms first time, instant after. Lazy — no startup cost.
diff --git a/Cargo.lock b/Cargo.lock
index caf04ae..7ab8ebc 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -174,7 +174,7 @@ dependencies = [
174 "anyhow", 174 "anyhow",
175 "arrayvec", 175 "arrayvec",
176 "log", 176 "log",
177 "nom", 177 "nom 8.0.0",
178 "num-rational", 178 "num-rational",
179 "v_frame", 179 "v_frame",
180] 180]
@@ -223,12 +223,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
223checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" 223checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
224 224
225[[package]] 225[[package]]
226name = "base64"
227version = "0.23.1"
228source = "registry+https://github.com/rust-lang/crates.io-index"
229checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5"
230
231[[package]]
226name = "base64ct" 232name = "base64ct"
227version = "1.8.3" 233version = "1.8.3"
228source = "registry+https://github.com/rust-lang/crates.io-index" 234source = "registry+https://github.com/rust-lang/crates.io-index"
229checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" 235checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06"
230 236
231[[package]] 237[[package]]
238name = "bindgen"
239version = "0.69.5"
240source = "registry+https://github.com/rust-lang/crates.io-index"
241checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088"
242dependencies = [
243 "bitflags 2.13.0",
244 "cexpr",
245 "clang-sys",
246 "itertools",
247 "lazy_static",
248 "lazycell",
249 "log",
250 "prettyplease",
251 "proc-macro2",
252 "quote",
253 "regex",
254 "rustc-hash 1.1.0",
255 "shlex 1.3.0",
256 "syn 2.0.118",
257 "which",
258]
259
260[[package]]
232name = "bit-set" 261name = "bit-set"
233version = "0.8.0" 262version = "0.8.0"
234source = "registry+https://github.com/rust-lang/crates.io-index" 263source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -370,7 +399,16 @@ dependencies = [
370 "find-msvc-tools", 399 "find-msvc-tools",
371 "jobserver", 400 "jobserver",
372 "libc", 401 "libc",
373 "shlex", 402 "shlex 2.0.1",
403]
404
405[[package]]
406name = "cexpr"
407version = "0.6.0"
408source = "registry+https://github.com/rust-lang/crates.io-index"
409checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
410dependencies = [
411 "nom 7.1.3",
374] 412]
375 413
376[[package]] 414[[package]]
@@ -466,6 +504,17 @@ dependencies = [
466] 504]
467 505
468[[package]] 506[[package]]
507name = "clang-sys"
508version = "1.9.1"
509source = "registry+https://github.com/rust-lang/crates.io-index"
510checksum = "157a8ba7b480713b56f4c09fd13fc3e0a22a5dfab8097ba61cbc5feef950788a"
511dependencies = [
512 "glob",
513 "libc",
514 "libloading",
515]
516
517[[package]]
469name = "cmake" 518name = "cmake"
470version = "0.1.58" 519version = "0.1.58"
471source = "registry+https://github.com/rust-lang/crates.io-index" 520source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -926,7 +975,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
926checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" 975checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb"
927dependencies = [ 976dependencies = [
928 "const-oid", 977 "const-oid",
929 "pem-rfc7468", 978 "pem-rfc7468 0.7.0",
979 "zeroize",
980]
981
982[[package]]
983name = "der"
984version = "0.8.2"
985source = "registry+https://github.com/rust-lang/crates.io-index"
986checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a"
987dependencies = [
988 "pem-rfc7468 1.0.0",
930 "zeroize", 989 "zeroize",
931] 990]
932 991
@@ -1086,7 +1145,7 @@ version = "0.16.9"
1086source = "registry+https://github.com/rust-lang/crates.io-index" 1145source = "registry+https://github.com/rust-lang/crates.io-index"
1087checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" 1146checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca"
1088dependencies = [ 1147dependencies = [
1089 "der", 1148 "der 0.7.10",
1090 "digest 0.10.7", 1149 "digest 0.10.7",
1091 "elliptic-curve", 1150 "elliptic-curve",
1092 "rfc6979", 1151 "rfc6979",
@@ -1147,7 +1206,7 @@ dependencies = [
1147 "generic-array", 1206 "generic-array",
1148 "group", 1207 "group",
1149 "hkdf 0.12.4", 1208 "hkdf 0.12.4",
1150 "pem-rfc7468", 1209 "pem-rfc7468 0.7.0",
1151 "pkcs8", 1210 "pkcs8",
1152 "rand_core 0.6.4", 1211 "rand_core 0.6.4",
1153 "sec1", 1212 "sec1",
@@ -1230,6 +1289,26 @@ dependencies = [
1230] 1289]
1231 1290
1232[[package]] 1291[[package]]
1292name = "espeak-rs"
1293version = "0.2.0"
1294source = "registry+https://github.com/rust-lang/crates.io-index"
1295checksum = "89534f20ce914016aae6cc63e9476aad41cee3e96b48e712e1702caf24c0c538"
1296dependencies = [
1297 "espeak-rs-sys",
1298]
1299
1300[[package]]
1301name = "espeak-rs-sys"
1302version = "0.2.0"
1303source = "registry+https://github.com/rust-lang/crates.io-index"
1304checksum = "2d45d148019084e930df6cc3964a58c4c211342451ec5d3c328d8a6cc6b3464d"
1305dependencies = [
1306 "bindgen",
1307 "cmake",
1308 "glob",
1309]
1310
1311[[package]]
1233name = "etcetera" 1312name = "etcetera"
1234version = "0.11.0" 1313version = "0.11.0"
1235source = "registry+https://github.com/rust-lang/crates.io-index" 1314source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1827,6 +1906,21 @@ dependencies = [
1827] 1906]
1828 1907
1829[[package]] 1908[[package]]
1909name = "hmac-sha256"
1910version = "1.1.14"
1911source = "registry+https://github.com/rust-lang/crates.io-index"
1912checksum = "ec9d92d097f4749b64e8cc33d924d9f40a2d4eb91402b458014b781f5733d60f"
1913
1914[[package]]
1915name = "home"
1916version = "0.5.12"
1917source = "registry+https://github.com/rust-lang/crates.io-index"
1918checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d"
1919dependencies = [
1920 "windows-sys 0.61.2",
1921]
1922
1923[[package]]
1830name = "hpke-rs" 1924name = "hpke-rs"
1831version = "0.6.1" 1925version = "0.6.1"
1832source = "registry+https://github.com/rust-lang/crates.io-index" 1926source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2012,7 +2106,7 @@ version = "0.1.20"
2012source = "registry+https://github.com/rust-lang/crates.io-index" 2106source = "registry+https://github.com/rust-lang/crates.io-index"
2013checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" 2107checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
2014dependencies = [ 2108dependencies = [
2015 "base64", 2109 "base64 0.22.1",
2016 "bytes", 2110 "bytes",
2017 "futures-channel", 2111 "futures-channel",
2018 "futures-util", 2112 "futures-util",
@@ -2323,6 +2417,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
2323checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" 2417checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
2324 2418
2325[[package]] 2419[[package]]
2420name = "lazycell"
2421version = "1.3.0"
2422source = "registry+https://github.com/rust-lang/crates.io-index"
2423checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
2424
2425[[package]]
2326name = "lebe" 2426name = "lebe"
2327version = "0.5.3" 2427version = "0.5.3"
2328source = "registry+https://github.com/rust-lang/crates.io-index" 2428source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2567,6 +2667,16 @@ dependencies = [
2567] 2667]
2568 2668
2569[[package]] 2669[[package]]
2670name = "libloading"
2671version = "0.8.9"
2672source = "registry+https://github.com/rust-lang/crates.io-index"
2673checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
2674dependencies = [
2675 "cfg-if",
2676 "windows-link",
2677]
2678
2679[[package]]
2570name = "libm" 2680name = "libm"
2571version = "0.2.16" 2681version = "0.2.16"
2572source = "registry+https://github.com/rust-lang/crates.io-index" 2682source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2596,6 +2706,12 @@ dependencies = [
2596 2706
2597[[package]] 2707[[package]]
2598name = "linux-raw-sys" 2708name = "linux-raw-sys"
2709version = "0.4.15"
2710source = "registry+https://github.com/rust-lang/crates.io-index"
2711checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab"
2712
2713[[package]]
2714name = "linux-raw-sys"
2599version = "0.12.1" 2715version = "0.12.1"
2600source = "registry+https://github.com/rust-lang/crates.io-index" 2716source = "registry+https://github.com/rust-lang/crates.io-index"
2601checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" 2717checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53"
@@ -2652,6 +2768,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
2652checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" 2768checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
2653 2769
2654[[package]] 2770[[package]]
2771name = "lzma-rust2"
2772version = "0.15.8"
2773source = "registry+https://github.com/rust-lang/crates.io-index"
2774checksum = "e20f57f9918e5bd7bc58c22cdd70a6afc7375d4dd9683af5f2b34bd3d2bba619"
2775
2776[[package]]
2655name = "markup5ever" 2777name = "markup5ever"
2656version = "0.39.0" 2778version = "0.39.0"
2657source = "registry+https://github.com/rust-lang/crates.io-index" 2779source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2739,6 +2861,12 @@ dependencies = [
2739] 2861]
2740 2862
2741[[package]] 2863[[package]]
2864name = "minimal-lexical"
2865version = "0.2.1"
2866source = "registry+https://github.com/rust-lang/crates.io-index"
2867checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
2868
2869[[package]]
2742name = "miniz_oxide" 2870name = "miniz_oxide"
2743version = "0.8.9" 2871version = "0.8.9"
2744source = "registry+https://github.com/rust-lang/crates.io-index" 2872source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2820,6 +2948,36 @@ dependencies = [
2820] 2948]
2821 2949
2822[[package]] 2950[[package]]
2951name = "ndarray"
2952version = "0.16.1"
2953source = "registry+https://github.com/rust-lang/crates.io-index"
2954checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
2955dependencies = [
2956 "matrixmultiply",
2957 "num-complex",
2958 "num-integer",
2959 "num-traits",
2960 "portable-atomic",
2961 "portable-atomic-util",
2962 "rawpointer",
2963]
2964
2965[[package]]
2966name = "ndarray"
2967version = "0.17.2"
2968source = "registry+https://github.com/rust-lang/crates.io-index"
2969checksum = "520080814a7a6b4a6e9070823bb24b4531daac8c4627e08ba5de8c5ef2f2752d"
2970dependencies = [
2971 "matrixmultiply",
2972 "num-complex",
2973 "num-integer",
2974 "num-traits",
2975 "portable-atomic",
2976 "portable-atomic-util",
2977 "rawpointer",
2978]
2979
2980[[package]]
2823name = "new_debug_unreachable" 2981name = "new_debug_unreachable"
2824version = "1.0.6" 2982version = "1.0.6"
2825source = "registry+https://github.com/rust-lang/crates.io-index" 2983source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2839,6 +2997,16 @@ checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451"
2839 2997
2840[[package]] 2998[[package]]
2841name = "nom" 2999name = "nom"
3000version = "7.1.3"
3001source = "registry+https://github.com/rust-lang/crates.io-index"
3002checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
3003dependencies = [
3004 "memchr",
3005 "minimal-lexical",
3006]
3007
3008[[package]]
3009name = "nom"
2842version = "8.0.0" 3010version = "8.0.0"
2843source = "registry+https://github.com/rust-lang/crates.io-index" 3011source = "registry+https://github.com/rust-lang/crates.io-index"
2844checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" 3012checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405"
@@ -3122,6 +3290,30 @@ dependencies = [
3122] 3290]
3123 3291
3124[[package]] 3292[[package]]
3293name = "ort"
3294version = "2.0.0-rc.12"
3295source = "registry+https://github.com/rust-lang/crates.io-index"
3296checksum = "d7de3af33d24a745ffb8fab904b13478438d1cd52868e6f17735ef6e1f8bf133"
3297dependencies = [
3298 "ndarray 0.17.2",
3299 "ort-sys",
3300 "smallvec",
3301 "tracing",
3302 "ureq",
3303]
3304
3305[[package]]
3306name = "ort-sys"
3307version = "2.0.0-rc.12"
3308source = "registry+https://github.com/rust-lang/crates.io-index"
3309checksum = "d7b497d21a8b6fbb4b5a544f8fadb77e801a09ae0add9e411d31c6f89e3c1e90"
3310dependencies = [
3311 "hmac-sha256",
3312 "lzma-rust2",
3313 "ureq",
3314]
3315
3316[[package]]
3125name = "owned_ttf_parser" 3317name = "owned_ttf_parser"
3126version = "0.25.1" 3318version = "0.25.1"
3127source = "registry+https://github.com/rust-lang/crates.io-index" 3319source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3247,6 +3439,15 @@ dependencies = [
3247] 3439]
3248 3440
3249[[package]] 3441[[package]]
3442name = "pem-rfc7468"
3443version = "1.0.0"
3444source = "registry+https://github.com/rust-lang/crates.io-index"
3445checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9"
3446dependencies = [
3447 "base64ct",
3448]
3449
3450[[package]]
3250name = "percent-encoding" 3451name = "percent-encoding"
3251version = "2.3.2" 3452version = "2.3.2"
3252source = "registry+https://github.com/rust-lang/crates.io-index" 3453source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3398,12 +3599,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3398checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 3599checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
3399 3600
3400[[package]] 3601[[package]]
3602name = "piper-rs"
3603version = "0.2.0"
3604source = "registry+https://github.com/rust-lang/crates.io-index"
3605checksum = "b53db829904e6c2959ee6ff4922401a9eccd23c7bd6582bdd8ebec627a953c9b"
3606dependencies = [
3607 "espeak-rs",
3608 "ndarray 0.16.1",
3609 "ort",
3610 "serde",
3611 "serde_json",
3612]
3613
3614[[package]]
3401name = "pkcs8" 3615name = "pkcs8"
3402version = "0.10.2" 3616version = "0.10.2"
3403source = "registry+https://github.com/rust-lang/crates.io-index" 3617source = "registry+https://github.com/rust-lang/crates.io-index"
3404checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" 3618checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7"
3405dependencies = [ 3619dependencies = [
3406 "der", 3620 "der 0.7.10",
3407 "spki", 3621 "spki",
3408] 3622]
3409 3623
@@ -3544,6 +3758,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
3544checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" 3758checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
3545 3759
3546[[package]] 3760[[package]]
3761name = "prettyplease"
3762version = "0.2.37"
3763source = "registry+https://github.com/rust-lang/crates.io-index"
3764checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
3765dependencies = [
3766 "proc-macro2",
3767 "syn 2.0.118",
3768]
3769
3770[[package]]
3547name = "primal-check" 3771name = "primal-check"
3548version = "0.3.4" 3772version = "0.3.4"
3549source = "registry+https://github.com/rust-lang/crates.io-index" 3773source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -3663,7 +3887,7 @@ dependencies = [
3663 "pin-project-lite", 3887 "pin-project-lite",
3664 "quinn-proto", 3888 "quinn-proto",
3665 "quinn-udp", 3889 "quinn-udp",
3666 "rustc-hash", 3890 "rustc-hash 2.1.2",
3667 "rustls 0.23.40", 3891 "rustls 0.23.40",
3668 "socket2 0.6.4", 3892 "socket2 0.6.4",
3669 "thiserror 2.0.18", 3893 "thiserror 2.0.18",
@@ -3683,7 +3907,7 @@ dependencies = [
3683 "lru-slab", 3907 "lru-slab",
3684 "rand 0.9.4", 3908 "rand 0.9.4",
3685 "ring", 3909 "ring",
3686 "rustc-hash", 3910 "rustc-hash 2.1.2",
3687 "rustls 0.23.40", 3911 "rustls 0.23.40",
3688 "rustls-pki-types", 3912 "rustls-pki-types",
3689 "slab", 3913 "slab",
@@ -3968,7 +4192,7 @@ version = "0.12.28"
3968source = "registry+https://github.com/rust-lang/crates.io-index" 4192source = "registry+https://github.com/rust-lang/crates.io-index"
3969checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" 4193checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147"
3970dependencies = [ 4194dependencies = [
3971 "base64", 4195 "base64 0.22.1",
3972 "bytes", 4196 "bytes",
3973 "encoding_rs", 4197 "encoding_rs",
3974 "futures-core", 4198 "futures-core",
@@ -4065,6 +4289,12 @@ dependencies = [
4065 4289
4066[[package]] 4290[[package]]
4067name = "rustc-hash" 4291name = "rustc-hash"
4292version = "1.1.0"
4293source = "registry+https://github.com/rust-lang/crates.io-index"
4294checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
4295
4296[[package]]
4297name = "rustc-hash"
4068version = "2.1.2" 4298version = "2.1.2"
4069source = "registry+https://github.com/rust-lang/crates.io-index" 4299source = "registry+https://github.com/rust-lang/crates.io-index"
4070checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe" 4300checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
@@ -4094,6 +4324,19 @@ dependencies = [
4094 4324
4095[[package]] 4325[[package]]
4096name = "rustix" 4326name = "rustix"
4327version = "0.38.44"
4328source = "registry+https://github.com/rust-lang/crates.io-index"
4329checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154"
4330dependencies = [
4331 "bitflags 2.13.0",
4332 "errno",
4333 "libc",
4334 "linux-raw-sys 0.4.15",
4335 "windows-sys 0.52.0",
4336]
4337
4338[[package]]
4339name = "rustix"
4097version = "1.1.4" 4340version = "1.1.4"
4098source = "registry+https://github.com/rust-lang/crates.io-index" 4341source = "registry+https://github.com/rust-lang/crates.io-index"
4099checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" 4342checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190"
@@ -4101,7 +4344,7 @@ dependencies = [
4101 "bitflags 2.13.0", 4344 "bitflags 2.13.0",
4102 "errno", 4345 "errno",
4103 "libc", 4346 "libc",
4104 "linux-raw-sys", 4347 "linux-raw-sys 0.12.1",
4105 "windows-sys 0.61.2", 4348 "windows-sys 0.61.2",
4106] 4349]
4107 4350
@@ -4320,7 +4563,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
4320checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" 4563checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
4321dependencies = [ 4564dependencies = [
4322 "base16ct", 4565 "base16ct",
4323 "der", 4566 "der 0.7.10",
4324 "generic-array", 4567 "generic-array",
4325 "pkcs8", 4568 "pkcs8",
4326 "subtle", 4569 "subtle",
@@ -4374,7 +4617,7 @@ dependencies = [
4374 "phf 0.13.1", 4617 "phf 0.13.1",
4375 "phf_codegen", 4618 "phf_codegen",
4376 "precomputed-hash", 4619 "precomputed-hash",
4377 "rustc-hash", 4620 "rustc-hash 2.1.2",
4378 "servo_arc", 4621 "servo_arc",
4379 "smallvec", 4622 "smallvec",
4380] 4623]
@@ -4395,10 +4638,12 @@ version = "0.4.1"
4395dependencies = [ 4638dependencies = [
4396 "chrono", 4639 "chrono",
4397 "cron", 4640 "cron",
4641 "dashmap 6.2.1",
4398 "dotenvy", 4642 "dotenvy",
4399 "futures-util", 4643 "futures-util",
4400 "glob", 4644 "glob",
4401 "image", 4645 "image",
4646 "piper-rs",
4402 "poise", 4647 "poise",
4403 "qrcode", 4648 "qrcode",
4404 "rand 0.10.1", 4649 "rand 0.10.1",
@@ -4544,7 +4789,7 @@ checksum = "9bde37f42765dfdc34e2a039e0c84afbf79a3101c1941763b0beb816c2f17541"
4544dependencies = [ 4789dependencies = [
4545 "arrayvec", 4790 "arrayvec",
4546 "async-trait", 4791 "async-trait",
4547 "base64", 4792 "base64 0.22.1",
4548 "bitflags 2.13.0", 4793 "bitflags 2.13.0",
4549 "bytes", 4794 "bytes",
4550 "chrono", 4795 "chrono",
@@ -4557,7 +4802,7 @@ dependencies = [
4557 "parking_lot 0.12.5", 4802 "parking_lot 0.12.5",
4558 "percent-encoding", 4803 "percent-encoding",
4559 "reqwest", 4804 "reqwest",
4560 "rustc-hash", 4805 "rustc-hash 2.1.2",
4561 "secrecy", 4806 "secrecy",
4562 "serde", 4807 "serde",
4563 "serde_cow", 4808 "serde_cow",
@@ -4668,6 +4913,12 @@ dependencies = [
4668 4913
4669[[package]] 4914[[package]]
4670name = "shlex" 4915name = "shlex"
4916version = "1.3.0"
4917source = "registry+https://github.com/rust-lang/crates.io-index"
4918checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
4919
4920[[package]]
4921name = "shlex"
4671version = "2.0.1" 4922version = "2.0.1"
4672source = "registry+https://github.com/rust-lang/crates.io-index" 4923source = "registry+https://github.com/rust-lang/crates.io-index"
4673checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" 4924checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba"
@@ -4795,6 +5046,17 @@ dependencies = [
4795] 5046]
4796 5047
4797[[package]] 5048[[package]]
5049name = "socks"
5050version = "0.3.4"
5051source = "registry+https://github.com/rust-lang/crates.io-index"
5052checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b"
5053dependencies = [
5054 "byteorder",
5055 "libc",
5056 "winapi",
5057]
5058
5059[[package]]
4798name = "songbird" 5060name = "songbird"
4799version = "0.6.0" 5061version = "0.6.0"
4800source = "registry+https://github.com/rust-lang/crates.io-index" 5062source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -4863,7 +5125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
4863checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" 5125checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d"
4864dependencies = [ 5126dependencies = [
4865 "base64ct", 5127 "base64ct",
4866 "der", 5128 "der 0.7.10",
4867] 5129]
4868 5130
4869[[package]] 5131[[package]]
@@ -4885,7 +5147,7 @@ version = "0.9.0"
4885source = "registry+https://github.com/rust-lang/crates.io-index" 5147source = "registry+https://github.com/rust-lang/crates.io-index"
4886checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb" 5148checksum = "05b44e85bf579a8eeb4ceaa77a3a523baf2bf0e9bac7e40f405d537b5d2d5ccb"
4887dependencies = [ 5149dependencies = [
4888 "base64", 5150 "base64 0.22.1",
4889 "bytes", 5151 "bytes",
4890 "cfg-if", 5152 "cfg-if",
4891 "chrono", 5153 "chrono",
@@ -4987,7 +5249,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
4987checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e" 5249checksum = "87a2bdd6e83f6b3ea525ca9fee568030508b58355a43d0b2c1674d5f79dcd65e"
4988dependencies = [ 5250dependencies = [
4989 "atoi", 5251 "atoi",
4990 "base64", 5252 "base64 0.22.1",
4991 "bitflags 2.13.0", 5253 "bitflags 2.13.0",
4992 "byteorder", 5254 "byteorder",
4993 "chrono", 5255 "chrono",
@@ -5546,7 +5808,7 @@ dependencies = [
5546 "fastrand", 5808 "fastrand",
5547 "getrandom 0.4.3", 5809 "getrandom 0.4.3",
5548 "once_cell", 5810 "once_cell",
5549 "rustix", 5811 "rustix 1.1.4",
5550 "windows-sys 0.61.2", 5812 "windows-sys 0.61.2",
5551] 5813]
5552 5814
@@ -5820,7 +6082,7 @@ version = "0.11.4"
5820source = "registry+https://github.com/rust-lang/crates.io-index" 6082source = "registry+https://github.com/rust-lang/crates.io-index"
5821checksum = "9fcaf159b4e7a376b05b5bfd77bfd38f3324f5fce751b4213bfc7eaa47affb4e" 6083checksum = "9fcaf159b4e7a376b05b5bfd77bfd38f3324f5fce751b4213bfc7eaa47affb4e"
5822dependencies = [ 6084dependencies = [
5823 "base64", 6085 "base64 0.22.1",
5824 "bytes", 6086 "bytes",
5825 "fastrand", 6087 "fastrand",
5826 "futures-core", 6088 "futures-core",
@@ -6208,6 +6470,36 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
6208checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" 6470checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
6209 6471
6210[[package]] 6472[[package]]
6473name = "ureq"
6474version = "3.4.0"
6475source = "registry+https://github.com/rust-lang/crates.io-index"
6476checksum = "972d7902c8735f2695410b8aed7df6ed12a47394aa1c8d7af49f0497b731a94d"
6477dependencies = [
6478 "base64 0.23.1",
6479 "der 0.8.2",
6480 "log",
6481 "native-tls",
6482 "percent-encoding",
6483 "rustls-pki-types",
6484 "socks",
6485 "ureq-proto",
6486 "utf8-zero",
6487 "webpki-root-certs",
6488]
6489
6490[[package]]
6491name = "ureq-proto"
6492version = "0.6.1"
6493source = "registry+https://github.com/rust-lang/crates.io-index"
6494checksum = "da5f78b09e6941e1a0f2e30e695e4b120377b54d5e0aec11b594bb57b3971613"
6495dependencies = [
6496 "base64 0.23.1",
6497 "http",
6498 "httparse",
6499 "log",
6500]
6501
6502[[package]]
6211name = "uriparse" 6503name = "uriparse"
6212version = "0.6.4" 6504version = "0.6.4"
6213source = "registry+https://github.com/rust-lang/crates.io-index" 6505source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -6243,6 +6535,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
6243checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" 6535checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
6244 6536
6245[[package]] 6537[[package]]
6538name = "utf8-zero"
6539version = "0.8.1"
6540source = "registry+https://github.com/rust-lang/crates.io-index"
6541checksum = "b8c0a043c9540bae7c578c88f91dda8bd82e59ae27c21baca69c8b191aaf5a6e"
6542
6543[[package]]
6246name = "utf8_iter" 6544name = "utf8_iter"
6247version = "1.0.4" 6545version = "1.0.4"
6248source = "registry+https://github.com/rust-lang/crates.io-index" 6546source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -6435,6 +6733,15 @@ dependencies = [
6435] 6733]
6436 6734
6437[[package]] 6735[[package]]
6736name = "webpki-root-certs"
6737version = "1.0.9"
6738source = "registry+https://github.com/rust-lang/crates.io-index"
6739checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
6740dependencies = [
6741 "rustls-pki-types",
6742]
6743
6744[[package]]
6438name = "webpki-roots" 6745name = "webpki-roots"
6439version = "0.26.11" 6746version = "0.26.11"
6440source = "registry+https://github.com/rust-lang/crates.io-index" 6747source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -6459,6 +6766,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
6459checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" 6766checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
6460 6767
6461[[package]] 6768[[package]]
6769name = "which"
6770version = "4.4.2"
6771source = "registry+https://github.com/rust-lang/crates.io-index"
6772checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7"
6773dependencies = [
6774 "either",
6775 "home",
6776 "once_cell",
6777 "rustix 0.38.44",
6778]
6779
6780[[package]]
6462name = "whoami" 6781name = "whoami"
6463version = "2.1.2" 6782version = "2.1.2"
6464source = "registry+https://github.com/rust-lang/crates.io-index" 6783source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 9049eef..a1d8b5b 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -76,3 +76,5 @@ uuid = { version = "1.23.3", features = ["v4"] }
76glob = "0.3.3" 76glob = "0.3.3"
77tempfile = "3.27" 77tempfile = "3.27"
78futures-util = "0.3" 78futures-util = "0.3"
79piper-rs = "0.2.0"
80dashmap = "6.2.1"
diff --git a/build.rs b/build.rs
new file mode 100644
index 0000000..d152090
--- /dev/null
+++ b/build.rs
@@ -0,0 +1,52 @@
1fn main() {
2 // espeak-rs-sys builds espeak-ng from source but misses linking sonic
3 // and the speechPlayer audio_object symbols. piper-rs only uses
4 // espeak-ng for phonemization (never audio output), so stub implementations
5 // satisfy the linker without providing real functionality.
6
7 let out_dir = std::env::var("OUT_DIR").unwrap();
8 let stubs_c = format!("{}/tts_stubs.c", out_dir);
9 let stubs_o = format!("{}/tts_stubs.o", out_dir);
10 let stubs_a = format!("{}/libtts_stubs.a", out_dir);
11
12 std::fs::write(
13 &stubs_c,
14 r#"
15#include <stddef.h>
16typedef void* sonicStream;
17sonicStream sonicCreateStream(int sampleRate, int channels) { return NULL; }
18void sonicDestroyStream(sonicStream s) {}
19void sonicSetSpeed(sonicStream s, float speed) {}
20float sonicGetSpeed(sonicStream s) { return 1.0f; }
21void sonicWriteShortToStream(sonicStream s, const short *samples, int n) {}
22int sonicReadShortFromStream(sonicStream s, short *samples, int n) { return 0; }
23void sonicFlushStream(sonicStream s) {}
24typedef void* audio_object_t;
25int audio_object_open(audio_object_t o) { return 0; }
26int audio_object_write(audio_object_t o, const void *buf, size_t n) { return (int)n; }
27int audio_object_close(audio_object_t o) { return 0; }
28int audio_object_flush(audio_object_t o) { return 0; }
29int audio_object_drain(audio_object_t o) { return 0; }
30const char* audio_object_strerror(audio_object_t o, int err) { return "stub"; }
31audio_object_t create_audio_device_object(void* d, const char* device) { return NULL; }
32"#,
33 )
34 .expect("write stubs");
35
36 // cc -c
37 let status = std::process::Command::new("cc")
38 .args(["-c", &stubs_c, "-o", &stubs_o])
39 .status()
40 .expect("run cc");
41 assert!(status.success(), "cc -c failed");
42
43 // ar rcs
44 let status = std::process::Command::new("ar")
45 .args(["rcs", &stubs_a, &stubs_o])
46 .status()
47 .expect("run ar");
48 assert!(status.success(), "ar rcs failed");
49
50 println!("cargo:rustc-link-search=native={}", out_dir);
51 println!("cargo:rustc-link-lib=static=tts_stubs");
52}
diff --git a/src/commands/ai.rs b/src/commands/ai.rs
index 61fb471..5dd98cb 100644
--- a/src/commands/ai.rs
+++ b/src/commands/ai.rs
@@ -30,15 +30,14 @@ pub async fn senator(
30 }) 30 })
31 .ok_or_else(lang::err_not_in_voice)?; 31 .ok_or_else(lang::err_not_in_voice)?;
32 32
33 let (audio_data, _) = 33 let audio_data = ctx.data().tts.synthesize(&response).await?;
34 crate::tts::synthesize(&ctx.data().config, &ctx.data().http_client, &response).await?;
35 34
36 let call = ctx 35 let call = ctx
37 .data() 36 .data()
38 .voice_manager 37 .voice_manager
39 .connect(guild_id, channel_id) 38 .connect(guild_id, channel_id)
40 .await?; 39 .await?;
41 let input = crate::voice::decode_audio(&audio_data)?; 40 let input = crate::tts::pcm_bytes_to_input(audio_data);
42 let mut call_lock = call.lock().await; 41 let mut call_lock = call.lock().await;
43 if call_lock.current_channel().is_some() { 42 if call_lock.current_channel().is_some() {
44 call_lock.play_input(input); 43 call_lock.play_input(input);
diff --git a/src/commands/tts.rs b/src/commands/tts.rs
index 8d20179..f45d2e8 100644
--- a/src/commands/tts.rs
+++ b/src/commands/tts.rs
@@ -15,7 +15,7 @@ pub enum OutputMode {
15 Vc, 15 Vc,
16} 16}
17 17
18/// Generate TTS (Piper locally, Google as fallback) and output to file or voice channel 18/// Generate TTS (local Piper) and output to file or voice channel
19#[poise::command(slash_command)] 19#[poise::command(slash_command)]
20pub async fn tts( 20pub async fn tts(
21 ctx: Context<'_>, 21 ctx: Context<'_>,
@@ -25,21 +25,21 @@ pub async fn tts(
25 ctx.defer().await?; 25 ctx.defer().await?;
26 26
27 let output = output.unwrap_or(OutputMode::Vc); 27 let output = output.unwrap_or(OutputMode::Vc);
28 let (audio_data, ext) = 28 let audio_data = ctx.data().tts.synthesize(&text).await?;
29 crate::tts::synthesize(&ctx.data().config, &ctx.data().http_client, &text).await?;
30 29
31 match output { 30 match output {
32 OutputMode::File => { 31 OutputMode::File => {
32 let wav = crate::tts::encode_wav(&audio_data, 48000, 2);
33
33 tokio::fs::create_dir_all(crate::config::TEMP_PATH) 34 tokio::fs::create_dir_all(crate::config::TEMP_PATH)
34 .await 35 .await
35 .ok(); 36 .ok();
36 let path = format!( 37 let path = format!(
37 "{}/tts_{}.{}", 38 "{}/tts_{}.wav",
38 crate::config::TEMP_PATH, 39 crate::config::TEMP_PATH,
39 uuid::Uuid::new_v4(), 40 uuid::Uuid::new_v4(),
40 ext
41 ); 41 );
42 tokio::fs::write(&path, &audio_data).await?; 42 tokio::fs::write(&path, &wav).await?;
43 if let Err(e) = ctx 43 if let Err(e) = ctx
44 .send( 44 .send(
45 poise::CreateReply::default() 45 poise::CreateReply::default()
@@ -69,7 +69,7 @@ pub async fn tts(
69 .voice_manager 69 .voice_manager
70 .connect(guild_id, channel_id) 70 .connect(guild_id, channel_id)
71 .await?; 71 .await?;
72 let input = crate::voice::decode_audio(&audio_data)?; 72 let input = crate::tts::pcm_bytes_to_input(audio_data);
73 let mut call_lock = call.lock().await; 73 let mut call_lock = call.lock().await;
74 if call_lock.current_channel().is_some() { 74 if call_lock.current_channel().is_some() {
75 call_lock.play_input(input); 75 call_lock.play_input(input);
diff --git a/src/events/mod.rs b/src/events/mod.rs
index e4f1b22..87333c8 100644
--- a/src/events/mod.rs
+++ b/src/events/mod.rs
@@ -28,15 +28,14 @@ fn format_duration(seconds: f64) -> String {
28} 28}
29 29
30async fn play_tts_in_channel( 30async fn play_tts_in_channel(
31 config: crate::config::Config, 31 tts: std::sync::Arc<crate::tts::TtsState>,
32 http_client: reqwest::Client,
33 voice_manager: crate::voice::VoiceManager, 32 voice_manager: crate::voice::VoiceManager,
34 guild_id: serenity::GuildId, 33 guild_id: serenity::GuildId,
35 channel_id: serenity::ChannelId, 34 channel_id: serenity::ChannelId,
36 text: String, 35 text: String,
37) { 36) {
38 let audio_data = match crate::tts::synthesize(&config, &http_client, &text).await { 37 let audio_data = match tts.synthesize(&text).await {
39 Ok((bytes, _)) => bytes, 38 Ok(bytes) => bytes,
40 Err(e) => { 39 Err(e) => {
41 warn!("Failed to synthesize TTS: {e}"); 40 warn!("Failed to synthesize TTS: {e}");
42 return; 41 return;
@@ -51,13 +50,7 @@ async fn play_tts_in_channel(
51 } 50 }
52 }; 51 };
53 52
54 let input = match crate::voice::decode_audio(&audio_data) { 53 let input = crate::tts::pcm_bytes_to_input(audio_data);
55 Ok(i) => i,
56 Err(e) => {
57 warn!("Failed to decode TTS audio: {e}");
58 return;
59 }
60 };
61 54
62 let mut call_lock = call.lock().await; 55 let mut call_lock = call.lock().await;
63 if call_lock.current_channel().is_some() { 56 if call_lock.current_channel().is_some() {
@@ -97,6 +90,16 @@ pub async fn handle_voice_state_update(
97 .map(|m| m.display_name().to_string()) 90 .map(|m| m.display_name().to_string())
98 .unwrap_or_default(); 91 .unwrap_or_default();
99 92
93 // Pre-generate all 6 event phrases for this user (background, non-blocking).
94 // First event per user pays ~120ms; subsequent events are instant from cache.
95 {
96 let tts = data.tts.clone();
97 let name = display_name.clone();
98 tokio::spawn(async move {
99 tts.pre_generate(&name).await;
100 });
101 }
102
100 let now = current_ts as f64; 103 let now = current_ts as f64;
101 104
102 let user_id_str = user_id.to_string(); 105 let user_id_str = user_id.to_string();
@@ -293,8 +296,7 @@ pub async fn handle_voice_state_update(
293 && let Some(cid) = tts_channel.or(new_channel) 296 && let Some(cid) = tts_channel.or(new_channel)
294 { 297 {
295 tokio::spawn(play_tts_in_channel( 298 tokio::spawn(play_tts_in_channel(
296 data.config.clone(), 299 data.tts.clone(),
297 data.http_client.clone(),
298 data.voice_manager.clone(), 300 data.voice_manager.clone(),
299 gid, 301 gid,
300 cid, 302 cid,
@@ -314,8 +316,7 @@ pub async fn handle_voice_state_update(
314 && let Some(cid) = tts_channel.or(new_channel) 316 && let Some(cid) = tts_channel.or(new_channel)
315 { 317 {
316 tokio::spawn(play_tts_in_channel( 318 tokio::spawn(play_tts_in_channel(
317 data.config.clone(), 319 data.tts.clone(),
318 data.http_client.clone(),
319 data.voice_manager.clone(), 320 data.voice_manager.clone(),
320 gid, 321 gid,
321 cid, 322 cid,
@@ -330,8 +331,7 @@ pub async fn handle_voice_state_update(
330 && let Some(gid) = guild_id 331 && let Some(gid) = guild_id
331 { 332 {
332 tokio::spawn(play_tts_in_channel( 333 tokio::spawn(play_tts_in_channel(
333 data.config.clone(), 334 data.tts.clone(),
334 data.http_client.clone(),
335 data.voice_manager.clone(), 335 data.voice_manager.clone(),
336 gid, 336 gid,
337 cid, 337 cid,
diff --git a/src/lang.rs b/src/lang.rs
index de0c7d7..53888eb 100644
--- a/src/lang.rs
+++ b/src/lang.rs
@@ -114,13 +114,6 @@ pub fn fmt_voice_join_failed(e: &dyn std::fmt::Display) -> String {
114pub const L_TTS_SAVED: &str = "TTS zapisany"; 114pub const L_TTS_SAVED: &str = "TTS zapisany";
115pub const L_TTS_PLAYING: &str = "Grom TTS na kanale głosowym"; 115pub const L_TTS_PLAYING: &str = "Grom TTS na kanale głosowym";
116 116
117pub fn fmt_tts_piper_launch(e: &str) -> String {
118 format!("Piper sie nie uruchomił w czas: `{e}`")
119}
120pub fn fmt_tts_piper_failed(e: &str) -> String {
121 format!("Piper TTS sie nie powiodło: `{e}`")
122}
123
124// ── Trigger command ─────────────────────────────────────────────────── 117// ── Trigger command ───────────────────────────────────────────────────
125 118
126pub const L_TRIGGER_UPDATED: &str = "Trigger zaktualizowany"; 119pub const L_TRIGGER_UPDATED: &str = "Trigger zaktualizowany";
diff --git a/src/main.rs b/src/main.rs
index 7f57421..5ab33ab 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -50,6 +50,8 @@ pub struct Data {
50 pub ac_search_cache: AcSearchCache, 50 pub ac_search_cache: AcSearchCache,
51 /// Active music track per guild (only one music source at a time). 51 /// Active music track per guild (only one music source at a time).
52 pub music_tracks: Arc<tokio::sync::Mutex<HashMap<u64, songbird::tracks::TrackHandle>>>, 52 pub music_tracks: Arc<tokio::sync::Mutex<HashMap<u64, songbird::tracks::TrackHandle>>>,
53 /// TTS engine: piper model + audio cache.
54 pub tts: Arc<tts::TtsState>,
53} 55}
54 56
55impl Data { 57impl Data {
@@ -187,7 +189,10 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
187 189
188 let music_tracks: Arc<tokio::sync::Mutex<HashMap<u64, songbird::tracks::TrackHandle>>> = Arc::new(tokio::sync::Mutex::new(HashMap::new())); 190 let music_tracks: Arc<tokio::sync::Mutex<HashMap<u64, songbird::tracks::TrackHandle>>> = Arc::new(tokio::sync::Mutex::new(HashMap::new()));
189 191
192 let tts = Arc::new(tts::TtsState::new(config.piper_model_path()));
193
190 let data = Data { 194 let data = Data {
195 tts,
191 config, 196 config,
192 db_pool, 197 db_pool,
193 voice_manager: vm, 198 voice_manager: vm,
diff --git a/src/tts.rs b/src/tts.rs
index ded76ad..3ae5140 100644
--- a/src/tts.rs
+++ b/src/tts.rs
@@ -1,147 +1,291 @@
1use std::io::Cursor;
1use std::path::PathBuf; 2use std::path::PathBuf;
2use std::process::Stdio; 3use std::sync::OnceLock;
3use std::time::Instant; 4use std::time::{Duration, Instant};
4 5
5use tokio::io::AsyncWriteExt; 6use dashmap::DashMap;
7use piper_rs::Piper;
8use tokio::sync::Mutex;
6use tracing::{debug, info, warn}; 9use tracing::{debug, info, warn};
7 10
8use crate::config;
9use crate::config::TEMP_PATH;
10use crate::error::{Error, Result}; 11use crate::error::{Error, Result};
11 12
12/// Synthesize speech for `text`. 13// ── Constants ──────────────────────────────────────────────────────────
13/// 14
14/// Primary engine is local Piper TTS (via the `piper` CLI). If Piper is not 15/// Target sample rate for songbird playback.
15/// installed, fails, or `TTS_MODEL_PATH` is unset, we fall back to Google 16const TARGET_SAMPLE_RATE: u32 = 48_000;
16/// Translate TTS over HTTP. 17
18/// Cache entries older than this are evicted on access.
19const CACHE_TTL: Duration = Duration::from_secs(3600);
20
21/// All voice-state TTS event types.
22const EVENT_TYPES: &[&str] = &[
23 "joined",
24 "left",
25 "deafened",
26 "undeafened",
27 "muted",
28 "unmuted",
29];
30
31// ── TtsState ───────────────────────────────────────────────────────────
32
33/// Shared TTS state: piper model instance + audio cache.
17/// 34///
18/// Returns the encoded audio bytes plus the file extension to use when saving 35/// Held in poise `Data`. Piper is loaded lazily on first synthesis call.
19/// (`"wav"` for Piper, `"mp3"` for Google). 36/// Cache maps full text string → resampled 48 kHz stereo PCM bytes.
20pub async fn synthesize( 37pub struct TtsState {
21 config: &config::Config, 38 piper: OnceLock<Mutex<Piper>>,
22 http_client: &reqwest::Client, 39 cache: DashMap<String, (Vec<u8>, Instant)>,
23 text: &str, 40 model_path: Option<PathBuf>,
24) -> Result<(Vec<u8>, &'static str)> { 41}
25 match config.piper_model_path() { 42
26 Some(model_path) => { 43impl TtsState {
27 let started = Instant::now(); 44 pub fn new(model_path: Option<PathBuf>) -> Self {
28 match piper_synthesize(&model_path, text).await { 45 Self {
29 Ok(bytes) => { 46 piper: OnceLock::new(),
30 info!( 47 cache: DashMap::new(),
31 model = %model_path.display(), 48 model_path,
32 text_len = text.len(),
33 bytes = bytes.len(),
34 elapsed_ms = started.elapsed().as_millis() as u64,
35 "Piper TTS synthesized"
36 );
37 return Ok((bytes, "wav"));
38 }
39 Err(e) => {
40 warn!(
41 model = %model_path.display(),
42 text_len = text.len(),
43 elapsed_ms = started.elapsed().as_millis() as u64,
44 "Piper TTS failed, falling back to Google TTS: {e}"
45 );
46 }
47 }
48 } 49 }
49 None => { 50 }
50 debug!("TTS_MODEL_PATH is not set; using Google TTS fallback"); 51
52 /// Get or initialize the Piper model. Returns error if model path is
53 /// unset or model fails to load.
54 async fn get_piper(&self) -> Result<&Mutex<Piper>> {
55 if let Some(piper) = self.piper.get() {
56 return Ok(piper);
51 } 57 }
58
59 let path = self
60 .model_path
61 .as_ref()
62 .ok_or_else(|| Error::Custom("TTS_MODEL_PATH is not set".into()))?;
63
64 // Resolve .onnx.json config path from model path
65 let onnx_path = path.clone();
66 let config_path = {
67 let s = path.to_string_lossy();
68 if s.ends_with(".onnx") {
69 PathBuf::from(format!("{}.json", s))
70 } else {
71 path.clone()
72 }
73 };
74
75 debug!(
76 onnx = %onnx_path.display(),
77 config = %config_path.display(),
78 "loading piper model"
79 );
80 let piper = Piper::new(&onnx_path, &config_path).map_err(|e| {
81 Error::Custom(format!("Failed to load piper model: {e}"))
82 })?;
83 info!("piper model loaded successfully");
84
85 // set() may fail if another task initialized first — that's fine
86 let _ = self.piper.set(Mutex::new(piper));
87 Ok(self.piper.get().unwrap())
52 } 88 }
53 89
54 let started = Instant::now(); 90 /// Look up cached audio for a text string. Returns `None` if uncached
55 let bytes = google_synthesize(http_client, text).await?; 91 /// or expired.
56 info!( 92 fn cache_lookup(&self, text: &str) -> Option<Vec<u8>> {
57 text_len = text.len(), 93 let entry = self.cache.get(text)?;
58 bytes = bytes.len(), 94 if entry.1.elapsed() > CACHE_TTL {
59 elapsed_ms = started.elapsed().as_millis() as u64, 95 drop(entry);
60 "Google TTS synthesized" 96 self.cache.remove(text);
61 ); 97 None
62 Ok((bytes, "mp3")) 98 } else {
63} 99 Some(entry.0.clone())
100 }
101 }
102
103 /// Store audio bytes in the cache.
104 fn cache_store(&self, text: String, audio: Vec<u8>) {
105 self.cache.insert(text, (audio, Instant::now()));
106 }
64 107
65/// Run `piper` as a subprocess, feeding `text` on stdin and collecting the 108 /// Evict all expired cache entries.
66/// generated WAV from `--output_file`. 109 fn cache_evict(&self) {
67async fn piper_synthesize(model_path: &PathBuf, text: &str) -> Result<Vec<u8>> { 110 self.cache
68 tokio::fs::create_dir_all(TEMP_PATH).await.ok(); 111 .retain(|_, (_, inserted_at)| inserted_at.elapsed() <= CACHE_TTL);
69
70 let out_path = format!("{}/tts_piper_{}.wav", TEMP_PATH, uuid::Uuid::new_v4());
71 let model = model_path.to_string_lossy().to_string();
72
73 debug!(model = %model, "spawning piper subprocess");
74 let started = Instant::now();
75
76 let mut child = tokio::process::Command::new("piper")
77 .arg("--model")
78 .arg(&model)
79 .arg("--output_file")
80 .arg(&out_path)
81 .stdin(Stdio::piped())
82 .stdout(Stdio::piped())
83 .stderr(Stdio::piped())
84 .spawn()
85 .map_err(|e| Error::Custom(crate::lang::fmt_tts_piper_launch(&e.to_string())))?;
86
87 {
88 let mut stdin = child
89 .stdin
90 .take()
91 .ok_or_else(|| Error::Custom("Piper stdin unavailable".into()))?;
92 stdin.write_all(text.as_bytes()).await?;
93 stdin.write_all(b"\n").await?;
94 } 112 }
95 113
96 let output = child.wait_with_output().await?; 114 /// Synthesize text → resampled 48 kHz stereo PCM bytes.
97 let elapsed_ms = started.elapsed().as_millis() as u64; 115 ///
116 /// Checks cache first. On miss, runs piper inference, resamples, caches,
117 /// and returns.
118 pub async fn synthesize(&self, text: &str) -> Result<Vec<u8>> {
119 // Cache hit
120 if let Some(cached) = self.cache_lookup(text) {
121 debug!(text_len = text.len(), "tts cache hit");
122 return Ok(cached);
123 }
124
125 // Cache miss — synthesize
126 let piper = self.get_piper().await?;
127 let started = Instant::now();
98 128
99 if !output.status.success() { 129 let (samples, sample_rate) = {
100 let stderr = String::from_utf8_lossy(&output.stderr); 130 let mut lock = piper.lock().await;
101 warn!( 131 lock.create(text, false, None, None, None, None)
102 model = %model, 132 .map_err(|e| Error::Custom(format!("piper synthesis failed: {e}")))?
133 };
134
135 let elapsed_ms = started.elapsed().as_millis() as u64;
136 debug!(
137 text_len = text.len(),
138 samples = samples.len(),
139 sample_rate,
103 elapsed_ms, 140 elapsed_ms,
104 "piper exited with error: {}", 141 "piper synthesized"
105 stderr.trim_end()
106 ); 142 );
107 let _ = tokio::fs::remove_file(&out_path).await; 143
108 return Err(Error::Custom(crate::lang::fmt_tts_piper_failed( 144 // Resample mono → stereo at target sample rate
109 stderr.trim_end(), 145 let pcm_bytes = resample_to_stereo_bytes(&samples, sample_rate, TARGET_SAMPLE_RATE);
110 ))); 146
147 self.cache_store(text.to_string(), pcm_bytes.clone());
148 Ok(pcm_bytes)
111 } 149 }
112 150
113 let bytes = tokio::fs::read(&out_path).await?; 151 /// Pre-generate all 6 event phrases for a display name.
114 debug!( 152 ///
115 model = %model, 153 /// Called on first voice event for a user. Synthesizes all phrases and
116 bytes = bytes.len(), 154 /// caches them so subsequent events are instant.
117 elapsed_ms, 155 pub async fn pre_generate(&self, display_name: &str) {
118 "piper succeeded" 156 let phrases: Vec<String> = EVENT_TYPES
119 ); 157 .iter()
120 let _ = tokio::fs::remove_file(&out_path).await; 158 .map(|evt| make_event_phrase(display_name, evt))
121 Ok(bytes) 159 .collect();
160
161 for phrase in &phrases {
162 if self.cache_lookup(phrase).is_some() {
163 continue;
164 }
165 if let Err(e) = self.synthesize(phrase).await {
166 warn!(
167 display_name,
168 phrase, "pre-generation failed: {e}"
169 );
170 }
171 }
172
173 // Evict expired entries while we're at it
174 self.cache_evict();
175 }
122} 176}
123 177
124/// Fetch Google Translate TTS (Polish) over HTTP. 178// ── Phrase construction ────────────────────────────────────────────────
125async fn google_synthesize(client: &reqwest::Client, text: &str) -> Result<Vec<u8>> { 179
126 let tts_url = format!( 180/// Build the TTS phrase for a voice event + display name.
127 "https://translate.google.com/translate_tts?ie=UTF-8&q={}&tl=pl&client=tw-ob", 181pub fn make_event_phrase(display_name: &str, event: &str) -> String {
128 urlencoding(text) 182 let suffix = match event {
129 ); 183 "joined" => "sie zespawnił",
130 debug!(url_len = tts_url.len(), "fetching Google TTS"); 184 "left" => "jest chujem",
131 let bytes = client.get(&tts_url).send().await?.bytes().await?; 185 "deafened" => "jest bity przez ojca",
132 Ok(bytes.to_vec()) 186 "undeafened" => "uciekł od ojca",
187 "muted" => "idzie robić jajecznice",
188 "unmuted" => "zrobił jajecznice",
189 _ => return format!("{display_name} {event}"),
190 };
191 format!("{display_name} {suffix}")
133} 192}
134 193
135pub fn urlencoding(s: &str) -> String { 194// ── Resampling ─────────────────────────────────────────────────────────
136 let mut out = String::with_capacity(s.len()); 195
137 for &byte in s.as_bytes() { 196/// Resample mono f32 PCM from `src_rate` to `dst_rate`, duplicate to stereo,
138 match byte { 197/// and return as little-endian bytes.
139 b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'-' | b'.' | b'~' => { 198fn resample_to_stereo_bytes(samples: &[f32], src_rate: u32, dst_rate: u32) -> Vec<u8> {
140 out.push(byte as char) 199 if src_rate == dst_rate {
141 } 200 // Just duplicate mono → stereo, no resampling needed
142 b' ' => out.push('+'), 201 let mut out = Vec::with_capacity(samples.len() * 2 * 4);
143 _ => out.push_str(&format!("%{:02X}", byte)), 202 for &s in samples {
203 out.extend_from_slice(&s.to_le_bytes());
204 out.extend_from_slice(&s.to_le_bytes());
144 } 205 }
206 return out;
145 } 207 }
208
209 let ratio = src_rate as f64 / dst_rate as f64;
210 let out_len = (samples.len() as f64 / ratio).ceil() as usize;
211 let mut out = Vec::with_capacity(out_len * 2 * 4);
212
213 for i in 0..out_len {
214 let src_idx = i as f64 * ratio;
215 let idx = src_idx as usize;
216 let frac = src_idx - idx as f64;
217
218 let sample = if idx + 1 < samples.len() {
219 // Linear interpolation
220 samples[idx] + (samples[idx + 1] - samples[idx]) * frac as f32
221 } else if idx < samples.len() {
222 samples[idx]
223 } else {
224 0.0
225 };
226
227 // Mono → stereo duplication
228 out.extend_from_slice(&sample.to_le_bytes());
229 out.extend_from_slice(&sample.to_le_bytes());
230 }
231
146 out 232 out
147} 233}
234
235// ── WAV encoding ───────────────────────────────────────────────────────
236
237/// Encode raw f32 stereo PCM bytes into a WAV file (16-bit PCM).
238///
239/// `pcm_f32_bytes` is the raw little-endian f32 interleaved stereo data
240/// (e.g. from `resample_to_stereo_bytes`).
241pub fn encode_wav(pcm_f32_bytes: &[u8], sample_rate: u32, channels: u16) -> Vec<u8> {
242 let bits_per_sample: u16 = 16;
243 let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8;
244 let block_align = channels * bits_per_sample / 8;
245
246 // Convert f32 samples to i16
247 let sample_count = pcm_f32_bytes.len() / 4; // 4 bytes per f32
248 let mut pcm_i16 = Vec::with_capacity(sample_count * 2);
249 for chunk in pcm_f32_bytes.chunks_exact(4) {
250 let f = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
251 let i = (f * 32767.0).clamp(-32768.0, 32767.0) as i16;
252 pcm_i16.extend_from_slice(&i.to_le_bytes());
253 }
254
255 let data_size = pcm_i16.len() as u32;
256 let file_size = 36 + data_size;
257
258 let mut wav = Vec::with_capacity(44 + pcm_i16.len());
259
260 // RIFF header
261 wav.extend_from_slice(b"RIFF");
262 wav.extend_from_slice(&file_size.to_le_bytes());
263 wav.extend_from_slice(b"WAVE");
264
265 // fmt chunk
266 wav.extend_from_slice(b"fmt ");
267 wav.extend_from_slice(&16u32.to_le_bytes()); // chunk size
268 wav.extend_from_slice(&1u16.to_le_bytes()); // PCM format
269 wav.extend_from_slice(&channels.to_le_bytes());
270 wav.extend_from_slice(&sample_rate.to_le_bytes());
271 wav.extend_from_slice(&byte_rate.to_le_bytes());
272 wav.extend_from_slice(&block_align.to_le_bytes());
273 wav.extend_from_slice(&bits_per_sample.to_le_bytes());
274
275 // data chunk
276 wav.extend_from_slice(b"data");
277 wav.extend_from_slice(&data_size.to_le_bytes());
278 wav.extend_from_slice(&pcm_i16);
279
280 wav
281}
282
283// ── Songbird Input helper ──────────────────────────────────────────────
284
285/// Create a songbird `Input` from resampled f32 stereo PCM bytes.
286pub fn pcm_bytes_to_input(pcm_f32_bytes: Vec<u8>) -> songbird::input::Input {
287 use songbird::input::RawAdapter;
288 let cursor = Cursor::new(pcm_f32_bytes);
289 // 2 channels (stereo), 48 kHz — matches TARGET_SAMPLE_RATE
290 RawAdapter::new(cursor, TARGET_SAMPLE_RATE, 2).into()
291}
diff --git a/tests/tts_smoke.rs b/tests/tts_smoke.rs
new file mode 100644
index 0000000..e98d56d
--- /dev/null
+++ b/tests/tts_smoke.rs
@@ -0,0 +1,54 @@
1//! Smoke test: verify piper-rs loads the Polish model and synthesizes audio.
2
3use std::path::Path;
4
5#[test]
6fn piper_load_and_synthesize() {
7 let model_dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("piper");
8 let onnx = model_dir.join("pl_PL-bass-high.onnx");
9 let json = model_dir.join("pl_PL-bass-high.onnx.json");
10
11 assert!(onnx.exists(), "model file missing: {}", onnx.display());
12 assert!(json.exists(), "config file missing: {}", json.display());
13
14 let mut piper =
15 piper_rs::Piper::new(&onnx, &json).expect("failed to load piper model");
16
17 let text = "Kamil sie zespawnił";
18 let (samples, sample_rate) = piper
19 .create(text, false, None, None, None, None)
20 .expect("synthesis failed");
21
22 assert!(!samples.is_empty(), "no samples produced");
23 assert!(sample_rate > 0, "invalid sample rate: {sample_rate}");
24 assert!(
25 samples.len() as u32 > sample_rate / 10,
26 "suspiciously short output: {} samples at {} Hz",
27 samples.len(),
28 sample_rate
29 );
30
31 let duration = samples.len() as f64 / sample_rate as f64;
32 println!(
33 "OK: '{}' → {} samples @ {} Hz ({:.2}s)",
34 text,
35 samples.len(),
36 sample_rate,
37 duration,
38 );
39
40 // Second call measures cached-model synthesis speed
41 let start = std::time::Instant::now();
42 let (samples2, sr2) = piper
43 .create("Kamil jest chujem", false, None, None, None, None)
44 .expect("second synthesis failed");
45 let elapsed = start.elapsed();
46 println!(
47 "Cached: '{}' → {} samples @ {} Hz in {:?}",
48 "Kamil jest chujem",
49 samples2.len(),
50 sr2,
51 elapsed,
52 );
53 assert!(!samples2.is_empty());
54}
generated by cgit v1.2.3 (git 2.46.0) at 2026-11-08 08:53:47 +0000