1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
|
use std::io::Cursor;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, Instant};
use dashmap::DashMap;
use piper_rs::Piper;
use tracing::{debug, info, warn};
use crate::error::{Error, Result};
// ── Constants ──────────────────────────────────────────────────────────
/// Target sample rate for songbird playback.
const TARGET_SAMPLE_RATE: u32 = 48_000;
/// Cache entries older than this are evicted on access.
const CACHE_TTL: Duration = Duration::from_secs(3600);
/// Hard cap on cached audio so arbitrary `/tts` text cannot exhaust memory.
const CACHE_MAX_ENTRIES: usize = 256;
const CACHE_MAX_BYTES: usize = 64 * 1024 * 1024;
/// Longest text accepted for synthesis. Keeps a single request (and its
/// cached PCM) bounded.
const MAX_TTS_CHARS: usize = 2_000;
/// All voice-state TTS event types.
const EVENT_TYPES: &[&str] = &[
"joined",
"left",
"deafened",
"undeafened",
"muted",
"unmuted",
];
// ── TtsState ───────────────────────────────────────────────────────────
/// Shared TTS state: piper model instance + audio cache.
///
/// Held in poise `Data`. Piper is loaded lazily on first synthesis call.
/// Cache maps full text string → resampled 48 kHz stereo PCM bytes.
pub struct TtsState {
piper: OnceLock<Arc<Mutex<Piper>>>,
cache: DashMap<String, (Vec<u8>, Instant)>,
cache_bytes: AtomicUsize,
model_path: Option<PathBuf>,
}
impl TtsState {
pub fn new(model_path: Option<PathBuf>) -> Self {
Self {
piper: OnceLock::new(),
cache: DashMap::new(),
cache_bytes: AtomicUsize::new(0),
model_path,
}
}
/// Get or initialize the Piper model. Returns error if model path is
/// unset or model fails to load.
async fn get_piper(&self) -> Result<Arc<Mutex<Piper>>> {
if let Some(piper) = self.piper.get() {
return Ok(piper.clone());
}
let path = self
.model_path
.as_ref()
.ok_or_else(|| Error::Custom("TTS_MODEL_PATH is not set".into()))?;
// Resolve .onnx.json config path from model path
let onnx_path = path.clone();
let config_path = {
let s = path.to_string_lossy();
if s.ends_with(".onnx") {
PathBuf::from(format!("{}.json", s))
} else {
path.clone()
}
};
debug!(
onnx = %onnx_path.display(),
config = %config_path.display(),
"loading piper model"
);
// Model loading is CPU/IO heavy; keep it off the async runtime too.
let piper = tokio::task::spawn_blocking(move || {
Piper::new(&onnx_path, &config_path)
.map_err(|e| Error::Custom(format!("Failed to load piper model: {e}")))
})
.await??;
info!("piper model loaded successfully");
// set() may fail if another task initialized first — that's fine
let _ = self.piper.set(Arc::new(Mutex::new(piper)));
Ok(self
.piper
.get()
.expect("piper was just initialized")
.clone())
}
/// Look up cached audio for a text string. Returns `None` if uncached
/// or expired.
fn cache_lookup(&self, text: &str) -> Option<Vec<u8>> {
let entry = self.cache.get(text)?;
if entry.1.elapsed() > CACHE_TTL {
drop(entry);
if let Some((old, _)) = self.cache.remove(text) {
self.cache_bytes.fetch_sub(old.len(), Ordering::Relaxed);
}
None
} else {
Some(entry.0.clone())
}
}
/// Store audio bytes in the cache, evicting expired/old entries first.
fn cache_store(&self, text: String, audio: Vec<u8>) {
let audio_len = audio.len();
if audio_len > CACHE_MAX_BYTES {
return;
}
self.cache_evict();
while self.cache.len() >= CACHE_MAX_ENTRIES {
if !self.cache_evict_oldest() {
break;
}
}
while self.cache_bytes.load(Ordering::Relaxed) + audio_len > CACHE_MAX_BYTES {
if !self.cache_evict_oldest() {
break;
}
}
if let Some((old, _)) = self.cache.insert(text, (audio, Instant::now())) {
self.cache_bytes.fetch_sub(old.len(), Ordering::Relaxed);
}
self.cache_bytes.fetch_add(audio_len, Ordering::Relaxed);
}
/// Evict all expired cache entries.
fn cache_evict(&self) {
let bytes = &self.cache_bytes;
self.cache.retain(|_, (audio, inserted_at)| {
let keep = inserted_at.elapsed() <= CACHE_TTL;
if !keep {
bytes.fetch_sub(audio.len(), Ordering::Relaxed);
}
keep
});
}
/// Evict the least-recently-inserted entry. Returns false when empty.
fn cache_evict_oldest(&self) -> bool {
let oldest = self
.cache
.iter()
.min_by_key(|entry| entry.value().1)
.map(|entry| entry.key().clone());
let Some(key) = oldest else {
return false;
};
if let Some((old, _)) = self.cache.remove(&key) {
self.cache_bytes.fetch_sub(old.len(), Ordering::Relaxed);
}
true
}
/// Synthesize text → resampled 48 kHz stereo PCM bytes.
///
/// Checks cache first. On miss, runs piper inference, resamples, caches,
/// and returns.
pub async fn synthesize(&self, text: &str) -> Result<Vec<u8>> {
if text.chars().count() > MAX_TTS_CHARS {
return Err(Error::Custom(format!(
"TTS text is too long (max {MAX_TTS_CHARS} characters)"
)));
}
// Cache hit
if let Some(cached) = self.cache_lookup(text) {
debug!(text_len = text.len(), "tts cache hit");
return Ok(cached);
}
// Cache miss — synthesize on a blocking thread. Piper inference is
// synchronous and can take >100 ms; it must not stall a Tokio worker.
let piper = self.get_piper().await?;
let text_owned = text.to_string();
let started = Instant::now();
let (samples, sample_rate) = tokio::task::spawn_blocking(move || {
let mut piper = piper
.lock()
.map_err(|_| Error::Custom("piper mutex poisoned".into()))?;
piper
.create(&text_owned, false, None, None, None, None)
.map_err(|e| Error::Custom(format!("piper synthesis failed: {e}")))
})
.await??;
let elapsed_ms = started.elapsed().as_millis() as u64;
debug!(
text_len = text.len(),
samples = samples.len(),
sample_rate,
elapsed_ms,
"piper synthesized"
);
// Resample mono → stereo at target sample rate
let pcm_bytes = resample_to_stereo_bytes(&samples, sample_rate, TARGET_SAMPLE_RATE);
self.cache_store(text.to_string(), pcm_bytes.clone());
Ok(pcm_bytes)
}
/// Pre-generate all 6 event phrases for a display name.
///
/// Called on first voice event for a user. Synthesizes all phrases and
/// caches them so subsequent events are instant.
pub async fn pre_generate(&self, display_name: &str) {
let phrases: Vec<String> = EVENT_TYPES
.iter()
.map(|evt| make_event_phrase(display_name, evt))
.collect();
for phrase in &phrases {
if self.cache_lookup(phrase).is_some() {
continue;
}
if let Err(e) = self.synthesize(phrase).await {
warn!(
display_name,
phrase, "pre-generation failed: {e}"
);
}
}
// Evict expired entries while we're at it
self.cache_evict();
}
}
// ── Phrase construction ────────────────────────────────────────────────
/// Build the TTS phrase for a voice event + display name.
pub fn make_event_phrase(display_name: &str, event: &str) -> String {
let suffix = match event {
"joined" => "sie zespawnił",
"left" => "jest chujem",
"deafened" => "jest bity przez ojca",
"undeafened" => "uciekł od ojca",
"muted" => "idzie robić jajecznice",
"unmuted" => "zrobił jajecznice",
_ => return format!("{display_name} {event}"),
};
format!("{display_name} {suffix}")
}
// ── Resampling ─────────────────────────────────────────────────────────
/// Resample mono f32 PCM from `src_rate` to `dst_rate`, duplicate to stereo,
/// and return as little-endian bytes.
fn resample_to_stereo_bytes(samples: &[f32], src_rate: u32, dst_rate: u32) -> Vec<u8> {
if src_rate == dst_rate {
// Just duplicate mono → stereo, no resampling needed
let mut out = Vec::with_capacity(samples.len() * 2 * 4);
for &s in samples {
out.extend_from_slice(&s.to_le_bytes());
out.extend_from_slice(&s.to_le_bytes());
}
return out;
}
let ratio = src_rate as f64 / dst_rate as f64;
let out_len = (samples.len() as f64 / ratio).ceil() as usize;
let mut out = Vec::with_capacity(out_len * 2 * 4);
for i in 0..out_len {
let src_idx = i as f64 * ratio;
let idx = src_idx as usize;
let frac = src_idx - idx as f64;
let sample = if idx + 1 < samples.len() {
// Linear interpolation
samples[idx] + (samples[idx + 1] - samples[idx]) * frac as f32
} else if idx < samples.len() {
samples[idx]
} else {
0.0
};
// Mono → stereo duplication
out.extend_from_slice(&sample.to_le_bytes());
out.extend_from_slice(&sample.to_le_bytes());
}
out
}
// ── WAV encoding ───────────────────────────────────────────────────────
/// Encode raw f32 stereo PCM bytes into a WAV file (16-bit PCM).
///
/// `pcm_f32_bytes` is the raw little-endian f32 interleaved stereo data
/// (e.g. from `resample_to_stereo_bytes`).
pub fn encode_wav(pcm_f32_bytes: &[u8], sample_rate: u32, channels: u16) -> Vec<u8> {
let bits_per_sample: u16 = 16;
let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8;
let block_align = channels * bits_per_sample / 8;
// Convert f32 samples to i16
let sample_count = pcm_f32_bytes.len() / 4; // 4 bytes per f32
let mut pcm_i16 = Vec::with_capacity(sample_count * 2);
for chunk in pcm_f32_bytes.chunks_exact(4) {
let f = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
let i = (f * 32767.0).clamp(-32768.0, 32767.0) as i16;
pcm_i16.extend_from_slice(&i.to_le_bytes());
}
let data_size = pcm_i16.len() as u32;
let file_size = 36 + data_size;
let mut wav = Vec::with_capacity(44 + pcm_i16.len());
// RIFF header
wav.extend_from_slice(b"RIFF");
wav.extend_from_slice(&file_size.to_le_bytes());
wav.extend_from_slice(b"WAVE");
// fmt chunk
wav.extend_from_slice(b"fmt ");
wav.extend_from_slice(&16u32.to_le_bytes()); // chunk size
wav.extend_from_slice(&1u16.to_le_bytes()); // PCM format
wav.extend_from_slice(&channels.to_le_bytes());
wav.extend_from_slice(&sample_rate.to_le_bytes());
wav.extend_from_slice(&byte_rate.to_le_bytes());
wav.extend_from_slice(&block_align.to_le_bytes());
wav.extend_from_slice(&bits_per_sample.to_le_bytes());
// data chunk
wav.extend_from_slice(b"data");
wav.extend_from_slice(&data_size.to_le_bytes());
wav.extend_from_slice(&pcm_i16);
wav
}
// ── Songbird Input helper ──────────────────────────────────────────────
/// Create a songbird `Input` from resampled f32 stereo PCM bytes.
pub fn pcm_bytes_to_input(pcm_f32_bytes: Vec<u8>) -> songbird::input::Input {
use songbird::input::RawAdapter;
let cursor = Cursor::new(pcm_f32_bytes);
// 2 channels (stereo), 48 kHz — matches TARGET_SAMPLE_RATE
RawAdapter::new(cursor, TARGET_SAMPLE_RATE, 2).into()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cache_is_bounded_by_entries_and_bytes() {
let state = TtsState::new(None);
for i in 0..(CACHE_MAX_ENTRIES + 32) {
state.cache_store(format!("phrase-{i}"), vec![0u8; 1024]);
}
assert!(state.cache.len() <= CACHE_MAX_ENTRIES);
assert!(state.cache_bytes.load(Ordering::Relaxed) <= CACHE_MAX_BYTES);
}
#[tokio::test]
async fn overlong_text_is_rejected() {
let state = TtsState::new(None);
let text = "x".repeat(MAX_TTS_CHARS + 1);
assert!(state.synthesize(&text).await.is_err());
}
}
|