aboutsummaryrefslogtreecommitdiffstats
path: root/src/tts.rs
diff options
context:
space:
mode:
Diffstat (limited to 'src/tts.rs')
-rw-r--r--src/tts.rs384
1 files changed, 264 insertions, 120 deletions
diff --git a/src/tts.rs b/src/tts.rs
index ded76ad..3ae5140 100644
--- a/src/tts.rs
+++ b/src/tts.rs
@@ -1,147 +1,291 @@
1use std::io::Cursor;
1use std::path::PathBuf; 2use std::path::PathBuf;
2use std::process::Stdio; 3use std::sync::OnceLock;
3use std::time::Instant; 4use std::time::{Duration, Instant};
4 5
5use tokio::io::AsyncWriteExt; 6use dashmap::DashMap;
7use piper_rs::Piper;
8use tokio::sync::Mutex;
6use tracing::{debug, info, warn}; 9use tracing::{debug, info, warn};
7 10
8use crate::config;
9use crate::config::TEMP_PATH;
10use crate::error::{Error, Result}; 11use crate::error::{Error, Result};
11 12
12/// Synthesize speech for `text`. 13// ── Constants ──────────────────────────────────────────────────────────
13/// 14
14/// Primary engine is local Piper TTS (via the `piper` CLI). If Piper is not 15/// Target sample rate for songbird playback.
15/// installed, fails, or `TTS_MODEL_PATH` is unset, we fall back to Google 16const TARGET_SAMPLE_RATE: u32 = 48_000;
16/// Translate TTS over HTTP. 17
18/// Cache entries older than this are evicted on access.
19const CACHE_TTL: Duration = Duration::from_secs(3600);
20
21/// All voice-state TTS event types.
22const EVENT_TYPES: &[&str] = &[
23 "joined",
24 "left",
25 "deafened",
26 "undeafened",
27 "muted",
28 "unmuted",
29];
30
31// ── TtsState ───────────────────────────────────────────────────────────
32
33/// Shared TTS state: piper model instance + audio cache.
17/// 34///
18/// Returns the encoded audio bytes plus the file extension to use when saving 35/// Held in poise `Data`. Piper is loaded lazily on first synthesis call.
19/// (`"wav"` for Piper, `"mp3"` for Google). 36/// Cache maps full text string → resampled 48 kHz stereo PCM bytes.
20pub async fn synthesize( 37pub struct TtsState {
21 config: &config::Config, 38 piper: OnceLock<Mutex<Piper>>,
22 http_client: &reqwest::Client, 39 cache: DashMap<String, (Vec<u8>, Instant)>,
23 text: &str, 40 model_path: Option<PathBuf>,
24) -> Result<(Vec<u8>, &'static str)> { 41}
25 match config.piper_model_path() { 42
26 Some(model_path) => { 43impl TtsState {
27 let started = Instant::now(); 44 pub fn new(model_path: Option<PathBuf>) -> Self {
28 match piper_synthesize(&model_path, text).await { 45 Self {
29 Ok(bytes) => { 46 piper: OnceLock::new(),
30 info!( 47 cache: DashMap::new(),
31 model = %model_path.display(), 48 model_path,
32 text_len = text.len(),
33 bytes = bytes.len(),
34 elapsed_ms = started.elapsed().as_millis() as u64,
35 "Piper TTS synthesized"
36 );
37 return Ok((bytes, "wav"));
38 }
39 Err(e) => {
40 warn!(
41 model = %model_path.display(),
42 text_len = text.len(),
43 elapsed_ms = started.elapsed().as_millis() as u64,
44 "Piper TTS failed, falling back to Google TTS: {e}"
45 );
46 }
47 }
48 } 49 }
49 None => { 50 }
50 debug!("TTS_MODEL_PATH is not set; using Google TTS fallback"); 51
52 /// Get or initialize the Piper model. Returns error if model path is
53 /// unset or model fails to load.
54 async fn get_piper(&self) -> Result<&Mutex<Piper>> {
55 if let Some(piper) = self.piper.get() {
56 return Ok(piper);
51 } 57 }
58
59 let path = self
60 .model_path
61 .as_ref()
62 .ok_or_else(|| Error::Custom("TTS_MODEL_PATH is not set".into()))?;
63
64 // Resolve .onnx.json config path from model path
65 let onnx_path = path.clone();
66 let config_path = {
67 let s = path.to_string_lossy();
68 if s.ends_with(".onnx") {
69 PathBuf::from(format!("{}.json", s))
70 } else {
71 path.clone()
72 }
73 };
74
75 debug!(
76 onnx = %onnx_path.display(),
77 config = %config_path.display(),
78 "loading piper model"
79 );
80 let piper = Piper::new(&onnx_path, &config_path).map_err(|e| {
81 Error::Custom(format!("Failed to load piper model: {e}"))
82 })?;
83 info!("piper model loaded successfully");
84
85 // set() may fail if another task initialized first — that's fine
86 let _ = self.piper.set(Mutex::new(piper));
87 Ok(self.piper.get().unwrap())
52 } 88 }
53 89
54 let started = Instant::now(); 90 /// Look up cached audio for a text string. Returns `None` if uncached
55 let bytes = google_synthesize(http_client, text).await?; 91 /// or expired.
56 info!( 92 fn cache_lookup(&self, text: &str) -> Option<Vec<u8>> {
57 text_len = text.len(), 93 let entry = self.cache.get(text)?;
58 bytes = bytes.len(), 94 if entry.1.elapsed() > CACHE_TTL {
59 elapsed_ms = started.elapsed().as_millis() as u64, 95 drop(entry);
60 "Google TTS synthesized" 96 self.cache.remove(text);
61 ); 97 None
62 Ok((bytes, "mp3")) 98 } else {
63} 99 Some(entry.0.clone())
100 }
101 }
102
103 /// Store audio bytes in the cache.
104 fn cache_store(&self, text: String, audio: Vec<u8>) {
105 self.cache.insert(text, (audio, Instant::now()));
106 }
64 107
65/// Run `piper` as a subprocess, feeding `text` on stdin and collecting the 108 /// Evict all expired cache entries.
66/// generated WAV from `--output_file`. 109 fn cache_evict(&self) {
67async fn piper_synthesize(model_path: &PathBuf, text: &str) -> Result<Vec<u8>> { 110 self.cache
68 tokio::fs::create_dir_all(TEMP_PATH).await.ok(); 111 .retain(|_, (_, inserted_at)| inserted_at.elapsed() <= CACHE_TTL);
69
70 let out_path = format!("{}/tts_piper_{}.wav", TEMP_PATH, uuid::Uuid::new_v4());
71 let model = model_path.to_string_lossy().to_string();
72
73 debug!(model = %model, "spawning piper subprocess");
74 let started = Instant::now();
75
76 let mut child = tokio::process::Command::new("piper")
77 .arg("--model")
78 .arg(&model)
79 .arg("--output_file")
80 .arg(&out_path)
81 .stdin(Stdio::piped())
82 .stdout(Stdio::piped())
83 .stderr(Stdio::piped())
84 .spawn()
85 .map_err(|e| Error::Custom(crate::lang::fmt_tts_piper_launch(&e.to_string())))?;
86
87 {
88 let mut stdin = child
89 .stdin
90 .take()
91 .ok_or_else(|| Error::Custom("Piper stdin unavailable".into()))?;
92 stdin.write_all(text.as_bytes()).await?;
93 stdin.write_all(b"\n").await?;
94 } 112 }
95 113
96 let output = child.wait_with_output().await?; 114 /// Synthesize text → resampled 48 kHz stereo PCM bytes.
97 let elapsed_ms = started.elapsed().as_millis() as u64; 115 ///
116 /// Checks cache first. On miss, runs piper inference, resamples, caches,
117 /// and returns.
118 pub async fn synthesize(&self, text: &str) -> Result<Vec<u8>> {
119 // Cache hit
120 if let Some(cached) = self.cache_lookup(text) {
121 debug!(text_len = text.len(), "tts cache hit");
122 return Ok(cached);
123 }
124
125 // Cache miss — synthesize
126 let piper = self.get_piper().await?;
127 let started = Instant::now();
98 128
99 if !output.status.success() { 129 let (samples, sample_rate) = {
100 let stderr = String::from_utf8_lossy(&output.stderr); 130 let mut lock = piper.lock().await;
101 warn!( 131 lock.create(text, false, None, None, None, None)
102 model = %model, 132 .map_err(|e| Error::Custom(format!("piper synthesis failed: {e}")))?
133 };
134
135 let elapsed_ms = started.elapsed().as_millis() as u64;
136 debug!(
137 text_len = text.len(),
138 samples = samples.len(),
139 sample_rate,
103 elapsed_ms, 140 elapsed_ms,
104 "piper exited with error: {}", 141 "piper synthesized"
105 stderr.trim_end()
106 ); 142 );
107 let _ = tokio::fs::remove_file(&out_path).await; 143
108 return Err(Error::Custom(crate::lang::fmt_tts_piper_failed( 144 // Resample mono → stereo at target sample rate
109 stderr.trim_end(), 145 let pcm_bytes = resample_to_stereo_bytes(&samples, sample_rate, TARGET_SAMPLE_RATE);
110 ))); 146
147 self.cache_store(text.to_string(), pcm_bytes.clone());
148 Ok(pcm_bytes)
111 } 149 }
112 150
113 let bytes = tokio::fs::read(&out_path).await?; 151 /// Pre-generate all 6 event phrases for a display name.
114 debug!( 152 ///
115 model = %model, 153 /// Called on first voice event for a user. Synthesizes all phrases and
116 bytes = bytes.len(), 154 /// caches them so subsequent events are instant.
117 elapsed_ms, 155 pub async fn pre_generate(&self, display_name: &str) {
118 "piper succeeded" 156 let phrases: Vec<String> = EVENT_TYPES
119 ); 157 .iter()
120 let _ = tokio::fs::remove_file(&out_path).await; 158 .map(|evt| make_event_phrase(display_name, evt))
121 Ok(bytes) 159 .collect();
160
161 for phrase in &phrases {
162 if self.cache_lookup(phrase).is_some() {
163 continue;
164 }
165 if let Err(e) = self.synthesize(phrase).await {
166 warn!(
167 display_name,
168 phrase, "pre-generation failed: {e}"
169 );
170 }
171 }
172
173 // Evict expired entries while we're at it
174 self.cache_evict();
175 }
122} 176}
123 177
124/// Fetch Google Translate TTS (Polish) over HTTP. 178// ── Phrase construction ────────────────────────────────────────────────
125async fn google_synthesize(client: &reqwest::Client, text: &str) -> Result<Vec<u8>> { 179
126 let tts_url = format!( 180/// Build the TTS phrase for a voice event + display name.
127 "https://translate.google.com/translate_tts?ie=UTF-8&q={}&tl=pl&client=tw-ob", 181pub fn make_event_phrase(display_name: &str, event: &str) -> String {
128 urlencoding(text) 182 let suffix = match event {
129 ); 183 "joined" => "sie zespawnił",
130 debug!(url_len = tts_url.len(), "fetching Google TTS"); 184 "left" => "jest chujem",
131 let bytes = client.get(&tts_url).send().await?.bytes().await?; 185 "deafened" => "jest bity przez ojca",
132 Ok(bytes.to_vec()) 186 "undeafened" => "uciekł od ojca",
187 "muted" => "idzie robić jajecznice",
188 "unmuted" => "zrobił jajecznice",
189 _ => return format!("{display_name} {event}"),
190 };
191 format!("{display_name} {suffix}")
133} 192}
134 193
135pub fn urlencoding(s: &str) -> String { 194// ── Resampling ─────────────────────────────────────────────────────────
136 let mut out = String::with_capacity(s.len()); 195
137 for &byte in s.as_bytes() { 196/// Resample mono f32 PCM from `src_rate` to `dst_rate`, duplicate to stereo,
138 match byte { 197/// and return as little-endian bytes.
139 b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'-' | b'.' | b'~' => { 198fn resample_to_stereo_bytes(samples: &[f32], src_rate: u32, dst_rate: u32) -> Vec<u8> {
140 out.push(byte as char) 199 if src_rate == dst_rate {
141 } 200 // Just duplicate mono → stereo, no resampling needed
142 b' ' => out.push('+'), 201 let mut out = Vec::with_capacity(samples.len() * 2 * 4);
143 _ => out.push_str(&format!("%{:02X}", byte)), 202 for &s in samples {
203 out.extend_from_slice(&s.to_le_bytes());
204 out.extend_from_slice(&s.to_le_bytes());
144 } 205 }
206 return out;
145 } 207 }
208
209 let ratio = src_rate as f64 / dst_rate as f64;
210 let out_len = (samples.len() as f64 / ratio).ceil() as usize;
211 let mut out = Vec::with_capacity(out_len * 2 * 4);
212
213 for i in 0..out_len {
214 let src_idx = i as f64 * ratio;
215 let idx = src_idx as usize;
216 let frac = src_idx - idx as f64;
217
218 let sample = if idx + 1 < samples.len() {
219 // Linear interpolation
220 samples[idx] + (samples[idx + 1] - samples[idx]) * frac as f32
221 } else if idx < samples.len() {
222 samples[idx]
223 } else {
224 0.0
225 };
226
227 // Mono → stereo duplication
228 out.extend_from_slice(&sample.to_le_bytes());
229 out.extend_from_slice(&sample.to_le_bytes());
230 }
231
146 out 232 out
147} 233}
234
235// ── WAV encoding ───────────────────────────────────────────────────────
236
237/// Encode raw f32 stereo PCM bytes into a WAV file (16-bit PCM).
238///
239/// `pcm_f32_bytes` is the raw little-endian f32 interleaved stereo data
240/// (e.g. from `resample_to_stereo_bytes`).
241pub fn encode_wav(pcm_f32_bytes: &[u8], sample_rate: u32, channels: u16) -> Vec<u8> {
242 let bits_per_sample: u16 = 16;
243 let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8;
244 let block_align = channels * bits_per_sample / 8;
245
246 // Convert f32 samples to i16
247 let sample_count = pcm_f32_bytes.len() / 4; // 4 bytes per f32
248 let mut pcm_i16 = Vec::with_capacity(sample_count * 2);
249 for chunk in pcm_f32_bytes.chunks_exact(4) {
250 let f = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]);
251 let i = (f * 32767.0).clamp(-32768.0, 32767.0) as i16;
252 pcm_i16.extend_from_slice(&i.to_le_bytes());
253 }
254
255 let data_size = pcm_i16.len() as u32;
256 let file_size = 36 + data_size;
257
258 let mut wav = Vec::with_capacity(44 + pcm_i16.len());
259
260 // RIFF header
261 wav.extend_from_slice(b"RIFF");
262 wav.extend_from_slice(&file_size.to_le_bytes());
263 wav.extend_from_slice(b"WAVE");
264
265 // fmt chunk
266 wav.extend_from_slice(b"fmt ");
267 wav.extend_from_slice(&16u32.to_le_bytes()); // chunk size
268 wav.extend_from_slice(&1u16.to_le_bytes()); // PCM format
269 wav.extend_from_slice(&channels.to_le_bytes());
270 wav.extend_from_slice(&sample_rate.to_le_bytes());
271 wav.extend_from_slice(&byte_rate.to_le_bytes());
272 wav.extend_from_slice(&block_align.to_le_bytes());
273 wav.extend_from_slice(&bits_per_sample.to_le_bytes());
274
275 // data chunk
276 wav.extend_from_slice(b"data");
277 wav.extend_from_slice(&data_size.to_le_bytes());
278 wav.extend_from_slice(&pcm_i16);
279
280 wav
281}
282
283// ── Songbird Input helper ──────────────────────────────────────────────
284
285/// Create a songbird `Input` from resampled f32 stereo PCM bytes.
286pub fn pcm_bytes_to_input(pcm_f32_bytes: Vec<u8>) -> songbird::input::Input {
287 use songbird::input::RawAdapter;
288 let cursor = Cursor::new(pcm_f32_bytes);
289 // 2 channels (stereo), 48 kHz — matches TARGET_SAMPLE_RATE
290 RawAdapter::new(cursor, TARGET_SAMPLE_RATE, 2).into()
291}