diff options
Diffstat (limited to '')
| -rw-r--r-- | src/tts.rs | 384 |
1 files changed, 264 insertions, 120 deletions
| @@ -1,147 +1,291 @@ | |||
| 1 | use std::io::Cursor; | ||
| 1 | use std::path::PathBuf; | 2 | use std::path::PathBuf; |
| 2 | use std::process::Stdio; | 3 | use std::sync::OnceLock; |
| 3 | use std::time::Instant; | 4 | use std::time::{Duration, Instant}; |
| 4 | 5 | ||
| 5 | use tokio::io::AsyncWriteExt; | 6 | use dashmap::DashMap; |
| 7 | use piper_rs::Piper; | ||
| 8 | use tokio::sync::Mutex; | ||
| 6 | use tracing::{debug, info, warn}; | 9 | use tracing::{debug, info, warn}; |
| 7 | 10 | ||
| 8 | use crate::config; | ||
| 9 | use crate::config::TEMP_PATH; | ||
| 10 | use crate::error::{Error, Result}; | 11 | use crate::error::{Error, Result}; |
| 11 | 12 | ||
| 12 | /// Synthesize speech for `text`. | 13 | // ── Constants ────────────────────────────────────────────────────────── |
| 13 | /// | 14 | |
| 14 | /// Primary engine is local Piper TTS (via the `piper` CLI). If Piper is not | 15 | /// Target sample rate for songbird playback. |
| 15 | /// installed, fails, or `TTS_MODEL_PATH` is unset, we fall back to Google | 16 | const TARGET_SAMPLE_RATE: u32 = 48_000; |
| 16 | /// Translate TTS over HTTP. | 17 | |
| 18 | /// Cache entries older than this are evicted on access. | ||
| 19 | const CACHE_TTL: Duration = Duration::from_secs(3600); | ||
| 20 | |||
| 21 | /// All voice-state TTS event types. | ||
| 22 | const EVENT_TYPES: &[&str] = &[ | ||
| 23 | "joined", | ||
| 24 | "left", | ||
| 25 | "deafened", | ||
| 26 | "undeafened", | ||
| 27 | "muted", | ||
| 28 | "unmuted", | ||
| 29 | ]; | ||
| 30 | |||
| 31 | // ── TtsState ─────────────────────────────────────────────────────────── | ||
| 32 | |||
| 33 | /// Shared TTS state: piper model instance + audio cache. | ||
| 17 | /// | 34 | /// |
| 18 | /// Returns the encoded audio bytes plus the file extension to use when saving | 35 | /// Held in poise `Data`. Piper is loaded lazily on first synthesis call. |
| 19 | /// (`"wav"` for Piper, `"mp3"` for Google). | 36 | /// Cache maps full text string → resampled 48 kHz stereo PCM bytes. |
| 20 | pub async fn synthesize( | 37 | pub struct TtsState { |
| 21 | config: &config::Config, | 38 | piper: OnceLock<Mutex<Piper>>, |
| 22 | http_client: &reqwest::Client, | 39 | cache: DashMap<String, (Vec<u8>, Instant)>, |
| 23 | text: &str, | 40 | model_path: Option<PathBuf>, |
| 24 | ) -> Result<(Vec<u8>, &'static str)> { | 41 | } |
| 25 | match config.piper_model_path() { | 42 | |
| 26 | Some(model_path) => { | 43 | impl TtsState { |
| 27 | let started = Instant::now(); | 44 | pub fn new(model_path: Option<PathBuf>) -> Self { |
| 28 | match piper_synthesize(&model_path, text).await { | 45 | Self { |
| 29 | Ok(bytes) => { | 46 | piper: OnceLock::new(), |
| 30 | info!( | 47 | cache: DashMap::new(), |
| 31 | model = %model_path.display(), | 48 | model_path, |
| 32 | text_len = text.len(), | ||
| 33 | bytes = bytes.len(), | ||
| 34 | elapsed_ms = started.elapsed().as_millis() as u64, | ||
| 35 | "Piper TTS synthesized" | ||
| 36 | ); | ||
| 37 | return Ok((bytes, "wav")); | ||
| 38 | } | ||
| 39 | Err(e) => { | ||
| 40 | warn!( | ||
| 41 | model = %model_path.display(), | ||
| 42 | text_len = text.len(), | ||
| 43 | elapsed_ms = started.elapsed().as_millis() as u64, | ||
| 44 | "Piper TTS failed, falling back to Google TTS: {e}" | ||
| 45 | ); | ||
| 46 | } | ||
| 47 | } | ||
| 48 | } | 49 | } |
| 49 | None => { | 50 | } |
| 50 | debug!("TTS_MODEL_PATH is not set; using Google TTS fallback"); | 51 | |
| 52 | /// Get or initialize the Piper model. Returns error if model path is | ||
| 53 | /// unset or model fails to load. | ||
| 54 | async fn get_piper(&self) -> Result<&Mutex<Piper>> { | ||
| 55 | if let Some(piper) = self.piper.get() { | ||
| 56 | return Ok(piper); | ||
| 51 | } | 57 | } |
| 58 | |||
| 59 | let path = self | ||
| 60 | .model_path | ||
| 61 | .as_ref() | ||
| 62 | .ok_or_else(|| Error::Custom("TTS_MODEL_PATH is not set".into()))?; | ||
| 63 | |||
| 64 | // Resolve .onnx.json config path from model path | ||
| 65 | let onnx_path = path.clone(); | ||
| 66 | let config_path = { | ||
| 67 | let s = path.to_string_lossy(); | ||
| 68 | if s.ends_with(".onnx") { | ||
| 69 | PathBuf::from(format!("{}.json", s)) | ||
| 70 | } else { | ||
| 71 | path.clone() | ||
| 72 | } | ||
| 73 | }; | ||
| 74 | |||
| 75 | debug!( | ||
| 76 | onnx = %onnx_path.display(), | ||
| 77 | config = %config_path.display(), | ||
| 78 | "loading piper model" | ||
| 79 | ); | ||
| 80 | let piper = Piper::new(&onnx_path, &config_path).map_err(|e| { | ||
| 81 | Error::Custom(format!("Failed to load piper model: {e}")) | ||
| 82 | })?; | ||
| 83 | info!("piper model loaded successfully"); | ||
| 84 | |||
| 85 | // set() may fail if another task initialized first — that's fine | ||
| 86 | let _ = self.piper.set(Mutex::new(piper)); | ||
| 87 | Ok(self.piper.get().unwrap()) | ||
| 52 | } | 88 | } |
| 53 | 89 | ||
| 54 | let started = Instant::now(); | 90 | /// Look up cached audio for a text string. Returns `None` if uncached |
| 55 | let bytes = google_synthesize(http_client, text).await?; | 91 | /// or expired. |
| 56 | info!( | 92 | fn cache_lookup(&self, text: &str) -> Option<Vec<u8>> { |
| 57 | text_len = text.len(), | 93 | let entry = self.cache.get(text)?; |
| 58 | bytes = bytes.len(), | 94 | if entry.1.elapsed() > CACHE_TTL { |
| 59 | elapsed_ms = started.elapsed().as_millis() as u64, | 95 | drop(entry); |
| 60 | "Google TTS synthesized" | 96 | self.cache.remove(text); |
| 61 | ); | 97 | None |
| 62 | Ok((bytes, "mp3")) | 98 | } else { |
| 63 | } | 99 | Some(entry.0.clone()) |
| 100 | } | ||
| 101 | } | ||
| 102 | |||
| 103 | /// Store audio bytes in the cache. | ||
| 104 | fn cache_store(&self, text: String, audio: Vec<u8>) { | ||
| 105 | self.cache.insert(text, (audio, Instant::now())); | ||
| 106 | } | ||
| 64 | 107 | ||
| 65 | /// Run `piper` as a subprocess, feeding `text` on stdin and collecting the | 108 | /// Evict all expired cache entries. |
| 66 | /// generated WAV from `--output_file`. | 109 | fn cache_evict(&self) { |
| 67 | async fn piper_synthesize(model_path: &PathBuf, text: &str) -> Result<Vec<u8>> { | 110 | self.cache |
| 68 | tokio::fs::create_dir_all(TEMP_PATH).await.ok(); | 111 | .retain(|_, (_, inserted_at)| inserted_at.elapsed() <= CACHE_TTL); |
| 69 | |||
| 70 | let out_path = format!("{}/tts_piper_{}.wav", TEMP_PATH, uuid::Uuid::new_v4()); | ||
| 71 | let model = model_path.to_string_lossy().to_string(); | ||
| 72 | |||
| 73 | debug!(model = %model, "spawning piper subprocess"); | ||
| 74 | let started = Instant::now(); | ||
| 75 | |||
| 76 | let mut child = tokio::process::Command::new("piper") | ||
| 77 | .arg("--model") | ||
| 78 | .arg(&model) | ||
| 79 | .arg("--output_file") | ||
| 80 | .arg(&out_path) | ||
| 81 | .stdin(Stdio::piped()) | ||
| 82 | .stdout(Stdio::piped()) | ||
| 83 | .stderr(Stdio::piped()) | ||
| 84 | .spawn() | ||
| 85 | .map_err(|e| Error::Custom(crate::lang::fmt_tts_piper_launch(&e.to_string())))?; | ||
| 86 | |||
| 87 | { | ||
| 88 | let mut stdin = child | ||
| 89 | .stdin | ||
| 90 | .take() | ||
| 91 | .ok_or_else(|| Error::Custom("Piper stdin unavailable".into()))?; | ||
| 92 | stdin.write_all(text.as_bytes()).await?; | ||
| 93 | stdin.write_all(b"\n").await?; | ||
| 94 | } | 112 | } |
| 95 | 113 | ||
| 96 | let output = child.wait_with_output().await?; | 114 | /// Synthesize text → resampled 48 kHz stereo PCM bytes. |
| 97 | let elapsed_ms = started.elapsed().as_millis() as u64; | 115 | /// |
| 116 | /// Checks cache first. On miss, runs piper inference, resamples, caches, | ||
| 117 | /// and returns. | ||
| 118 | pub async fn synthesize(&self, text: &str) -> Result<Vec<u8>> { | ||
| 119 | // Cache hit | ||
| 120 | if let Some(cached) = self.cache_lookup(text) { | ||
| 121 | debug!(text_len = text.len(), "tts cache hit"); | ||
| 122 | return Ok(cached); | ||
| 123 | } | ||
| 124 | |||
| 125 | // Cache miss — synthesize | ||
| 126 | let piper = self.get_piper().await?; | ||
| 127 | let started = Instant::now(); | ||
| 98 | 128 | ||
| 99 | if !output.status.success() { | 129 | let (samples, sample_rate) = { |
| 100 | let stderr = String::from_utf8_lossy(&output.stderr); | 130 | let mut lock = piper.lock().await; |
| 101 | warn!( | 131 | lock.create(text, false, None, None, None, None) |
| 102 | model = %model, | 132 | .map_err(|e| Error::Custom(format!("piper synthesis failed: {e}")))? |
| 133 | }; | ||
| 134 | |||
| 135 | let elapsed_ms = started.elapsed().as_millis() as u64; | ||
| 136 | debug!( | ||
| 137 | text_len = text.len(), | ||
| 138 | samples = samples.len(), | ||
| 139 | sample_rate, | ||
| 103 | elapsed_ms, | 140 | elapsed_ms, |
| 104 | "piper exited with error: {}", | 141 | "piper synthesized" |
| 105 | stderr.trim_end() | ||
| 106 | ); | 142 | ); |
| 107 | let _ = tokio::fs::remove_file(&out_path).await; | 143 | |
| 108 | return Err(Error::Custom(crate::lang::fmt_tts_piper_failed( | 144 | // Resample mono → stereo at target sample rate |
| 109 | stderr.trim_end(), | 145 | let pcm_bytes = resample_to_stereo_bytes(&samples, sample_rate, TARGET_SAMPLE_RATE); |
| 110 | ))); | 146 | |
| 147 | self.cache_store(text.to_string(), pcm_bytes.clone()); | ||
| 148 | Ok(pcm_bytes) | ||
| 111 | } | 149 | } |
| 112 | 150 | ||
| 113 | let bytes = tokio::fs::read(&out_path).await?; | 151 | /// Pre-generate all 6 event phrases for a display name. |
| 114 | debug!( | 152 | /// |
| 115 | model = %model, | 153 | /// Called on first voice event for a user. Synthesizes all phrases and |
| 116 | bytes = bytes.len(), | 154 | /// caches them so subsequent events are instant. |
| 117 | elapsed_ms, | 155 | pub async fn pre_generate(&self, display_name: &str) { |
| 118 | "piper succeeded" | 156 | let phrases: Vec<String> = EVENT_TYPES |
| 119 | ); | 157 | .iter() |
| 120 | let _ = tokio::fs::remove_file(&out_path).await; | 158 | .map(|evt| make_event_phrase(display_name, evt)) |
| 121 | Ok(bytes) | 159 | .collect(); |
| 160 | |||
| 161 | for phrase in &phrases { | ||
| 162 | if self.cache_lookup(phrase).is_some() { | ||
| 163 | continue; | ||
| 164 | } | ||
| 165 | if let Err(e) = self.synthesize(phrase).await { | ||
| 166 | warn!( | ||
| 167 | display_name, | ||
| 168 | phrase, "pre-generation failed: {e}" | ||
| 169 | ); | ||
| 170 | } | ||
| 171 | } | ||
| 172 | |||
| 173 | // Evict expired entries while we're at it | ||
| 174 | self.cache_evict(); | ||
| 175 | } | ||
| 122 | } | 176 | } |
| 123 | 177 | ||
| 124 | /// Fetch Google Translate TTS (Polish) over HTTP. | 178 | // ── Phrase construction ──────────────────────────────────────────────── |
| 125 | async fn google_synthesize(client: &reqwest::Client, text: &str) -> Result<Vec<u8>> { | 179 | |
| 126 | let tts_url = format!( | 180 | /// Build the TTS phrase for a voice event + display name. |
| 127 | "https://translate.google.com/translate_tts?ie=UTF-8&q={}&tl=pl&client=tw-ob", | 181 | pub fn make_event_phrase(display_name: &str, event: &str) -> String { |
| 128 | urlencoding(text) | 182 | let suffix = match event { |
| 129 | ); | 183 | "joined" => "sie zespawnił", |
| 130 | debug!(url_len = tts_url.len(), "fetching Google TTS"); | 184 | "left" => "jest chujem", |
| 131 | let bytes = client.get(&tts_url).send().await?.bytes().await?; | 185 | "deafened" => "jest bity przez ojca", |
| 132 | Ok(bytes.to_vec()) | 186 | "undeafened" => "uciekł od ojca", |
| 187 | "muted" => "idzie robić jajecznice", | ||
| 188 | "unmuted" => "zrobił jajecznice", | ||
| 189 | _ => return format!("{display_name} {event}"), | ||
| 190 | }; | ||
| 191 | format!("{display_name} {suffix}") | ||
| 133 | } | 192 | } |
| 134 | 193 | ||
| 135 | pub fn urlencoding(s: &str) -> String { | 194 | // ── Resampling ───────────────────────────────────────────────────────── |
| 136 | let mut out = String::with_capacity(s.len()); | 195 | |
| 137 | for &byte in s.as_bytes() { | 196 | /// Resample mono f32 PCM from `src_rate` to `dst_rate`, duplicate to stereo, |
| 138 | match byte { | 197 | /// and return as little-endian bytes. |
| 139 | b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'-' | b'.' | b'~' => { | 198 | fn resample_to_stereo_bytes(samples: &[f32], src_rate: u32, dst_rate: u32) -> Vec<u8> { |
| 140 | out.push(byte as char) | 199 | if src_rate == dst_rate { |
| 141 | } | 200 | // Just duplicate mono → stereo, no resampling needed |
| 142 | b' ' => out.push('+'), | 201 | let mut out = Vec::with_capacity(samples.len() * 2 * 4); |
| 143 | _ => out.push_str(&format!("%{:02X}", byte)), | 202 | for &s in samples { |
| 203 | out.extend_from_slice(&s.to_le_bytes()); | ||
| 204 | out.extend_from_slice(&s.to_le_bytes()); | ||
| 144 | } | 205 | } |
| 206 | return out; | ||
| 145 | } | 207 | } |
| 208 | |||
| 209 | let ratio = src_rate as f64 / dst_rate as f64; | ||
| 210 | let out_len = (samples.len() as f64 / ratio).ceil() as usize; | ||
| 211 | let mut out = Vec::with_capacity(out_len * 2 * 4); | ||
| 212 | |||
| 213 | for i in 0..out_len { | ||
| 214 | let src_idx = i as f64 * ratio; | ||
| 215 | let idx = src_idx as usize; | ||
| 216 | let frac = src_idx - idx as f64; | ||
| 217 | |||
| 218 | let sample = if idx + 1 < samples.len() { | ||
| 219 | // Linear interpolation | ||
| 220 | samples[idx] + (samples[idx + 1] - samples[idx]) * frac as f32 | ||
| 221 | } else if idx < samples.len() { | ||
| 222 | samples[idx] | ||
| 223 | } else { | ||
| 224 | 0.0 | ||
| 225 | }; | ||
| 226 | |||
| 227 | // Mono → stereo duplication | ||
| 228 | out.extend_from_slice(&sample.to_le_bytes()); | ||
| 229 | out.extend_from_slice(&sample.to_le_bytes()); | ||
| 230 | } | ||
| 231 | |||
| 146 | out | 232 | out |
| 147 | } | 233 | } |
| 234 | |||
| 235 | // ── WAV encoding ─────────────────────────────────────────────────────── | ||
| 236 | |||
| 237 | /// Encode raw f32 stereo PCM bytes into a WAV file (16-bit PCM). | ||
| 238 | /// | ||
| 239 | /// `pcm_f32_bytes` is the raw little-endian f32 interleaved stereo data | ||
| 240 | /// (e.g. from `resample_to_stereo_bytes`). | ||
| 241 | pub fn encode_wav(pcm_f32_bytes: &[u8], sample_rate: u32, channels: u16) -> Vec<u8> { | ||
| 242 | let bits_per_sample: u16 = 16; | ||
| 243 | let byte_rate = sample_rate * channels as u32 * bits_per_sample as u32 / 8; | ||
| 244 | let block_align = channels * bits_per_sample / 8; | ||
| 245 | |||
| 246 | // Convert f32 samples to i16 | ||
| 247 | let sample_count = pcm_f32_bytes.len() / 4; // 4 bytes per f32 | ||
| 248 | let mut pcm_i16 = Vec::with_capacity(sample_count * 2); | ||
| 249 | for chunk in pcm_f32_bytes.chunks_exact(4) { | ||
| 250 | let f = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); | ||
| 251 | let i = (f * 32767.0).clamp(-32768.0, 32767.0) as i16; | ||
| 252 | pcm_i16.extend_from_slice(&i.to_le_bytes()); | ||
| 253 | } | ||
| 254 | |||
| 255 | let data_size = pcm_i16.len() as u32; | ||
| 256 | let file_size = 36 + data_size; | ||
| 257 | |||
| 258 | let mut wav = Vec::with_capacity(44 + pcm_i16.len()); | ||
| 259 | |||
| 260 | // RIFF header | ||
| 261 | wav.extend_from_slice(b"RIFF"); | ||
| 262 | wav.extend_from_slice(&file_size.to_le_bytes()); | ||
| 263 | wav.extend_from_slice(b"WAVE"); | ||
| 264 | |||
| 265 | // fmt chunk | ||
| 266 | wav.extend_from_slice(b"fmt "); | ||
| 267 | wav.extend_from_slice(&16u32.to_le_bytes()); // chunk size | ||
| 268 | wav.extend_from_slice(&1u16.to_le_bytes()); // PCM format | ||
| 269 | wav.extend_from_slice(&channels.to_le_bytes()); | ||
| 270 | wav.extend_from_slice(&sample_rate.to_le_bytes()); | ||
| 271 | wav.extend_from_slice(&byte_rate.to_le_bytes()); | ||
| 272 | wav.extend_from_slice(&block_align.to_le_bytes()); | ||
| 273 | wav.extend_from_slice(&bits_per_sample.to_le_bytes()); | ||
| 274 | |||
| 275 | // data chunk | ||
| 276 | wav.extend_from_slice(b"data"); | ||
| 277 | wav.extend_from_slice(&data_size.to_le_bytes()); | ||
| 278 | wav.extend_from_slice(&pcm_i16); | ||
| 279 | |||
| 280 | wav | ||
| 281 | } | ||
| 282 | |||
| 283 | // ── Songbird Input helper ────────────────────────────────────────────── | ||
| 284 | |||
| 285 | /// Create a songbird `Input` from resampled f32 stereo PCM bytes. | ||
| 286 | pub fn pcm_bytes_to_input(pcm_f32_bytes: Vec<u8>) -> songbird::input::Input { | ||
| 287 | use songbird::input::RawAdapter; | ||
| 288 | let cursor = Cursor::new(pcm_f32_bytes); | ||
| 289 | // 2 channels (stereo), 48 kHz — matches TARGET_SAMPLE_RATE | ||
| 290 | RawAdapter::new(cursor, TARGET_SAMPLE_RATE, 2).into() | ||
| 291 | } | ||
