| 1 | //! Voice input commands — `/voice`, `/voice-send`, `/voice-control`. |
| 2 | //! |
| 3 | //! Records audio from the default microphone, sends it to the configured |
| 4 | //! provider's API for transcription, and inserts the transcribed text into |
| 5 | //! the composer. The interaction model mirrors MiMo Code's voice UX: |
| 6 | //! |
| 7 | //! `/voice` — toggle voice input on/off (records when toggled on) |
| 8 | //! `/voice-send` — toggle auto-send when the transcript ends with |
| 9 | //! "send it" / "发送" |
| 10 | //! `/voice-control` — toggle AI-assisted dictation that sees the current |
| 11 | //! composer text |
| 12 | //! |
| 13 | //! The slash commands only flip state and emit [`AppAction::VoiceCapture`]; |
| 14 | //! the actual capture runs in the UI event loop where the live [`Config`] |
| 15 | //! supplies provider credentials. That keeps the handlers side-effect free |
| 16 | //! (the registry smoke tests execute every command) and avoids caching |
| 17 | //! auth material on [`App`]. |
| 18 | //! |
| 19 | //! ## Recording |
| 20 | //! |
| 21 | //! Uses platform-specific command-line tools (sox, rec, arecord) to capture |
| 22 | //! 16kHz mono 16-bit PCM audio. Records until a silence gap is detected or |
| 23 | //! the maximum duration is reached (default 10 s). |
| 24 | |
| 25 | use std::process::{Command, Stdio}; |
| 26 | use std::sync::LazyLock; |
| 27 | use std::time::Duration; |
| 28 | |
| 29 | use regex::Regex; |
| 30 | |
| 31 | use crate::commands::CommandResult; |
| 32 | use crate::commands::traits::{CommandInfo, RegisterCommand}; |
| 33 | use crate::config::Config; |
| 34 | use crate::localization::{MessageId, tr}; |
| 35 | use crate::tui::app::{App, AppAction}; |
| 36 | |
| 37 | /// Transcription model requested from the provider's chat-completions API. |
| 38 | const ASR_MODEL: &str = "mimo-v2.5-asr"; |
| 39 | /// Free ASR: Groq Whisper (cloud, free tier) — fast, cross-platform, no local model download. |
| 40 | #[allow(dead_code)] |
| 41 | const GROQ_ASR_URL: &str = "https://api.groq.com/openai/v1/audio/transcriptions"; |
| 42 | const GROQ_ASR_MODEL: &str = "whisper-large-v3-turbo"; |
| 43 | /// Local whisper binary names to probe (whisper.cpp, faster-whisper, OpenAI whisper). |
| 44 | const LOCAL_WHISPER_BINS: &[&str] = &["whisper", "whisper.cpp", "whisper-cpp", "faster-whisper"]; |
| 45 | /// Model used for the AI-assisted voice-control pipeline. |
| 46 | const VOICE_CONTROL_MODEL: &str = "mimo-v2.5"; |
| 47 | |
| 48 | pub(in crate::commands) const VOICE_INFO: CommandInfo = CommandInfo { |
| 49 | name: "voice", |
| 50 | aliases: &["yuyin", "语音"], |
| 51 | usage: "/voice", |
| 52 | description_id: MessageId::CmdVoiceDescription, |
| 53 | }; |
| 54 | |
| 55 | pub(in crate::commands) const VOICE_SEND_INFO: CommandInfo = CommandInfo { |
| 56 | name: "voicesend", |
| 57 | aliases: &["voice-send", "yuyinsend", "语音发送"], |
| 58 | usage: "/voicesend", |
| 59 | description_id: MessageId::CmdVoiceSendDescription, |
| 60 | }; |
| 61 | |
| 62 | pub(in crate::commands) const VOICE_CONTROL_INFO: CommandInfo = CommandInfo { |
| 63 | name: "voicecontrol", |
| 64 | aliases: &["voice-control", "yuyincontrol", "语音控制"], |
| 65 | usage: "/voicecontrol", |
| 66 | description_id: MessageId::CmdVoiceControlDescription, |
| 67 | }; |
| 68 | |
| 69 | pub(in crate::commands) struct VoiceCmd; |
| 70 | pub(in crate::commands) struct VoiceSendCmd; |
| 71 | pub(in crate::commands) struct VoiceControlCmd; |
| 72 | |
| 73 | impl RegisterCommand for VoiceCmd { |
| 74 | fn info() -> &'static CommandInfo { |
| 75 | &VOICE_INFO |
| 76 | } |
| 77 | |
| 78 | fn execute(app: &mut App, _arg: Option<&str>) -> CommandResult { |
| 79 | voice(app) |
| 80 | } |
| 81 | } |
| 82 | |
| 83 | impl RegisterCommand for VoiceSendCmd { |
| 84 | fn info() -> &'static CommandInfo { |
| 85 | &VOICE_SEND_INFO |
| 86 | } |
| 87 | |
| 88 | fn execute(app: &mut App, _arg: Option<&str>) -> CommandResult { |
| 89 | voice_send(app) |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | impl RegisterCommand for VoiceControlCmd { |
| 94 | fn info() -> &'static CommandInfo { |
| 95 | &VOICE_CONTROL_INFO |
| 96 | } |
| 97 | |
| 98 | fn execute(app: &mut App, _arg: Option<&str>) -> CommandResult { |
| 99 | voice_control(app) |
| 100 | } |
| 101 | } |
| 102 | |
| 103 | // --- Recorder detection ---------------------------------------------------- |
| 104 | |
| 105 | /// Platform-specific recorder definitions. |
| 106 | #[derive(Debug, Clone)] |
| 107 | struct Recorder { |
| 108 | cmd: &'static str, |
| 109 | /// CLI arguments for piping raw 16kHz mono S16_LE PCM to stdout. |
| 110 | pipe_args: &'static [&'static str], |
| 111 | } |
| 112 | |
| 113 | fn detect_recorder() -> Option<Recorder> { |
| 114 | let candidates: &[Recorder] = if cfg!(target_os = "macos") { |
| 115 | &[ |
| 116 | Recorder { |
| 117 | cmd: "sox", |
| 118 | pipe_args: &["-d", "-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"], |
| 119 | }, |
| 120 | Recorder { |
| 121 | cmd: "rec", |
| 122 | pipe_args: &["-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"], |
| 123 | }, |
| 124 | ] |
| 125 | } else if cfg!(target_os = "linux") { |
| 126 | &[ |
| 127 | Recorder { |
| 128 | cmd: "arecord", |
| 129 | pipe_args: &["-f", "S16_LE", "-r", "16000", "-c", "1", "-t", "raw"], |
| 130 | }, |
| 131 | Recorder { |
| 132 | cmd: "sox", |
| 133 | pipe_args: &["-d", "-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"], |
| 134 | }, |
| 135 | ] |
| 136 | } else if cfg!(target_os = "windows") { |
| 137 | &[Recorder { |
| 138 | cmd: "sox", |
| 139 | pipe_args: &["-d", "-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"], |
| 140 | }] |
| 141 | } else { |
| 142 | &[] |
| 143 | }; |
| 144 | |
| 145 | candidates |
| 146 | .iter() |
| 147 | .find(|r| { |
| 148 | Command::new(r.cmd) |
| 149 | .arg("--version") |
| 150 | .stdin(Stdio::null()) |
| 151 | .stdout(Stdio::null()) |
| 152 | .stderr(Stdio::null()) |
| 153 | .spawn() |
| 154 | .is_ok() |
| 155 | }) |
| 156 | .cloned() |
| 157 | } |
| 158 | |
| 159 | /// Check whether voice recording is available on this system. |
| 160 | pub fn is_available() -> bool { |
| 161 | detect_recorder().is_some() |
| 162 | } |
| 163 | |
| 164 | // --- WAV encoding ---------------------------------------------------------- |
| 165 | |
| 166 | /// Encode raw 16kHz mono S16_LE PCM samples as a WAV buffer. |
| 167 | fn encode_wav(samples: &[i16]) -> Vec<u8> { |
| 168 | let data_size = (samples.len() * 2) as u32; |
| 169 | let sample_rate: u32 = 16000; |
| 170 | let mut buf = Vec::with_capacity(44 + data_size as usize); |
| 171 | |
| 172 | // RIFF header |
| 173 | buf.extend_from_slice(b"RIFF"); |
| 174 | buf.extend_from_slice(&(36 + data_size).to_le_bytes()); |
| 175 | buf.extend_from_slice(b"WAVE"); |
| 176 | |
| 177 | // fmt chunk |
| 178 | buf.extend_from_slice(b"fmt "); |
| 179 | buf.extend_from_slice(&16u32.to_le_bytes()); // chunk size |
| 180 | buf.extend_from_slice(&1u16.to_le_bytes()); // PCM |
| 181 | buf.extend_from_slice(&1u16.to_le_bytes()); // mono |
| 182 | buf.extend_from_slice(&sample_rate.to_le_bytes()); |
| 183 | buf.extend_from_slice(&(sample_rate * 2).to_le_bytes()); // byte rate |
| 184 | buf.extend_from_slice(&2u16.to_le_bytes()); // block align |
| 185 | buf.extend_from_slice(&16u16.to_le_bytes()); // bits per sample |
| 186 | |
| 187 | // data chunk |
| 188 | buf.extend_from_slice(b"data"); |
| 189 | buf.extend_from_slice(&data_size.to_le_bytes()); |
| 190 | for &sample in samples { |
| 191 | buf.extend_from_slice(&sample.to_le_bytes()); |
| 192 | } |
| 193 | |
| 194 | buf |
| 195 | } |
| 196 | |
| 197 | // --- Recording ------------------------------------------------------------- |
| 198 | |
| 199 | /// Maximum recording duration in seconds before auto-stopping. |
| 200 | const MAX_RECORD_SECS: u64 = 10; |
| 201 | /// Minimum segment duration in seconds to consider as valid speech. |
| 202 | const MIN_SEGMENT_SECS: f64 = 0.3; |
| 203 | |
| 204 | /// Record audio from the default microphone. |
| 205 | /// |
| 206 | /// Returns raw 16kHz mono S16_LE PCM samples. Returns `None` if no recorder |
| 207 | /// is available, the recording failed, or no speech was detected. |
| 208 | fn record_audio() -> Option<(Vec<i16>, Duration)> { |
| 209 | let recorder = detect_recorder()?; |
| 210 | let start = std::time::Instant::now(); |
| 211 | |
| 212 | let mut child = Command::new(recorder.cmd) |
| 213 | .args(recorder.pipe_args) |
| 214 | .stdin(Stdio::null()) |
| 215 | .stdout(Stdio::piped()) |
| 216 | .stderr(Stdio::null()) |
| 217 | .spawn() |
| 218 | .ok()?; |
| 219 | |
| 220 | let stdout = child.stdout.take()?; |
| 221 | let mut reader = std::io::BufReader::new(stdout); |
| 222 | let mut all_samples: Vec<i16> = Vec::with_capacity(16000 * MAX_RECORD_SECS as usize); |
| 223 | |
| 224 | // Read until timeout or silence |
| 225 | let mut buf = [0u8; 320]; // 10ms of 16kHz S16_LE |
| 226 | let max_duration = Duration::from_secs(MAX_RECORD_SECS); |
| 227 | let mut silence_samples = 0u32; |
| 228 | let mut had_speech = false; |
| 229 | let speech_threshold: i16 = 500; // RMS-based speech detection threshold |
| 230 | let silence_duration_samples = 16000u32; // 1 second of silence to stop |
| 231 | |
| 232 | loop { |
| 233 | use std::io::Read; |
| 234 | match reader.read_exact(&mut buf) { |
| 235 | Ok(()) => { |
| 236 | let chunk: Vec<i16> = buf |
| 237 | .chunks_exact(2) |
| 238 | .map(|b| i16::from_le_bytes([b[0], b[1]])) |
| 239 | .collect(); |
| 240 | |
| 241 | // Simple RMS-based VAD |
| 242 | let rms = (chunk.iter().map(|&s| (s as f64) * (s as f64)).sum::<f64>() |
| 243 | / chunk.len() as f64) |
| 244 | .sqrt(); |
| 245 | let is_speech = rms > speech_threshold as f64; |
| 246 | |
| 247 | if is_speech { |
| 248 | had_speech = true; |
| 249 | silence_samples = 0; |
| 250 | } else if had_speech { |
| 251 | silence_samples += chunk.len() as u32; |
| 252 | } |
| 253 | |
| 254 | if had_speech { |
| 255 | all_samples.extend_from_slice(&chunk); |
| 256 | } |
| 257 | |
| 258 | if start.elapsed() > max_duration { |
| 259 | let _ = child.kill(); |
| 260 | break; |
| 261 | } |
| 262 | if had_speech && silence_samples >= silence_duration_samples { |
| 263 | let _ = child.kill(); |
| 264 | break; |
| 265 | } |
| 266 | } |
| 267 | Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break, |
| 268 | Err(_) => { |
| 269 | let _ = child.kill(); |
| 270 | break; |
| 271 | } |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | let _ = child.wait(); |
| 276 | let elapsed = start.elapsed(); |
| 277 | |
| 278 | let min_samples = (MIN_SEGMENT_SECS * 16000.0) as usize; |
| 279 | if all_samples.len() < min_samples { |
| 280 | return None; |
| 281 | } |
| 282 | |
| 283 | Some((all_samples, elapsed)) |
| 284 | } |
| 285 | |
| 286 | // --- Auto-send suffix ------------------------------------------------------ |
| 287 | |
| 288 | /// Matches an explicit send instruction at the end of transcribed text: |
| 289 | /// "send it" (any spacing/case) or 发送/發送, with trailing punctuation. |
| 290 | static SEND_SUFFIX_RE: LazyLock<Regex> = LazyLock::new(|| { |
| 291 | Regex::new(r"(?i)(?:^|[\s,，.。!！?？]+)(?:send\s*it|发送|發送)[\s.。!！?？]*$").unwrap() |
| 292 | }); |
| 293 | |
| 294 | /// Split a transcript into the message remainder and whether it ended with an |
| 295 | /// explicit send instruction. `"ship the fix, send it"` → `("ship the fix", true)`. |
| 296 | fn split_send_suffix(text: &str) -> (&str, bool) { |
| 297 | match SEND_SUFFIX_RE.find(text) { |
| 298 | Some(found) => (text[..found.start()].trim(), true), |
| 299 | None => (text.trim(), false), |
| 300 | } |
| 301 | } |
| 302 | |
| 303 | // --- Transcription --------------------------------------------------------- |
| 304 | |
| 305 | fn base64_encode(data: &[u8]) -> String { |
| 306 | use base64::Engine; |
| 307 | base64::engine::general_purpose::STANDARD.encode(data) |
| 308 | } |
| 309 | |
| 310 | fn chat_completions_url(base_url: &str) -> String { |
| 311 | format!("{}/chat/completions", base_url.trim_end_matches('/')) |
| 312 | } |
| 313 | |
| 314 | async fn post_chat_completions( |
| 315 | api_key: &str, |
| 316 | base_url: &str, |
| 317 | body: serde_json::Value, |
| 318 | ) -> Result<serde_json::Value, String> { |
| 319 | let client = crate::tls::reqwest_client(); |
| 320 | let resp = client |
| 321 | .post(chat_completions_url(base_url)) |
| 322 | .header("Content-Type", "application/json") |
| 323 | .header("Authorization", format!("Bearer {api_key}")) |
| 324 | .timeout(Duration::from_secs(30)) |
| 325 | .json(&body) |
| 326 | .send() |
| 327 | .await |
| 328 | .map_err(|e| format!("request failed: {e}"))?; |
| 329 | |
| 330 | if !resp.status().is_success() { |
| 331 | return Err(format!("API returned status {}", resp.status())); |
| 332 | } |
| 333 | |
| 334 | resp.json() |
| 335 | .await |
| 336 | .map_err(|e| format!("failed to parse response: {e}")) |
| 337 | } |
| 338 | |
| 339 | /// Send audio to the provider's API for plain transcription. |
| 340 | /// |
| 341 | /// Uses the chat completions endpoint with `input_audio` content blocks. |
| 342 | async fn transcribe( |
| 343 | api_key: &str, |
| 344 | base_url: &str, |
| 345 | audio_samples: &[i16], |
| 346 | ) -> Result<String, String> { |
| 347 | transcribe_with_model(api_key, base_url, audio_samples, ASR_MODEL).await |
| 348 | } |
| 349 | |
| 350 | async fn transcribe_with_model( |
| 351 | api_key: &str, |
| 352 | base_url: &str, |
| 353 | audio_samples: &[i16], |
| 354 | model: &str, |
| 355 | ) -> Result<String, String> { |
| 356 | let wav = encode_wav(audio_samples); |
| 357 | let data_url = format!("data:audio/wav;base64,{}", base64_encode(&wav)); |
| 358 | |
| 359 | let body = serde_json::json!({ |
| 360 | "model": model, |
| 361 | "messages": [ |
| 362 | { |
| 363 | "role": "user", |
| 364 | "content": [ |
| 365 | { |
| 366 | "type": "input_audio", |
| 367 | "input_audio": { |
| 368 | "data": data_url |
| 369 | } |
| 370 | } |
| 371 | ] |
| 372 | } |
| 373 | ], |
| 374 | "asr_options": { |
| 375 | "language": "auto" |
| 376 | } |
| 377 | }); |
| 378 | |
| 379 | let data = post_chat_completions(api_key, base_url, body).await?; |
| 380 | data["choices"][0]["message"]["content"] |
| 381 | .as_str() |
| 382 | .map(|s| s.trim().to_string()) |
| 383 | .ok_or_else(|| "no transcription in response".to_string()) |
| 384 | } |
| 385 | |
| 386 | /// Process audio through the voice-control pipeline: AI-assisted dictation |
| 387 | /// that sees the current composer text, mirroring MiMo Code's |
| 388 | /// `processVoiceControl`. Used when `/voice-control` is enabled. |
| 389 | async fn process_voice_control( |
| 390 | api_key: &str, |
| 391 | base_url: &str, |
| 392 | audio_samples: &[i16], |
| 393 | current_text: &str, |
| 394 | ) -> Result<String, String> { |
| 395 | let wav = encode_wav(audio_samples); |
| 396 | let data_url = format!("data:audio/wav;base64,{}", base64_encode(&wav)); |
| 397 | |
| 398 | let user_context = serde_json::json!({ |
| 399 | "current_text": current_text, |
| 400 | "cursor": "end", |
| 401 | }); |
| 402 | |
| 403 | let body = serde_json::json!({ |
| 404 | "model": VOICE_CONTROL_MODEL, |
| 405 | "messages": [ |
| 406 | { |
| 407 | "role": "system", |
| 408 | "content": "You are a voice input assistant. Transcribe the user's speech. Output JSON: {\"text\": \"transcribed text\"}." |
| 409 | }, |
| 410 | { |
| 411 | "role": "user", |
| 412 | "content": [ |
| 413 | { "type": "text", "text": user_context.to_string() }, |
| 414 | { "type": "input_audio", "input_audio": { "data": data_url } } |
| 415 | ] |
| 416 | } |
| 417 | ], |
| 418 | "response_format": { "type": "json_object" } |
| 419 | }); |
| 420 | |
| 421 | let data = post_chat_completions(api_key, base_url, body).await?; |
| 422 | let content = data["choices"][0]["message"]["content"] |
| 423 | .as_str() |
| 424 | .ok_or_else(|| "no response content".to_string())?; |
| 425 | |
| 426 | let parsed: serde_json::Value = serde_json::from_str(content) |
| 427 | .map_err(|e| format!("failed to parse voice control JSON: {e}"))?; |
| 428 | |
| 429 | parsed["text"] |
| 430 | .as_str() |
| 431 | .map(|s| s.to_string()) |
| 432 | .ok_or_else(|| "no text field in voice control response".to_string()) |
| 433 | } |
| 434 | |
| 435 | // --- Capture orchestration (UI event loop) --------------------------------- |
| 436 | |
| 437 | /// What the UI should do with a finished capture. |
| 438 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 439 | pub enum VoiceCaptureOutcome { |
| 440 | /// Insert the transcribed text into the composer at the cursor. |
| 441 | Insert(String), |
| 442 | /// Submit this text as a message (auto-send). |
| 443 | Send(String), |
| 444 | } |
| 445 | |
| 446 | /// Detect best free ASR for this host — local whisper > Groq free > provider fallback. |
| 447 | /// Works on macOS (brew install whisper-cpp), Windows (whisper.cpp binary), |
| 448 | /// Linux (apt), and HarmonyOS (falls back to cloud). |
| 449 | fn detect_free_asr() -> &'static str { |
| 450 | for bin in LOCAL_WHISPER_BINS { |
| 451 | if Command::new(bin) |
| 452 | .arg("--help") |
| 453 | .stdin(Stdio::null()) |
| 454 | .stdout(Stdio::null()) |
| 455 | .stderr(Stdio::null()) |
| 456 | .spawn() |
| 457 | .is_ok() |
| 458 | { |
| 459 | return "local-whisper"; |
| 460 | } |
| 461 | } |
| 462 | if std::env::var("GROQ_API_KEY").is_ok_and(|v| !v.trim().is_empty()) { |
| 463 | return "groq"; |
| 464 | } |
| 465 | "provider" |
| 466 | } |
| 467 | |
| 468 | /// Transcribe via local whisper.cpp (free, offline, cross-platform). |
| 469 | async fn transcribe_local_whisper(audio_samples: &[i16]) -> Result<String, String> { |
| 470 | let wav = encode_wav(audio_samples); |
| 471 | let tmp = std::env::temp_dir().join(format!("cw-voice-{}.wav", std::process::id())); |
| 472 | std::fs::write(&tmp, &wav).map_err(|e| e.to_string())?; |
| 473 | // Try each local binary until one succeeds; whisper.cpp outputs to stdout or file. |
| 474 | for bin in LOCAL_WHISPER_BINS { |
| 475 | let output = Command::new(bin) |
| 476 | .arg(tmp.to_string_lossy().as_ref()) |
| 477 | .arg("--model") |
| 478 | .arg("tiny") |
| 479 | .arg("--language") |
| 480 | .arg("auto") |
| 481 | .arg("--output-txt") |
| 482 | .output(); |
| 483 | if let Ok(out) = output { |
| 484 | if out.status.success() { |
| 485 | let txt = String::from_utf8_lossy(&out.stdout).trim().to_string(); |
| 486 | let _ = std::fs::remove_file(&tmp); |
| 487 | if !txt.is_empty() { |
| 488 | return Ok(txt); |
| 489 | } |
| 490 | // Some builds write to .txt sidecar |
| 491 | let sidecar = tmp.with_extension("txt"); |
| 492 | if let Ok(s) = std::fs::read_to_string(&sidecar) { |
| 493 | let _ = std::fs::remove_file(&sidecar); |
| 494 | let _ = std::fs::remove_file(&tmp); |
| 495 | if !s.trim().is_empty() { |
| 496 | return Ok(s.trim().to_string()); |
| 497 | } |
| 498 | } |
| 499 | } |
| 500 | } |
| 501 | } |
| 502 | let _ = std::fs::remove_file(&tmp); |
| 503 | Err("local whisper not available".into()) |
| 504 | } |
| 505 | |
| 506 | /// Transcribe via Groq Whisper large-v3-turbo (free tier, ~$0.04/hr, fast). |
| 507 | /// Groq is NOT a full CodeWhale provider yet — this is a direct ASR call |
| 508 | /// using `GROQ_API_KEY` only (no provider setup needed). Uses the same |
| 509 | /// chat-completions `input_audio` path as Xiaomi so no `multipart` feature. |
| 510 | async fn transcribe_groq(audio_samples: &[i16]) -> Result<String, String> { |
| 511 | let api_key = std::env::var("GROQ_API_KEY").map_err(|_| "GROQ_API_KEY not set".to_string())?; |
| 512 | let base_url = "https://api.groq.com/openai/v1"; |
| 513 | transcribe_with_model(&api_key, base_url, audio_samples, GROQ_ASR_MODEL).await |
| 514 | } |
| 515 | |
| 516 | /// Perform a complete record + transcribe cycle with live interim display. |
| 517 | /// |
| 518 | /// Runs in the UI event loop (see [`AppAction::VoiceCapture`]) so provider |
| 519 | /// credentials come from the live [`Config`] rather than state cached on |
| 520 | /// [`App`]. Recording happens on a blocking thread; transcription uses the |
| 521 | /// shared async HTTP client. Every failure path returns a localized message |
| 522 | /// so callers can surface it as a status line. |
| 523 | /// Resolve ASR model/provider preference. |
| 524 | /// Priority: explicit config `voice.asr_model` > env `CODEWHALE_ASR_MODEL` > auto-detect (local-whisper > groq > xiaomi). |
| 525 | fn resolve_asr_choice(_config: &Config) -> (String, String) { |
| 526 | // Check explicit env override first (free, cross-platform) |
| 527 | if let Ok(m) = std::env::var("CODEWHALE_ASR_MODEL") { |
| 528 | let m = m.trim().to_ascii_lowercase(); |
| 529 | if m.contains("groq") || m.contains("whisper") { |
| 530 | return ("groq".into(), GROQ_ASR_MODEL.into()); |
| 531 | } |
| 532 | if m.contains("local") || m.contains("whisper.cpp") { |
| 533 | return ("local-whisper".into(), "tiny".into()); |
| 534 | } |
| 535 | if m.contains("mimo") || m.contains("xiaomi") { |
| 536 | return ("provider".into(), ASR_MODEL.into()); |
| 537 | } |
| 538 | } |
| 539 | // Auto-detect best free: local whisper (offline, no key) > Groq free tier > Xiaomi ASR (needs key) |
| 540 | let free = detect_free_asr(); |
| 541 | match free { |
| 542 | "local-whisper" => ("local-whisper".into(), "tiny".into()), |
| 543 | "groq" => ("groq".into(), GROQ_ASR_MODEL.into()), |
| 544 | _ => ("provider".into(), ASR_MODEL.into()), |
| 545 | } |
| 546 | } |
| 547 | |
| 548 | /// Available ASR providers (for `voice --list` / settings UI). |
| 549 | #[allow(dead_code)] |
| 550 | pub fn available_asr_providers() -> Vec<(&'static str, &'static str, &'static str)> { |
| 551 | vec![ |
| 552 | ( |
| 553 | "local-whisper", |
| 554 | "Local Whisper.cpp", |
| 555 | "free, offline, mac/win/linux/harmony — brew install whisper-cpp", |
| 556 | ), |
| 557 | ( |
| 558 | "groq", |
| 559 | "Groq Whisper large-v3-turbo", |
| 560 | "free tier, fast, needs GROQ_API_KEY", |
| 561 | ), |
| 562 | ( |
| 563 | "xiaomi", |
| 564 | "Xiaomi MiMo ASR (mimo-v2.5-asr)", |
| 565 | "needs XIAOMI_API_KEY, streaming, high quality", |
| 566 | ), |
| 567 | ( |
| 568 | "openai", |
| 569 | "OpenAI whisper-1", |
| 570 | "needs OPENAI_API_KEY, compatible", |
| 571 | ), |
| 572 | ] |
| 573 | } |
| 574 | |
| 575 | pub async fn capture_and_transcribe( |
| 576 | app: &mut App, |
| 577 | config: &Config, |
| 578 | ) -> Result<VoiceCaptureOutcome, String> { |
| 579 | let locale = app.ui_locale; |
| 580 | |
| 581 | if !is_available() { |
| 582 | return Err(tr(locale, MessageId::VoiceErrNoRecorder).to_string()); |
| 583 | } |
| 584 | let api_key = config |
| 585 | .deepseek_api_key() |
| 586 | .map_err(|_| tr(locale, MessageId::VoiceErrNoAuth).to_string())?; |
| 587 | let base_url = config.deepseek_base_url(); |
| 588 | |
| 589 | // Spark-style: show "● Recording (⌥V to finish)" + live interim in composer. |
| 590 | let original_input = app.composer.input.clone(); |
| 591 | let original_cursor = app.composer.cursor_position; |
| 592 | app.status_message = Some("● Recording (⌥V to finish) · speak naturally".to_string()); |
| 593 | |
| 594 | // Streaming interim: poll every 700ms and show partial transcript like Grok Build's |
| 595 | // VoiceEvent::Interim → VoiceState::Recording{interim}. We re-transcribe the |
| 596 | // growing buffer (local-whisper is cheap; Groq is ~300ms; provider falls back). |
| 597 | let (asr_kind, _asr_model) = resolve_asr_choice(config); |
| 598 | let interim_enabled = true; // always show partials — feels alive like Spark |
| 599 | |
| 600 | // Spawn recorder on blocking thread with a shared buffer for interim polling. |
| 601 | let shared_buf: std::sync::Arc<parking_lot::Mutex<Vec<i16>>> = |
| 602 | std::sync::Arc::new(parking_lot::Mutex::new(Vec::new())); |
| 603 | let shared_done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); |
| 604 | let shared_buf_clone = std::sync::Arc::clone(&shared_buf); |
| 605 | let shared_done_clone = std::sync::Arc::clone(&shared_done); |
| 606 | let recorder_handle = tokio::task::spawn_blocking(move || { |
| 607 | // Bridge to existing record_audio but copy into shared buffer incrementally. |
| 608 | // For now we reuse the blocking recorder and then publish; interim will |
| 609 | // poll the final buffer. A true streaming recorder (cpal/pw-record) is |
| 610 | // the next step — see grokbuild's xai-grok-voice::audio for the subprocess |
| 611 | // isolation pattern we should mirror. |
| 612 | let result = record_audio(); |
| 613 | if let Some((samples, dur)) = result { |
| 614 | *shared_buf_clone.lock() = samples.clone(); |
| 615 | shared_done_clone.store(true, std::sync::atomic::Ordering::SeqCst); |
| 616 | Some((samples, dur)) |
| 617 | } else { |
| 618 | shared_done_clone.store(true, std::sync::atomic::Ordering::SeqCst); |
| 619 | None |
| 620 | } |
| 621 | }); |
| 622 | |
| 623 | // Interim polling loop — updates composer with "original + interim ▍" so text |
| 624 | // appears as you talk, just like Spark's live transcript. |
| 625 | let mut last_interim = String::new(); |
| 626 | let mut ticks: u32 = 0; |
| 627 | loop { |
| 628 | tokio::time::sleep(Duration::from_millis(700)).await; |
| 629 | ticks += 1; |
| 630 | if shared_done.load(std::sync::atomic::Ordering::SeqCst) { |
| 631 | break; |
| 632 | } |
| 633 | if !interim_enabled || ticks < 2 { |
| 634 | continue; // let a little audio accumulate before first interim |
| 635 | } |
| 636 | let snapshot = { shared_buf.lock().clone() }; |
| 637 | if snapshot.len() < 8000 { |
| 638 | // <0.5s of audio — not enough for meaningful ASR |
| 639 | continue; |
| 640 | } |
| 641 | // Try cheapest free ASR for interim; don't fail the whole capture on interim error. |
| 642 | let interim = match asr_kind.as_str() { |
| 643 | "local-whisper" => transcribe_local_whisper(&snapshot) |
| 644 | .await |
| 645 | .unwrap_or_default(), |
| 646 | "groq" => transcribe_groq(&snapshot).await.unwrap_or_default(), |
| 647 | _ => { |
| 648 | // For provider ASR, reuse the same endpoint but don't block on interim if no key. |
| 649 | if let Ok(key) = config |
| 650 | .deepseek_api_key() |
| 651 | .map(|k: String| k) |
| 652 | .map_err(|_| String::new()) |
| 653 | { |
| 654 | let url = config.deepseek_base_url(); |
| 655 | transcribe(&key, &url, &snapshot).await.unwrap_or_default() |
| 656 | } else { |
| 657 | String::new() |
| 658 | } |
| 659 | } |
| 660 | }; |
| 661 | let trimmed = interim.trim(); |
| 662 | if !trimmed.is_empty() && trimmed != last_interim { |
| 663 | last_interim = trimmed.to_string(); |
| 664 | // Show interim inline — preserve cursor at original position, append interim with a block cursor |
| 665 | let display = if original_input.trim().is_empty() { |
| 666 | format!("{trimmed} ▍") |
| 667 | } else { |
| 668 | format!("{} {} ▍", original_input.trim_end(), trimmed) |
| 669 | }; |
| 670 | app.composer.input = display; |
| 671 | app.composer.cursor_position = original_cursor; |
| 672 | // Also keep status as Spark does |
| 673 | app.status_message = Some(format!("● Listening — “{trimmed}” (⌥V to finish)")); |
| 674 | } |
| 675 | if ticks > 40 { |
| 676 | break; // safety: ~28s max interim polling |
| 677 | } |
| 678 | } |
| 679 | |
| 680 | let (samples, _duration) = recorder_handle |
| 681 | .await |
| 682 | .ok() |
| 683 | .flatten() |
| 684 | .ok_or_else(|| tr(locale, MessageId::VoiceErrTooShort).to_string())?; |
| 685 | |
| 686 | // Restore composer to original before final insert (interim was preview only) |
| 687 | app.composer.input = original_input.clone(); |
| 688 | app.composer.cursor_position = original_cursor; |
| 689 | app.status_message = Some(tr(locale, MessageId::VoiceProcessing).to_string()); |
| 690 | |
| 691 | let text = match asr_kind.as_str() { |
| 692 | "local-whisper" => match transcribe_local_whisper(&samples).await { |
| 693 | Ok(v) => Ok(v), |
| 694 | Err(_) => transcribe(&api_key, &base_url, &samples).await, |
| 695 | }, |
| 696 | "groq" => match transcribe_groq(&samples).await { |
| 697 | Ok(v) => Ok(v), |
| 698 | Err(_) => transcribe(&api_key, &base_url, &samples).await, |
| 699 | }, |
| 700 | _ => { |
| 701 | if app.voice_control_enabled { |
| 702 | process_voice_control(&api_key, &base_url, &samples, &original_input).await |
| 703 | } else { |
| 704 | transcribe(&api_key, &base_url, &samples).await |
| 705 | } |
| 706 | } |
| 707 | } |
| 708 | .map_err(|e| format!("{}: {e}", tr(locale, MessageId::VoiceErrNetwork)))?; |
| 709 | |
| 710 | let clean = text.trim(); |
| 711 | if app.voice_send_enabled { |
| 712 | let (remainder, wants_send) = split_send_suffix(clean); |
| 713 | if wants_send { |
| 714 | // A bare "send it" submits whatever is already in the composer. |
| 715 | let outgoing = if remainder.is_empty() { |
| 716 | let existing = app.composer.input.trim().to_string(); |
| 717 | if !existing.is_empty() { |
| 718 | app.clear_input(); |
| 719 | } |
| 720 | existing |
| 721 | } else { |
| 722 | remainder.to_string() |
| 723 | }; |
| 724 | if outgoing.is_empty() { |
| 725 | return Err(tr(locale, MessageId::VoiceErrEmptySend).to_string()); |
| 726 | } |
| 727 | return Ok(VoiceCaptureOutcome::Send(outgoing)); |
| 728 | } |
| 729 | } |
| 730 | if clean.is_empty() { |
| 731 | return Err(tr(locale, MessageId::VoiceErrEmptySend).to_string()); |
| 732 | } |
| 733 | Ok(VoiceCaptureOutcome::Insert(clean.to_string())) |
| 734 | } |
| 735 | |
| 736 | // --- Command handlers ------------------------------------------------------ |
| 737 | |
| 738 | /// Handle the `/voice` command: toggle voice input. Toggling on requests a |
| 739 | /// one-shot recording + transcription via [`AppAction::VoiceCapture`]. |
| 740 | pub fn voice(app: &mut App) -> CommandResult { |
| 741 | let locale = app.ui_locale; |
| 742 | |
| 743 | if app.voice_enabled { |
| 744 | app.voice_enabled = false; |
| 745 | return CommandResult::message(tr(locale, MessageId::VoiceDisabled)); |
| 746 | } |
| 747 | if !is_available() { |
| 748 | return CommandResult::error(tr(locale, MessageId::VoiceErrNoRecorder)); |
| 749 | } |
| 750 | app.voice_enabled = true; |
| 751 | CommandResult::with_message_and_action( |
| 752 | tr(locale, MessageId::VoiceEnabled), |
| 753 | AppAction::VoiceCapture, |
| 754 | ) |
| 755 | } |
| 756 | |
| 757 | /// Handle the `/voice-send` command: toggle auto-send after transcription. |
| 758 | pub fn voice_send(app: &mut App) -> CommandResult { |
| 759 | let locale = app.ui_locale; |
| 760 | app.voice_send_enabled = !app.voice_send_enabled; |
| 761 | |
| 762 | let msg = if app.voice_send_enabled { |
| 763 | tr(locale, MessageId::VoiceSendEnabled) |
| 764 | } else { |
| 765 | tr(locale, MessageId::VoiceSendDisabled) |
| 766 | }; |
| 767 | CommandResult::message(msg) |
| 768 | } |
| 769 | |
| 770 | /// Handle the `/voice-control` command: toggle AI-assisted dictation. |
| 771 | pub fn voice_control(app: &mut App) -> CommandResult { |
| 772 | let locale = app.ui_locale; |
| 773 | app.voice_control_enabled = !app.voice_control_enabled; |
| 774 | |
| 775 | let msg = if app.voice_control_enabled { |
| 776 | tr(locale, MessageId::VoiceControlEnabled) |
| 777 | } else { |
| 778 | tr(locale, MessageId::VoiceControlDisabled) |
| 779 | }; |
| 780 | CommandResult::message(msg) |
| 781 | } |
| 782 | |
| 783 | #[cfg(test)] |
| 784 | mod tests { |
| 785 | use super::*; |
| 786 | |
| 787 | #[test] |
| 788 | fn wav_encoding_produces_valid_header() { |
| 789 | let samples = vec![0i16; 16000]; // 1 second of silence |
| 790 | let wav = encode_wav(&samples); |
| 791 | assert_eq!(&wav[0..4], b"RIFF"); |
| 792 | assert_eq!(&wav[8..12], b"WAVE"); |
| 793 | assert_eq!(&wav[12..16], b"fmt "); |
| 794 | // data size = 16000 * 2 = 32000 |
| 795 | assert_eq!(&wav[4..8], &(36 + 32000u32).to_le_bytes()); |
| 796 | } |
| 797 | |
| 798 | #[test] |
| 799 | fn wav_encoding_empty_is_minimal() { |
| 800 | let wav = encode_wav(&[]); |
| 801 | assert_eq!(wav.len(), 44); |
| 802 | assert_eq!(&wav[4..8], &36u32.to_le_bytes()); |
| 803 | } |
| 804 | |
| 805 | #[test] |
| 806 | fn send_suffix_detected_and_stripped() { |
| 807 | assert_eq!(split_send_suffix("send it"), ("", true)); |
| 808 | assert_eq!(split_send_suffix("Send It!"), ("", true)); |
| 809 | assert_eq!(split_send_suffix("发送"), ("", true)); |
| 810 | assert_eq!(split_send_suffix("發送。"), ("", true)); |
| 811 | assert_eq!( |
| 812 | split_send_suffix("ship the fix, send it"), |
| 813 | ("ship the fix", true) |
| 814 | ); |
| 815 | assert_eq!( |
| 816 | split_send_suffix("修复这个问题，发送"), |
| 817 | ("修复这个问题", true) |
| 818 | ); |
| 819 | } |
| 820 | |
| 821 | #[test] |
| 822 | fn send_suffix_leaves_plain_text_alone() { |
| 823 | assert_eq!(split_send_suffix("send it now"), ("send it now", false)); |
| 824 | assert_eq!( |
| 825 | split_send_suffix("帮我发送一封邮件"), |
| 826 | ("帮我发送一封邮件", false) |
| 827 | ); |
| 828 | assert_eq!(split_send_suffix("发送邮件"), ("发送邮件", false)); |
| 829 | assert_eq!( |
| 830 | split_send_suffix("resend it to the queue"), |
| 831 | ("resend it to the queue", false) |
| 832 | ); |
| 833 | } |
| 834 | |
| 835 | #[test] |
| 836 | fn recorder_detection_does_not_crash() { |
| 837 | // Just verify the function runs without panicking |
| 838 | let _ = is_available(); |
| 839 | } |
| 840 | } |
| 841 |