返回 CodeWhale
voice.rs
根目录 / crates / tui / src / commands / groups / core / voice.rs
1 //! Voice input commands — `/voice`, `/voice-send`, `/voice-control`.
2 //!
3 //! Records audio from the default microphone, sends it to the configured
4 //! provider's API for transcription, and inserts the transcribed text into
5 //! the composer. The interaction model mirrors MiMo Code's voice UX:
6 //!
7 //! `/voice` — toggle voice input on/off (records when toggled on)
8 //! `/voice-send` — toggle auto-send when the transcript ends with
9 //! "send it" / "发送"
10 //! `/voice-control` — toggle AI-assisted dictation that sees the current
11 //! composer text
12 //!
13 //! The slash commands only flip state and emit [`AppAction::VoiceCapture`];
14 //! the actual capture runs in the UI event loop where the live [`Config`]
15 //! supplies provider credentials. That keeps the handlers side-effect free
16 //! (the registry smoke tests execute every command) and avoids caching
17 //! auth material on [`App`].
18 //!
19 //! ## Recording
20 //!
21 //! Uses platform-specific command-line tools (sox, rec, arecord) to capture
22 //! 16kHz mono 16-bit PCM audio. Records until a silence gap is detected or
23 //! the maximum duration is reached (default 10 s).
24
25 use std::process::{Command, Stdio};
26 use std::sync::LazyLock;
27 use std::time::Duration;
28
29 use regex::Regex;
30
31 use crate::commands::CommandResult;
32 use crate::commands::traits::{CommandInfo, RegisterCommand};
33 use crate::config::Config;
34 use crate::tui::app::{App, AppAction};
35 use codewhale_localization::{MessageId, tr};
36
37 /// Transcription model requested from the provider's chat-completions API.
38 const ASR_MODEL: &str = "mimo-v2.5-asr";
39 const GROQ_ASR_MODEL: &str = "whisper-large-v3-turbo";
40 /// Local whisper binary names to probe (whisper.cpp, faster-whisper, OpenAI whisper).
41 const LOCAL_WHISPER_BINS: &[&str] = &["whisper", "whisper.cpp", "whisper-cpp", "faster-whisper"];
42 /// Model used for the AI-assisted voice-control pipeline.
43 const VOICE_CONTROL_MODEL: &str = "mimo-v2.5";
44
45 pub(in crate::commands) const VOICE_INFO: CommandInfo = CommandInfo {
46 name: "voice",
47 aliases: &["yuyin", "语音"],
48 usage: "/voice",
49 description_id: MessageId::CmdVoiceDescription,
50 };
51
52 pub(in crate::commands) const VOICE_SEND_INFO: CommandInfo = CommandInfo {
53 name: "voicesend",
54 aliases: &["voice-send", "yuyinsend", "语音发送"],
55 usage: "/voicesend",
56 description_id: MessageId::CmdVoiceSendDescription,
57 };
58
59 pub(in crate::commands) const VOICE_CONTROL_INFO: CommandInfo = CommandInfo {
60 name: "voicecontrol",
61 aliases: &["voice-control", "yuyincontrol", "语音控制"],
62 usage: "/voicecontrol",
63 description_id: MessageId::CmdVoiceControlDescription,
64 };
65
66 pub(in crate::commands) struct VoiceCmd;
67 pub(in crate::commands) struct VoiceSendCmd;
68 pub(in crate::commands) struct VoiceControlCmd;
69
70 impl RegisterCommand for VoiceCmd {
71 fn info() -> &'static CommandInfo {
72 &VOICE_INFO
73 }
74
75 fn execute(app: &mut App, _arg: Option<&str>) -> CommandResult {
76 voice(app)
77 }
78 }
79
80 impl RegisterCommand for VoiceSendCmd {
81 fn info() -> &'static CommandInfo {
82 &VOICE_SEND_INFO
83 }
84
85 fn execute(app: &mut App, _arg: Option<&str>) -> CommandResult {
86 voice_send(app)
87 }
88 }
89
90 impl RegisterCommand for VoiceControlCmd {
91 fn info() -> &'static CommandInfo {
92 &VOICE_CONTROL_INFO
93 }
94
95 fn execute(app: &mut App, _arg: Option<&str>) -> CommandResult {
96 voice_control(app)
97 }
98 }
99
100 // --- Recorder detection ----------------------------------------------------
101
102 /// Platform-specific recorder definitions.
103 #[derive(Debug, Clone)]
104 struct Recorder {
105 cmd: &'static str,
106 /// CLI arguments for piping raw 16kHz mono S16_LE PCM to stdout.
107 pipe_args: &'static [&'static str],
108 }
109
110 fn detect_recorder() -> Option<Recorder> {
111 // Operator kill-switch: a headless `serve --http` host has no business
112 // opening a microphone; disabling voice here makes `GET /v1/voice`
113 // report `available: false` and every dictate call fail closed.
114 if std::env::var_os("CODEWHALE_DISABLE_VOICE").is_some() {
115 return None;
116 }
117 let candidates: &[Recorder] = if cfg!(target_os = "macos") {
118 &[
119 Recorder {
120 cmd: "sox",
121 pipe_args: &["-d", "-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"],
122 },
123 Recorder {
124 cmd: "rec",
125 pipe_args: &["-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"],
126 },
127 ]
128 } else if cfg!(target_os = "linux") {
129 &[
130 Recorder {
131 cmd: "arecord",
132 pipe_args: &["-f", "S16_LE", "-r", "16000", "-c", "1", "-t", "raw"],
133 },
134 Recorder {
135 cmd: "sox",
136 pipe_args: &["-d", "-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"],
137 },
138 ]
139 } else if cfg!(target_os = "windows") {
140 &[Recorder {
141 cmd: "sox",
142 pipe_args: &["-d", "-r", "16000", "-c", "1", "-b", "16", "-t", "raw", "-"],
143 }]
144 } else {
145 &[]
146 };
147
148 candidates
149 .iter()
150 .find(|r| {
151 Command::new(r.cmd)
152 .arg("--version")
153 .stdin(Stdio::null())
154 .stdout(Stdio::null())
155 .stderr(Stdio::null())
156 .spawn()
157 .is_ok()
158 })
159 .cloned()
160 }
161
162 /// Check whether voice recording is available on this system.
163 pub fn is_available() -> bool {
164 detect_recorder().is_some()
165 }
166
167 // --- WAV encoding ----------------------------------------------------------
168
169 /// Encode raw 16kHz mono S16_LE PCM samples as a WAV buffer.
170 fn encode_wav(samples: &[i16]) -> Vec<u8> {
171 let data_size = (samples.len() * 2) as u32;
172 let sample_rate: u32 = 16000;
173 let mut buf = Vec::with_capacity(44 + data_size as usize);
174
175 // RIFF header
176 buf.extend_from_slice(b"RIFF");
177 buf.extend_from_slice(&(36 + data_size).to_le_bytes());
178 buf.extend_from_slice(b"WAVE");
179
180 // fmt chunk
181 buf.extend_from_slice(b"fmt ");
182 buf.extend_from_slice(&16u32.to_le_bytes()); // chunk size
183 buf.extend_from_slice(&1u16.to_le_bytes()); // PCM
184 buf.extend_from_slice(&1u16.to_le_bytes()); // mono
185 buf.extend_from_slice(&sample_rate.to_le_bytes());
186 buf.extend_from_slice(&(sample_rate * 2).to_le_bytes()); // byte rate
187 buf.extend_from_slice(&2u16.to_le_bytes()); // block align
188 buf.extend_from_slice(&16u16.to_le_bytes()); // bits per sample
189
190 // data chunk
191 buf.extend_from_slice(b"data");
192 buf.extend_from_slice(&data_size.to_le_bytes());
193 for &sample in samples {
194 buf.extend_from_slice(&sample.to_le_bytes());
195 }
196
197 buf
198 }
199
200 // --- Recording -------------------------------------------------------------
201
202 /// Maximum recording duration in seconds before auto-stopping.
203 pub const MAX_RECORD_SECS: u64 = 10;
204 /// Minimum segment duration in seconds to consider as valid speech.
205 const MIN_SEGMENT_SECS: f64 = 0.3;
206
207 /// Record audio from the default microphone.
208 ///
209 /// Returns raw 16kHz mono S16_LE PCM samples. Returns `None` if no recorder
210 /// is available, the recording failed, or no speech was detected.
211 fn record_audio() -> Option<(Vec<i16>, Duration)> {
212 let recorder = detect_recorder()?;
213 let start = std::time::Instant::now();
214
215 let mut child = Command::new(recorder.cmd)
216 .args(recorder.pipe_args)
217 .stdin(Stdio::null())
218 .stdout(Stdio::piped())
219 .stderr(Stdio::null())
220 .spawn()
221 .ok()?;
222
223 let stdout = child.stdout.take()?;
224 let mut reader = std::io::BufReader::new(stdout);
225 let mut all_samples: Vec<i16> = Vec::with_capacity(16000 * MAX_RECORD_SECS as usize);
226
227 // Read until timeout or silence
228 let mut buf = [0u8; 320]; // 10ms of 16kHz S16_LE
229 let max_duration = Duration::from_secs(MAX_RECORD_SECS);
230 let mut silence_samples = 0u32;
231 let mut had_speech = false;
232 let speech_threshold: i16 = 500; // RMS-based speech detection threshold
233 let silence_duration_samples = 16000u32; // 1 second of silence to stop
234
235 loop {
236 use std::io::Read;
237 match reader.read_exact(&mut buf) {
238 Ok(()) => {
239 let chunk: Vec<i16> = buf
240 .as_chunks::<2>()
241 .0
242 .iter()
243 .copied()
244 .map(i16::from_le_bytes)
245 .collect();
246
247 // Simple RMS-based VAD
248 let rms = (chunk.iter().map(|&s| (s as f64) * (s as f64)).sum::<f64>()
249 / chunk.len() as f64)
250 .sqrt();
251 let is_speech = rms > speech_threshold as f64;
252
253 if is_speech {
254 had_speech = true;
255 silence_samples = 0;
256 } else if had_speech {
257 silence_samples += chunk.len() as u32;
258 }
259
260 if had_speech {
261 all_samples.extend_from_slice(&chunk);
262 }
263
264 if start.elapsed() > max_duration {
265 let _ = child.kill();
266 break;
267 }
268 if had_speech && silence_samples >= silence_duration_samples {
269 let _ = child.kill();
270 break;
271 }
272 }
273 Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
274 Err(_) => {
275 let _ = child.kill();
276 break;
277 }
278 }
279 }
280
281 let _ = child.wait();
282 let elapsed = start.elapsed();
283
284 let min_samples = (MIN_SEGMENT_SECS * 16000.0) as usize;
285 if all_samples.len() < min_samples {
286 return None;
287 }
288
289 Some((all_samples, elapsed))
290 }
291
292 // --- Auto-send suffix ------------------------------------------------------
293
294 /// Trailing phrases that mean "submit this" — the human-readable form of
295 /// `SEND_SUFFIX_RE`; keep in sync with the regex when either changes.
296 pub const SEND_PHRASES: &[&str] = &["send it", "发送", "發送"];
297
298 /// Matches an explicit send instruction at the end of transcribed text:
299 /// "send it" (any spacing/case) or 发送/發送, with trailing punctuation.
300 static SEND_SUFFIX_RE: LazyLock<Regex> = LazyLock::new(|| {
301 Regex::new(r"(?i)(?:^|[\s,,.。!!??]+)(?:send\s*it|发送|發送)[\s.。!!??]*$").unwrap()
302 });
303
304 /// Split a transcript into the message remainder and whether it ended with an
305 /// explicit send instruction. `"ship the fix, send it"` → `("ship the fix", true)`.
306 fn split_send_suffix(text: &str) -> (&str, bool) {
307 match SEND_SUFFIX_RE.find(text) {
308 Some(found) => (text[..found.start()].trim(), true),
309 None => (text.trim(), false),
310 }
311 }
312
313 // --- Transcription ---------------------------------------------------------
314
315 fn base64_encode(data: &[u8]) -> String {
316 use base64::Engine;
317 base64::engine::general_purpose::STANDARD.encode(data)
318 }
319
320 fn chat_completions_url(base_url: &str) -> String {
321 format!("{}/chat/completions", base_url.trim_end_matches('/'))
322 }
323
324 async fn post_chat_completions(
325 api_key: &str,
326 base_url: &str,
327 mut body: serde_json::Value,
328 openrouter_vendor: Option<&str>,
329 ) -> Result<serde_json::Value, String> {
330 crate::client::apply_openrouter_vendor(&mut body, openrouter_vendor);
331 let _inference = crate::client::acquire_remote_control_inference_participant().await;
332 let client = crate::tls::reqwest_client();
333 let resp = client
334 .post(chat_completions_url(base_url))
335 .header("Content-Type", "application/json")
336 .header("Authorization", format!("Bearer {api_key}"))
337 .timeout(Duration::from_secs(30))
338 .json(&body)
339 .send()
340 .await
341 .map_err(|e| format!("request failed: {e}"))?;
342
343 if !resp.status().is_success() {
344 return Err(format!("API returned status {}", resp.status()));
345 }
346
347 resp.json()
348 .await
349 .map_err(|e| format!("failed to parse response: {e}"))
350 }
351
352 /// Send audio to the provider's API for plain transcription.
353 ///
354 /// Uses the chat completions endpoint with `input_audio` content blocks.
355 async fn transcribe(
356 api_key: &str,
357 base_url: &str,
358 audio_samples: &[i16],
359 openrouter_vendor: Option<&str>,
360 ) -> Result<String, String> {
361 transcribe_with_model(
362 api_key,
363 base_url,
364 audio_samples,
365 ASR_MODEL,
366 openrouter_vendor,
367 )
368 .await
369 }
370
371 async fn transcribe_with_model(
372 api_key: &str,
373 base_url: &str,
374 audio_samples: &[i16],
375 model: &str,
376 openrouter_vendor: Option<&str>,
377 ) -> Result<String, String> {
378 let wav = encode_wav(audio_samples);
379 let data_url = format!("data:audio/wav;base64,{}", base64_encode(&wav));
380
381 let body = serde_json::json!({
382 "model": model,
383 "messages": [
384 {
385 "role": "user",
386 "content": [
387 {
388 "type": "input_audio",
389 "input_audio": {
390 "data": data_url
391 }
392 }
393 ]
394 }
395 ],
396 "asr_options": {
397 "language": "auto"
398 }
399 });
400
401 let data = post_chat_completions(api_key, base_url, body, openrouter_vendor).await?;
402 data["choices"][0]["message"]["content"]
403 .as_str()
404 .map(|s| s.trim().to_string())
405 .ok_or_else(|| "no transcription in response".to_string())
406 }
407
408 /// Process audio through the voice-control pipeline: AI-assisted dictation
409 /// that sees the current composer text, mirroring MiMo Code's
410 /// `processVoiceControl`. Used when `/voice-control` is enabled.
411 async fn process_voice_control(
412 api_key: &str,
413 base_url: &str,
414 audio_samples: &[i16],
415 current_text: &str,
416 openrouter_vendor: Option<&str>,
417 ) -> Result<String, String> {
418 let wav = encode_wav(audio_samples);
419 let data_url = format!("data:audio/wav;base64,{}", base64_encode(&wav));
420
421 let user_context = serde_json::json!({
422 "current_text": current_text,
423 "cursor": "end",
424 });
425
426 let body = serde_json::json!({
427 "model": VOICE_CONTROL_MODEL,
428 "messages": [
429 {
430 "role": "system",
431 "content": "You are a voice input assistant. Transcribe the user's speech. Output JSON: {\"text\": \"transcribed text\"}."
432 },
433 {
434 "role": "user",
435 "content": [
436 { "type": "text", "text": user_context.to_string() },
437 { "type": "input_audio", "input_audio": { "data": data_url } }
438 ]
439 }
440 ],
441 "response_format": { "type": "json_object" }
442 });
443
444 let data = post_chat_completions(api_key, base_url, body, openrouter_vendor).await?;
445 let content = data["choices"][0]["message"]["content"]
446 .as_str()
447 .ok_or_else(|| "no response content".to_string())?;
448
449 let parsed: serde_json::Value = serde_json::from_str(content)
450 .map_err(|e| format!("failed to parse voice control JSON: {e}"))?;
451
452 parsed["text"]
453 .as_str()
454 .map(|s| s.to_string())
455 .ok_or_else(|| "no text field in voice control response".to_string())
456 }
457
458 // --- Capture orchestration (UI event loop) ---------------------------------
459
460 /// What the UI should do with a finished capture.
461 #[derive(Debug, Clone, PartialEq, Eq)]
462 pub enum VoiceCaptureOutcome {
463 /// Insert the transcribed text into the composer at the cursor.
464 Insert(String),
465 /// Submit this text as a message (auto-send).
466 Send(String),
467 }
468
469 /// Detect best free ASR for this host — local whisper > Groq free > provider fallback.
470 /// Works on macOS (brew install whisper-cpp), Windows (whisper.cpp binary),
471 /// Linux (apt), and HarmonyOS (falls back to cloud).
472 fn detect_free_asr() -> &'static str {
473 for bin in LOCAL_WHISPER_BINS {
474 if Command::new(bin)
475 .arg("--help")
476 .stdin(Stdio::null())
477 .stdout(Stdio::null())
478 .stderr(Stdio::null())
479 .spawn()
480 .is_ok()
481 {
482 return "local-whisper";
483 }
484 }
485 if std::env::var("GROQ_API_KEY").is_ok_and(|v| !v.trim().is_empty()) {
486 return "groq";
487 }
488 "provider"
489 }
490
491 /// Transcribe via local whisper.cpp (free, offline, cross-platform).
492 ///
493 /// The whole body is synchronous — temp-file I/O plus `Command::output()`,
494 /// which blocks for the entire subprocess run — so it runs on the blocking
495 /// pool rather than a Tokio worker (blocking-call convention, #6149).
496 async fn transcribe_local_whisper(audio_samples: &[i16]) -> Result<String, String> {
497 let wav = encode_wav(audio_samples);
498 tokio::task::spawn_blocking(move || transcribe_local_whisper_blocking(&wav))
499 .await
500 .map_err(|e| e.to_string())?
501 }
502
503 fn transcribe_local_whisper_blocking(wav: &[u8]) -> Result<String, String> {
504 let tmp = std::env::temp_dir().join(format!("cw-voice-{}.wav", std::process::id()));
505 std::fs::write(&tmp, wav).map_err(|e| e.to_string())?;
506 // Try each local binary until one succeeds; whisper.cpp outputs to stdout or file.
507 for bin in LOCAL_WHISPER_BINS {
508 let output = Command::new(bin)
509 .arg(tmp.to_string_lossy().as_ref())
510 .arg("--model")
511 .arg("tiny")
512 .arg("--language")
513 .arg("auto")
514 .arg("--output-txt")
515 .output();
516 if let Ok(out) = output
517 && out.status.success()
518 {
519 let txt = String::from_utf8_lossy(&out.stdout).trim().to_string();
520 let _ = std::fs::remove_file(&tmp);
521 if !txt.is_empty() {
522 return Ok(txt);
523 }
524 // Some builds write to .txt sidecar
525 let sidecar = tmp.with_extension("txt");
526 if let Ok(s) = std::fs::read_to_string(&sidecar) {
527 let _ = std::fs::remove_file(&sidecar);
528 let _ = std::fs::remove_file(&tmp);
529 if !s.trim().is_empty() {
530 return Ok(s.trim().to_string());
531 }
532 }
533 }
534 }
535 let _ = std::fs::remove_file(&tmp);
536 Err("local whisper not available".into())
537 }
538
539 /// Transcribe via Groq Whisper large-v3-turbo (free tier, ~$0.04/hr, fast).
540 /// Groq is NOT a full CodeWhale provider yet — this is a direct ASR call
541 /// using `GROQ_API_KEY` only (no provider setup needed). Uses the same
542 /// chat-completions `input_audio` path as Xiaomi so no `multipart` feature.
543 async fn transcribe_groq(audio_samples: &[i16]) -> Result<String, String> {
544 let api_key = std::env::var("GROQ_API_KEY").map_err(|_| "GROQ_API_KEY not set".to_string())?;
545 let base_url = "https://api.groq.com/openai/v1";
546 transcribe_with_model(&api_key, base_url, audio_samples, GROQ_ASR_MODEL, None).await
547 }
548
549 /// Perform a complete record + transcribe cycle with live interim display.
550 ///
551 /// Runs in the UI event loop (see [`AppAction::VoiceCapture`]) so provider
552 /// credentials come from the live [`Config`] rather than state cached on
553 /// [`App`]. Recording happens on a blocking thread; transcription uses the
554 /// shared async HTTP client. Every failure path returns a localized message
555 /// so callers can surface it as a status line.
556 /// Resolve ASR model/provider preference.
557 /// Priority: explicit config `voice.asr_model` > env `CODEWHALE_ASR_MODEL` > auto-detect (local-whisper > groq > xiaomi).
558 fn resolve_asr_choice(_config: &Config) -> (String, String) {
559 // Check explicit env override first (free, cross-platform)
560 if let Ok(m) = std::env::var("CODEWHALE_ASR_MODEL") {
561 let m = m.trim().to_ascii_lowercase();
562 if m.contains("groq") || m.contains("whisper") {
563 return ("groq".into(), GROQ_ASR_MODEL.into());
564 }
565 if m.contains("local") || m.contains("whisper.cpp") {
566 return ("local-whisper".into(), "tiny".into());
567 }
568 if m.contains("mimo") || m.contains("xiaomi") {
569 return ("provider".into(), ASR_MODEL.into());
570 }
571 }
572 // Auto-detect best free: local whisper (offline, no key) > Groq free tier > Xiaomi ASR (needs key)
573 let free = detect_free_asr();
574 match free {
575 "local-whisper" => ("local-whisper".into(), "tiny".into()),
576 "groq" => ("groq".into(), GROQ_ASR_MODEL.into()),
577 _ => ("provider".into(), ASR_MODEL.into()),
578 }
579 }
580
581 pub async fn capture_and_transcribe(
582 app: &mut App,
583 config: &Config,
584 ) -> Result<VoiceCaptureOutcome, String> {
585 let locale = app.ui_locale;
586
587 if !is_available() {
588 return Err(tr(locale, MessageId::VoiceErrNoRecorder).to_string());
589 }
590 let api_key = config
591 .active_route_api_key()
592 .map_err(|_| tr(locale, MessageId::VoiceErrNoAuth).to_string())?;
593 let base_url = config.active_route_base_url();
594 let openrouter_vendor = config
595 .openrouter_vendor()
596 .map_err(|error| error.to_string())?;
597
598 // Spark-style: show "● Recording (⌥V to finish)" + live interim in composer.
599 let original_input = app.composer.input.clone();
600 let original_cursor = app.composer.cursor_position;
601 app.status_message = Some("● Recording (⌥V to finish) · speak naturally".to_string());
602
603 // Streaming interim: poll every 700ms and show partial transcript like Grok Build's
604 // VoiceEvent::Interim → VoiceState::Recording{interim}. We re-transcribe the
605 // growing buffer (local-whisper is cheap; Groq is ~300ms; provider falls back).
606 let (asr_kind, _asr_model) = resolve_asr_choice(config);
607 let interim_enabled = true; // always show partials — feels alive like Spark
608
609 // Spawn recorder on blocking thread with a shared buffer for interim polling.
610 let shared_buf: std::sync::Arc<parking_lot::Mutex<Vec<i16>>> =
611 std::sync::Arc::new(parking_lot::Mutex::new(Vec::new()));
612 let shared_done = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false));
613 let shared_buf_clone = std::sync::Arc::clone(&shared_buf);
614 let shared_done_clone = std::sync::Arc::clone(&shared_done);
615 let recorder_handle = tokio::task::spawn_blocking(move || {
616 // Bridge to existing record_audio but copy into shared buffer incrementally.
617 // For now we reuse the blocking recorder and then publish; interim will
618 // poll the final buffer. A true streaming recorder (cpal/pw-record) is
619 // the next step — see grokbuild's xai-grok-voice::audio for the subprocess
620 // isolation pattern we should mirror.
621 let result = record_audio();
622 if let Some((samples, dur)) = result {
623 *shared_buf_clone.lock() = samples.clone();
624 shared_done_clone.store(true, std::sync::atomic::Ordering::SeqCst);
625 Some((samples, dur))
626 } else {
627 shared_done_clone.store(true, std::sync::atomic::Ordering::SeqCst);
628 None
629 }
630 });
631
632 // Interim polling loop — updates composer with "original + interim ▍" so text
633 // appears as you talk, just like Spark's live transcript.
634 let mut last_interim = String::new();
635 let mut ticks: u32 = 0;
636 loop {
637 tokio::time::sleep(Duration::from_millis(700)).await;
638 ticks += 1;
639 if shared_done.load(std::sync::atomic::Ordering::SeqCst) {
640 break;
641 }
642 if !interim_enabled || ticks < 2 {
643 continue; // let a little audio accumulate before first interim
644 }
645 let snapshot = { shared_buf.lock().clone() };
646 if snapshot.len() < 8000 {
647 // <0.5s of audio — not enough for meaningful ASR
648 continue;
649 }
650 // Try cheapest free ASR for interim; don't fail the whole capture on interim error.
651 let interim = match asr_kind.as_str() {
652 "local-whisper" => transcribe_local_whisper(&snapshot)
653 .await
654 .unwrap_or_default(),
655 "groq" => transcribe_groq(&snapshot).await.unwrap_or_default(),
656 _ => {
657 // For provider ASR, reuse the same endpoint but don't block on interim if no key.
658 if let Ok(key) = config
659 .active_route_api_key()
660 .map(|k: String| k)
661 .map_err(|_| String::new())
662 {
663 let url = config.active_route_base_url();
664 transcribe(&key, &url, &snapshot, openrouter_vendor.as_deref())
665 .await
666 .unwrap_or_default()
667 } else {
668 String::new()
669 }
670 }
671 };
672 let trimmed = interim.trim();
673 if !trimmed.is_empty() && trimmed != last_interim {
674 last_interim = trimmed.to_string();
675 // Show interim inline — preserve cursor at original position, append interim with a block cursor
676 let display = if original_input.trim().is_empty() {
677 format!("{trimmed} ▍")
678 } else {
679 format!("{} {} ▍", original_input.trim_end(), trimmed)
680 };
681 app.composer.input = display;
682 app.composer.cursor_position = original_cursor;
683 // Also keep status as Spark does
684 app.status_message = Some(format!("● Listening — “{trimmed}” (⌥V to finish)"));
685 }
686 if ticks > 40 {
687 break; // safety: ~28s max interim polling
688 }
689 }
690
691 let (samples, _duration) = recorder_handle
692 .await
693 .ok()
694 .flatten()
695 .ok_or_else(|| tr(locale, MessageId::VoiceErrTooShort).to_string())?;
696
697 // Restore composer to original before final insert (interim was preview only)
698 app.composer.input = original_input.clone();
699 app.composer.cursor_position = original_cursor;
700 app.status_message = Some(tr(locale, MessageId::VoiceProcessing).to_string());
701
702 let text = match asr_kind.as_str() {
703 "local-whisper" => match transcribe_local_whisper(&samples).await {
704 Ok(v) => Ok(v),
705 Err(_) => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await,
706 },
707 "groq" => match transcribe_groq(&samples).await {
708 Ok(v) => Ok(v),
709 Err(_) => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await,
710 },
711 _ => {
712 if app.voice_control_enabled {
713 process_voice_control(
714 &api_key,
715 &base_url,
716 &samples,
717 &original_input,
718 openrouter_vendor.as_deref(),
719 )
720 .await
721 } else {
722 transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref()).await
723 }
724 }
725 }
726 .map_err(|e| format!("{}: {e}", tr(locale, MessageId::VoiceErrNetwork)))?;
727
728 let clean = text.trim();
729 if app.voice_send_enabled {
730 let (remainder, wants_send) = split_send_suffix(clean);
731 if wants_send {
732 // A bare "send it" submits whatever is already in the composer.
733 let outgoing = if remainder.is_empty() {
734 let existing = app.composer.input.trim().to_string();
735 if !existing.is_empty() {
736 app.clear_input();
737 }
738 existing
739 } else {
740 remainder.to_string()
741 };
742 if outgoing.is_empty() {
743 return Err(tr(locale, MessageId::VoiceErrEmptySend).to_string());
744 }
745 return Ok(VoiceCaptureOutcome::Send(outgoing));
746 }
747 }
748 if clean.is_empty() {
749 return Err(tr(locale, MessageId::VoiceErrEmptySend).to_string());
750 }
751 Ok(VoiceCaptureOutcome::Insert(clean.to_string()))
752 }
753
754 // --- Headless capture (HTTP/native-client path) ----------------------------
755
756 /// What a headless dictation should do with the finished transcript.
757 #[derive(Debug, Clone)]
758 pub enum DictateMode {
759 /// Transcribe and return the text for insertion into the composer.
760 Insert,
761 /// Transcribe, then apply the "send it" / 发送 suffix contract. The
762 /// outcome's `send` flag tells the client to submit; a bare send
763 /// instruction yields empty `text` so the client submits its own draft.
764 Send,
765 /// AI-assisted dictation that sees the client's composer text — the
766 /// `/voice-control` pipeline. Only provider ASR can see context; free
767 /// ASR kinds degrade to plain transcription with `assisted: false`.
768 Control(String),
769 }
770
771 /// Machine-readable failure for the headless path so HTTP clients can
772 /// localize by `reason` rather than parsing message text.
773 #[derive(Debug)]
774 pub enum DictateError {
775 /// No supported recorder binary on this host.
776 NoRecorder,
777 /// Recording produced no usable speech segment.
778 NoSpeech,
779 /// The selected/fallback ASR needs a provider key that isn't configured.
780 NoProviderAuth,
781 /// ASR request or transcription failed.
782 Transcription(String),
783 }
784
785 impl DictateError {
786 pub fn reason(&self) -> &'static str {
787 match self {
788 Self::NoRecorder => "no_recorder",
789 Self::NoSpeech => "no_speech",
790 Self::NoProviderAuth => "no_provider_auth",
791 Self::Transcription(_) => "transcription_failed",
792 }
793 }
794 }
795
796 impl std::fmt::Display for DictateError {
797 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
798 match self {
799 Self::NoRecorder => write!(f, "no supported voice recorder on this host"),
800 Self::NoSpeech => write!(f, "no speech detected"),
801 Self::NoProviderAuth => {
802 write!(f, "provider ASR requires a configured API key")
803 }
804 Self::Transcription(e) => write!(f, "{e}"),
805 }
806 }
807 }
808
809 /// Result of one headless record→transcribe cycle.
810 #[derive(Debug)]
811 pub struct DictationOutcome {
812 /// Final transcript (send suffix already stripped for `Send` mode).
813 pub text: String,
814 /// `Send` mode only: the transcript ended with an explicit send phrase.
815 pub send: bool,
816 /// `Control` mode only: the composer context reached the model. False
817 /// when a free ASR kind handled the audio and never saw the context.
818 pub assisted: bool,
819 /// Which ASR backend was selected for this capture.
820 pub asr_kind: String,
821 pub asr_model: String,
822 }
823
824 /// Detected recorder binary name, if any (`"sox"`, `"arecord"`, `"rec"`).
825 pub fn recorder_command() -> Option<&'static str> {
826 detect_recorder().map(|r| r.cmd)
827 }
828
829 /// Resolved ASR selection (`kind`, `model`) for capability reporting.
830 pub fn asr_choice(config: &Config) -> (String, String) {
831 resolve_asr_choice(config)
832 }
833
834 /// One record→transcribe cycle with no UI surface: the HTTP/native-client
835 /// equivalent of [`capture_and_transcribe`]. Recording runs on a blocking
836 /// thread; transcription follows the same ASR dispatch as the TUI —
837 /// explicit `CODEWHALE_ASR_MODEL` > local whisper > Groq > provider —
838 /// but resolves the provider key lazily so free ASR kinds work without
839 /// provider auth.
840 pub async fn dictate_once(
841 config: &Config,
842 mode: DictateMode,
843 ) -> Result<DictationOutcome, DictateError> {
844 if !is_available() {
845 return Err(DictateError::NoRecorder);
846 }
847 let (samples, _duration) = tokio::task::spawn_blocking(record_audio)
848 .await
849 .ok()
850 .flatten()
851 .ok_or(DictateError::NoSpeech)?;
852
853 let (asr_kind, asr_model) = resolve_asr_choice(config);
854 let base_url = config.active_route_base_url();
855 let openrouter_vendor = config
856 .openrouter_vendor()
857 .map_err(|e| DictateError::Transcription(e.to_string()))?;
858 let provider_key = || {
859 config
860 .active_route_api_key()
861 .map_err(|_| DictateError::NoProviderAuth)
862 };
863
864 let mut assisted = false;
865 let text = match asr_kind.as_str() {
866 "local-whisper" => match transcribe_local_whisper(&samples).await {
867 Ok(v) => v,
868 Err(_) => transcribe(
869 &provider_key()?,
870 &base_url,
871 &samples,
872 openrouter_vendor.as_deref(),
873 )
874 .await
875 .map_err(DictateError::Transcription)?,
876 },
877 "groq" => match transcribe_groq(&samples).await {
878 Ok(v) => v,
879 Err(_) => transcribe(
880 &provider_key()?,
881 &base_url,
882 &samples,
883 openrouter_vendor.as_deref(),
884 )
885 .await
886 .map_err(DictateError::Transcription)?,
887 },
888 _ => {
889 let api_key = provider_key()?;
890 match &mode {
891 DictateMode::Control(composer) => {
892 assisted = true;
893 process_voice_control(
894 &api_key,
895 &base_url,
896 &samples,
897 composer,
898 openrouter_vendor.as_deref(),
899 )
900 .await
901 .map_err(DictateError::Transcription)?
902 }
903 _ => transcribe(&api_key, &base_url, &samples, openrouter_vendor.as_deref())
904 .await
905 .map_err(DictateError::Transcription)?,
906 }
907 }
908 };
909
910 let clean = text.trim().to_string();
911 let (text, send) = match mode {
912 DictateMode::Send => {
913 let (remainder, wants_send) = split_send_suffix(&clean);
914 (remainder.to_string(), wants_send)
915 }
916 _ => (clean, false),
917 };
918 Ok(DictationOutcome {
919 text,
920 send,
921 assisted,
922 asr_kind,
923 asr_model,
924 })
925 }
926
927 // --- Command handlers ------------------------------------------------------
928
929 /// Handle the `/voice` command: toggle voice input. Toggling on requests a
930 /// one-shot recording + transcription via [`AppAction::VoiceCapture`].
931 pub fn voice(app: &mut App) -> CommandResult {
932 let locale = app.ui_locale;
933
934 if app.voice_enabled {
935 app.voice_enabled = false;
936 return CommandResult::message(tr(locale, MessageId::VoiceDisabled));
937 }
938 if !is_available() {
939 return CommandResult::error(tr(locale, MessageId::VoiceErrNoRecorder));
940 }
941 app.voice_enabled = true;
942 CommandResult::with_message_and_action(
943 tr(locale, MessageId::VoiceEnabled),
944 AppAction::VoiceCapture,
945 )
946 }
947
948 /// Handle the `/voice-send` command: toggle auto-send after transcription.
949 pub fn voice_send(app: &mut App) -> CommandResult {
950 let locale = app.ui_locale;
951 app.voice_send_enabled = !app.voice_send_enabled;
952
953 let msg = if app.voice_send_enabled {
954 tr(locale, MessageId::VoiceSendEnabled)
955 } else {
956 tr(locale, MessageId::VoiceSendDisabled)
957 };
958 CommandResult::message(msg)
959 }
960
961 /// Handle the `/voice-control` command: toggle AI-assisted dictation.
962 pub fn voice_control(app: &mut App) -> CommandResult {
963 let locale = app.ui_locale;
964 app.voice_control_enabled = !app.voice_control_enabled;
965
966 let msg = if app.voice_control_enabled {
967 tr(locale, MessageId::VoiceControlEnabled)
968 } else {
969 tr(locale, MessageId::VoiceControlDisabled)
970 };
971 CommandResult::message(msg)
972 }
973
974 #[cfg(test)]
975 mod tests {
976 use super::*;
977
978 #[tokio::test]
979 async fn voice_requests_preserve_openrouter_vendor_pin() {
980 use wiremock::matchers::{method, path};
981 use wiremock::{Mock, MockServer, ResponseTemplate};
982
983 let server = MockServer::start().await;
984 Mock::given(method("POST"))
985 .and(path("/v1/chat/completions"))
986 .respond_with(ResponseTemplate::new(200).set_body_json(serde_json::json!({
987 "choices": [{ "message": { "content": "{\"text\":\"hello\"}" } }]
988 })))
989 .expect(3)
990 .mount(&server)
991 .await;
992 let base_url = format!("{}/v1", server.uri());
993 transcribe(
994 "fixture-key",
995 &base_url,
996 &[0; 16],
997 Some("chutes/region-fixture"),
998 )
999 .await
1000 .unwrap();
1001 process_voice_control(
1002 "fixture-key",
1003 &base_url,
1004 &[0; 16],
1005 "existing text",
1006 Some("chutes/region-fixture"),
1007 )
1008 .await
1009 .unwrap();
1010 transcribe_with_model("fixture-key", &base_url, &[0; 16], GROQ_ASR_MODEL, None)
1011 .await
1012 .unwrap();
1013
1014 let requests = server.received_requests().await.unwrap();
1015 assert_eq!(requests.len(), 3);
1016 for request in &requests[..2] {
1017 let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap();
1018 assert_eq!(
1019 body["provider"],
1020 serde_json::json!({"order": ["chutes/region-fixture"], "allow_fallbacks": false})
1021 );
1022 }
1023 let independent: serde_json::Value = serde_json::from_slice(&requests[2].body).unwrap();
1024 assert!(independent.get("provider").is_none());
1025 }
1026
1027 #[test]
1028 fn wav_encoding_produces_valid_header() {
1029 let samples = vec![0i16; 16000]; // 1 second of silence
1030 let wav = encode_wav(&samples);
1031 assert_eq!(&wav[0..4], b"RIFF");
1032 assert_eq!(&wav[8..12], b"WAVE");
1033 assert_eq!(&wav[12..16], b"fmt ");
1034 // data size = 16000 * 2 = 32000
1035 assert_eq!(&wav[4..8], &(36 + 32000u32).to_le_bytes());
1036 }
1037
1038 #[test]
1039 fn wav_encoding_empty_is_minimal() {
1040 let wav = encode_wav(&[]);
1041 assert_eq!(wav.len(), 44);
1042 assert_eq!(&wav[4..8], &36u32.to_le_bytes());
1043 }
1044
1045 #[test]
1046 fn send_suffix_detected_and_stripped() {
1047 assert_eq!(split_send_suffix("send it"), ("", true));
1048 assert_eq!(split_send_suffix("Send It!"), ("", true));
1049 assert_eq!(split_send_suffix("发送"), ("", true));
1050 assert_eq!(split_send_suffix("發送。"), ("", true));
1051 assert_eq!(
1052 split_send_suffix("ship the fix, send it"),
1053 ("ship the fix", true)
1054 );
1055 assert_eq!(
1056 split_send_suffix("修复这个问题,发送"),
1057 ("修复这个问题", true)
1058 );
1059 }
1060
1061 #[test]
1062 fn send_suffix_leaves_plain_text_alone() {
1063 assert_eq!(split_send_suffix("send it now"), ("send it now", false));
1064 assert_eq!(
1065 split_send_suffix("帮我发送一封邮件"),
1066 ("帮我发送一封邮件", false)
1067 );
1068 assert_eq!(split_send_suffix("发送邮件"), ("发送邮件", false));
1069 assert_eq!(
1070 split_send_suffix("resend it to the queue"),
1071 ("resend it to the queue", false)
1072 );
1073 }
1074
1075 #[test]
1076 fn recorder_detection_does_not_crash() {
1077 // Just verify the function runs without panicking
1078 let _ = is_available();
1079 }
1080 }
1081
1081 lines RUST