返回 CodeWhale
speech.rs
根目录 / crates / tui / src / tools / speech.rs
1 //! Model-visible Xiaomi MiMo speech/TTS generation tool.
2 //!
3 //! This mirrors the CLI `speech` / `tts` command as a first-class API tool so
4 //! the TUI model can generate narrated audio without shelling out to a nested
5 //! CodeWhale process.
6
7 use std::path::{Path, PathBuf};
8
9 use anyhow::Context as _;
10 use async_trait::async_trait;
11 use base64::{Engine as _, engine::general_purpose};
12 use serde_json::{Value, json};
13
14 use crate::client::{CodewhaleClient, SpeechSynthesisRequest};
15 use crate::config::{ApiProvider, normalize_model_name_for_provider};
16 use crate::network_policy::{Decision, host_from_url};
17
18 use super::spec::{
19 ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec,
20 optional_bool, optional_str, required_str,
21 };
22
23 pub(crate) const DEFAULT_FORMAT: &str = "wav";
24 pub(crate) const DEFAULT_VOICE: &str = "mimo_default";
25 const VOICE_CLONE_BASE64_MAX_BYTES: usize = 10 * 1024 * 1024;
26 pub(crate) const SUPPORTED_SPEECH_FORMATS: &[&str] = &["wav", "mp3", "pcm16"];
27
28 pub const SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS: &[&str] = &[
29 "mimo-v2.5-tts-voiceclone",
30 "mimo-v2.5-tts-voicedesign",
31 "mimo-v2.5-tts",
32 "mimo-v2-tts",
33 ];
34
35 pub(crate) const SPEECH_MODEL_EXAMPLES: &[&str] = &[
36 "mimo-v2.5-tts",
37 "mimo-v2.5-tts-voicedesign",
38 "mimo-v2.5-tts-voiceclone",
39 "mimo-v2-tts",
40 ];
41
42 pub struct SpeechTool {
43 name: &'static str,
44 client: Option<CodewhaleClient>,
45 output_dir: Option<PathBuf>,
46 /// The canonical `speech` entry is model-visible; `tts` is a hidden
47 /// compat alias so one capability costs one catalog entry (#5941).
48 visible: bool,
49 }
50
51 impl SpeechTool {
52 #[must_use]
53 pub fn new(
54 name: &'static str,
55 client: Option<CodewhaleClient>,
56 output_dir: Option<PathBuf>,
57 ) -> Self {
58 Self {
59 name,
60 client,
61 output_dir,
62 visible: true,
63 }
64 }
65
66 /// A hidden alias for saved-transcript replay: same behaviour, not
67 /// advertised to the model.
68 #[must_use]
69 pub fn alias(
70 name: &'static str,
71 client: Option<CodewhaleClient>,
72 output_dir: Option<PathBuf>,
73 ) -> Self {
74 Self {
75 name,
76 client,
77 output_dir,
78 visible: false,
79 }
80 }
81 }
82
83 #[async_trait]
84 impl ToolSpec for SpeechTool {
85 fn name(&self) -> &str {
86 self.name
87 }
88
89 fn model_visible(&self) -> bool {
90 self.visible
91 }
92
93 fn description(&self) -> &str {
94 "Generate speech/audio through the configured speech (TTS) provider. Use this when the user asks for speech, TTS, narration, read-aloud, voice design, or voice cloning. The provider decides which models and voices exist; omit model to take its default."
95 }
96
97 fn input_schema(&self) -> Value {
98 json!({
99 "type": "object",
100 "properties": {
101 "text": {
102 "type": "string",
103 "description": "Text to synthesize; the spoken content. Provider style/audio tags may be included when the configured provider supports them."
104 },
105 "output": {
106 "type": "string",
107 "description": "Audio file path to write, relative to the workspace unless absolute. Default: speech.<format> in output_dir, configured [speech].output_dir, or the workspace."
108 },
109 "output_dir": {
110 "type": "string",
111 "description": "Directory for the default speech.<format> output file when output is omitted. Relative paths stay inside the workspace."
112 },
113 "model": {
114 "type": "string",
115 "description": "TTS model. Omit to take the configured provider's default, or its voice-design/voice-clone model when voice_prompt/clone_voice is set (for the MiMo provider: mimo-v2.5-tts and variants).",
116 "enum": SPEECH_MODEL_EXAMPLES
117 },
118 "voice": {
119 "type": "string",
120 "description": "Built-in voice ID from the configured provider (MiMo examples: mimo_default, 冰糖, 茉莉, 苏打, 白桦, Mia, Chloe, Milo, Dean) or a data:audio/...;base64,... URI for voice clone."
121 },
122 "instruction": {
123 "type": "string",
124 "description": "Natural-language style, emotion, speed, scene, or performance instruction. It is not spoken verbatim."
125 },
126 "voice_prompt": {
127 "type": "string",
128 "description": "Voice design prompt. When model is omitted the provider's voice-design model is used (MiMo: mimo-v2.5-tts-voicedesign)."
129 },
130 "clone_voice": {
131 "type": "string",
132 "description": "Path to a .mp3 or .wav voice sample for cloning. When model is omitted the provider's voice-clone model is used (MiMo: mimo-v2.5-tts-voiceclone)."
133 },
134 "format": {
135 "type": "string",
136 "description": "Requested audio format. Default: wav. Providers commonly document wav and pcm16; mp3 is accepted when the API returns it.",
137 "enum": SUPPORTED_SPEECH_FORMATS
138 },
139 "stream": {
140 "type": "boolean",
141 "description": "Low-latency streaming request. The direct tool currently writes complete audio files only, so leave this false."
142 }
143 },
144 "required": ["text"]
145 })
146 }
147
148 fn capabilities(&self) -> Vec<ToolCapability> {
149 vec![
150 ToolCapability::WritesFiles,
151 ToolCapability::Network,
152 ToolCapability::Sandboxable,
153 ]
154 }
155
156 fn approval_requirement(&self) -> ApprovalRequirement {
157 // Speech generation is an explicit user-facing generation action.
158 // Path resolution still enforces workspace/trusted-root boundaries.
159 ApprovalRequirement::Auto
160 }
161
162 async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> {
163 let text = required_str(&input, "text")?.trim().to_string();
164 if text.is_empty() {
165 return Err(ToolError::invalid_input("speech text cannot be empty"));
166 }
167
168 let client = self.client.clone().ok_or_else(|| {
169 ToolError::not_available(
170 "speech tool requires an active Xiaomi MiMo API client; configure provider = \"xiaomi-mimo\" and an API key first",
171 )
172 })?;
173
174 let requested_format_raw = optional_str(&input, "format")?
175 .map(str::trim)
176 .filter(|value| !value.is_empty())
177 .unwrap_or(DEFAULT_FORMAT);
178 let requested_format = normalize_speech_format(requested_format_raw).ok_or_else(|| {
179 ToolError::invalid_input(format!(
180 "unsupported speech format '{requested_format_raw}' (allowed: {})",
181 SUPPORTED_SPEECH_FORMATS.join(", ")
182 ))
183 })?;
184 if optional_bool(&input, "stream", false)? {
185 return Err(ToolError::invalid_input(
186 "stream=true low-latency speech output is not implemented in the direct tool yet; use stream=false to generate a complete audio file",
187 ));
188 }
189 let output_raw = optional_str(&input, "output")?
190 .map(str::trim)
191 .filter(|value| !value.is_empty());
192 let output_path = resolve_speech_output_path(
193 &input,
194 context,
195 output_raw,
196 &requested_format,
197 self.output_dir.as_ref(),
198 )?;
199 let output_label = output_raw
200 .map(str::to_string)
201 .unwrap_or_else(|| output_path.display().to_string());
202
203 let raw_voice = optional_str(&input, "voice")?
204 .map(str::trim)
205 .filter(|value| !value.is_empty())
206 .map(str::to_string);
207 let raw_instruction = optional_str(&input, "instruction")?
208 .map(str::trim)
209 .filter(|value| !value.is_empty())
210 .map(str::to_string);
211 let voice_prompt = optional_str(&input, "voice_prompt")?
212 .map(str::trim)
213 .filter(|value| !value.is_empty())
214 .map(str::to_string);
215 let clone_voice = optional_str(&input, "clone_voice")?
216 .map(str::trim)
217 .filter(|value| !value.is_empty())
218 .map(str::to_string);
219
220 let voice_is_data_uri = raw_voice
221 .as_deref()
222 .is_some_and(|value| value.starts_with("data:audio/"));
223 if clone_voice.is_some() && raw_voice.is_some() {
224 return Err(ToolError::invalid_input(
225 "use either clone_voice or voice for cloned voice data, not both",
226 ));
227 }
228 let model = infer_speech_model(
229 optional_str(&input, "model")?,
230 clone_voice.is_some() || voice_is_data_uri,
231 voice_prompt.is_some(),
232 );
233 let model_lower = model.to_ascii_lowercase();
234 if !model_lower.contains("tts") {
235 return Err(ToolError::invalid_input(format!(
236 "speech tool requires a TTS model (examples: {}), got '{model}'",
237 SPEECH_MODEL_EXAMPLES.join(", ")
238 )));
239 }
240
241 let is_voice_design = model_lower.contains("voicedesign");
242 let is_voice_clone = model_lower.contains("voiceclone");
243 let instruction = combine_speech_instructions(raw_instruction, voice_prompt);
244 if is_voice_design
245 && instruction
246 .as_deref()
247 .is_none_or(|value| value.trim().is_empty())
248 {
249 return Err(ToolError::invalid_input(
250 "mimo-v2.5-tts-voicedesign requires voice_prompt or instruction",
251 ));
252 }
253
254 let voice = if let Some(clone_path) = clone_voice {
255 let clone_path = context.resolve_path(&clone_path)?;
256 Some(encode_voice_clone_data_uri(&clone_path).await?)
257 } else if is_voice_design {
258 None
259 } else if let Some(value) = raw_voice {
260 Some(value)
261 } else if is_voice_clone {
262 return Err(ToolError::invalid_input(
263 "mimo-v2.5-tts-voiceclone requires clone_voice <mp3|wav> or voice <data-uri>",
264 ));
265 } else {
266 Some(DEFAULT_VOICE.to_string())
267 };
268
269 check_network_policy(context, client.base_url())?;
270
271 let response = client
272 .synthesize_speech(SpeechSynthesisRequest {
273 model: model.clone(),
274 text,
275 instruction,
276 audio_format: requested_format,
277 voice,
278 })
279 .await
280 .map_err(|err| {
281 ToolError::execution_failed(format!("speech synthesis failed: {err}"))
282 })?;
283
284 if let Some(parent) = output_path
285 .parent()
286 .filter(|path| !path.as_os_str().is_empty())
287 {
288 tokio::fs::create_dir_all(parent).await.map_err(|err| {
289 ToolError::execution_failed(format!(
290 "failed to create output directory {}: {err}",
291 parent.display()
292 ))
293 })?;
294 }
295 tokio::fs::write(&output_path, &response.audio_bytes)
296 .await
297 .map_err(|err| {
298 ToolError::execution_failed(format!(
299 "failed to write audio file {}: {err}",
300 output_path.display()
301 ))
302 })?;
303
304 let result = json!({
305 "mode": "speech",
306 "success": true,
307 "api": "Xiaomi MiMo OpenAI-compatible chat/completions speech synthesis",
308 "base_url": openai_compatible_base_url(client.base_url()),
309 "model": response.model,
310 "format": response.audio_format,
311 "stream": false,
312 "output": output_label,
313 "absolute_output": output_path.display().to_string(),
314 "bytes": response.audio_bytes.len(),
315 "voice": response.voice.as_deref().map(describe_speech_voice),
316 "transcript": response.transcript,
317 "supported_formats": SUPPORTED_SPEECH_FORMATS,
318 "supported_xiaomi_mimo_models": SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS,
319 });
320 ToolResult::json(&result).map_err(|err| {
321 ToolError::execution_failed(format!("failed to serialize result: {err}"))
322 })
323 }
324 }
325
326 pub(crate) fn infer_speech_model(
327 model: Option<&str>,
328 has_clone_voice: bool,
329 has_voice_prompt: bool,
330 ) -> String {
331 match model.map(str::trim).filter(|value| !value.is_empty()) {
332 Some(value) => normalize_model_name_for_provider(ApiProvider::XiaomiMimo, value)
333 .unwrap_or_else(|| value.into()),
334 None if has_clone_voice => "mimo-v2.5-tts-voiceclone".to_string(),
335 None if has_voice_prompt => "mimo-v2.5-tts-voicedesign".to_string(),
336 None => "mimo-v2.5-tts".to_string(),
337 }
338 }
339
340 pub(crate) fn combine_speech_instructions(
341 instruction: Option<String>,
342 voice_prompt: Option<String>,
343 ) -> Option<String> {
344 match (instruction, voice_prompt) {
345 (Some(instruction), Some(voice_prompt)) => {
346 let instruction = instruction.trim();
347 let voice_prompt = voice_prompt.trim();
348 if instruction.is_empty() {
349 Some(voice_prompt.to_string()).filter(|value| !value.is_empty())
350 } else if voice_prompt.is_empty() {
351 Some(instruction.to_string()).filter(|value| !value.is_empty())
352 } else {
353 Some(format!("{voice_prompt}\n\n{instruction}"))
354 }
355 }
356 (Some(value), None) | (None, Some(value)) => {
357 let value = value.trim().to_string();
358 if value.is_empty() { None } else { Some(value) }
359 }
360 (None, None) => None,
361 }
362 }
363
364 pub(crate) fn normalize_speech_format(format: &str) -> Option<String> {
365 let normalized = format.trim().to_ascii_lowercase();
366 match normalized.as_str() {
367 "wav" | "mp3" | "pcm16" => Some(normalized),
368 "pcm" => Some("pcm16".to_string()),
369 _ => None,
370 }
371 }
372
373 pub(crate) fn default_speech_output_name(format: &str) -> String {
374 format!(
375 "speech.{}",
376 normalize_speech_format(format)
377 .as_deref()
378 .unwrap_or(DEFAULT_FORMAT)
379 )
380 }
381
382 fn resolve_speech_output_path(
383 input: &Value,
384 context: &ToolContext,
385 output_raw: Option<&str>,
386 format: &str,
387 configured_output_dir: Option<&PathBuf>,
388 ) -> Result<PathBuf, ToolError> {
389 if let Some(output) = output_raw {
390 return context.resolve_path(output);
391 }
392
393 let filename = default_speech_output_name(format);
394 if let Some(output_dir) = optional_str(input, "output_dir")?
395 .map(str::trim)
396 .filter(|value| !value.is_empty())
397 {
398 return Ok(context.resolve_path(output_dir)?.join(filename));
399 }
400
401 if let Some(output_dir) = configured_output_dir {
402 return Ok(output_dir.join(filename));
403 }
404
405 Ok(context.workspace.join(filename))
406 }
407
408 async fn encode_voice_clone_data_uri(path: &Path) -> Result<String, ToolError> {
409 let bytes = tokio::fs::read(path).await.map_err(|err| {
410 ToolError::execution_failed(format!(
411 "failed to read voice clone sample {}: {err}",
412 path.display()
413 ))
414 })?;
415
416 voice_clone_data_uri_from_bytes(path, &bytes)
417 .map_err(|err| ToolError::invalid_input(err.to_string()))
418 }
419
420 pub(crate) fn encode_voice_clone_sample_data_uri(path: &Path) -> anyhow::Result<String> {
421 let bytes = std::fs::read(path)
422 .with_context(|| format!("Failed to read voice clone sample {}", path.display()))?;
423
424 voice_clone_data_uri_from_bytes(path, &bytes)
425 }
426
427 fn voice_clone_data_uri_from_bytes(path: &Path, bytes: &[u8]) -> anyhow::Result<String> {
428 let base64_audio = general_purpose::STANDARD.encode(bytes);
429 if base64_audio.len() > VOICE_CLONE_BASE64_MAX_BYTES {
430 anyhow::bail!(
431 "voice clone sample is too large after base64 encoding ({} bytes > 10 MB)",
432 base64_audio.len()
433 );
434 }
435
436 let extension = path
437 .extension()
438 .and_then(|value| value.to_str())
439 .unwrap_or_default()
440 .to_ascii_lowercase();
441 let mime = match extension.as_str() {
442 "mp3" => "audio/mpeg",
443 "wav" => "audio/wav",
444 other => {
445 anyhow::bail!("unsupported voice clone sample extension '{other}'. Use .mp3 or .wav.");
446 }
447 };
448
449 Ok(format!("data:{mime};base64,{base64_audio}"))
450 }
451
452 pub(crate) fn describe_speech_voice(voice: &str) -> String {
453 if voice.starts_with("data:") {
454 "embedded voice clone sample".to_string()
455 } else {
456 voice.to_string()
457 }
458 }
459
460 fn openai_compatible_base_url(base_url: &str) -> String {
461 let trimmed = base_url.trim_end_matches('/');
462 if trimmed.ends_with("/v1") || trimmed.ends_with("/beta") {
463 trimmed.to_string()
464 } else {
465 format!("{trimmed}/v1")
466 }
467 }
468
469 fn check_network_policy(context: &ToolContext, base_url: &str) -> Result<(), ToolError> {
470 let Some(decider) = context.network_policy.as_ref() else {
471 return Ok(());
472 };
473 let display_url = openai_compatible_base_url(base_url);
474 let Some(host) = host_from_url(&display_url) else {
475 return Ok(());
476 };
477 match decider.evaluate(&host, "speech") {
478 Decision::Allow => Ok(()),
479 Decision::Deny => Err(ToolError::permission_denied(format!(
480 "speech network call to '{host}' blocked by network policy"
481 ))),
482 Decision::Prompt => Err(ToolError::permission_denied(format!(
483 "speech network call to '{host}' requires approval; re-run after `/network allow {host}` or set network.default = \"allow\" in config"
484 ))),
485 }
486 }
487
488 #[cfg(test)]
489 mod tests {
490 use super::*;
491
492 #[test]
493 fn infers_speech_model_from_requested_mode() {
494 assert_eq!(infer_speech_model(None, false, false), "mimo-v2.5-tts");
495 assert_eq!(
496 infer_speech_model(None, false, true),
497 "mimo-v2.5-tts-voicedesign"
498 );
499 assert_eq!(
500 infer_speech_model(None, true, false),
501 "mimo-v2.5-tts-voiceclone"
502 );
503 assert_eq!(
504 infer_speech_model(Some("mimo-tts"), false, false),
505 "mimo-v2.5-tts"
506 );
507 assert_eq!(
508 infer_speech_model(Some("mimo-v2-tts"), false, false),
509 "mimo-v2-tts"
510 );
511 }
512
513 #[test]
514 fn combines_voice_prompt_before_instruction() {
515 assert_eq!(
516 combine_speech_instructions(
517 Some("Speak warmly.".to_string()),
518 Some("Young Chinese female voice".to_string())
519 )
520 .as_deref(),
521 Some("Young Chinese female voice\n\nSpeak warmly.")
522 );
523 assert_eq!(
524 combine_speech_instructions(Some(" calm ".to_string()), None).as_deref(),
525 Some("calm")
526 );
527 }
528
529 #[test]
530 fn normalizes_documented_speech_formats() {
531 assert_eq!(normalize_speech_format("WAV").as_deref(), Some("wav"));
532 assert_eq!(normalize_speech_format("pcm16").as_deref(), Some("pcm16"));
533 assert_eq!(normalize_speech_format("pcm").as_deref(), Some("pcm16"));
534 assert_eq!(normalize_speech_format("flac"), None);
535 }
536
537 #[test]
538 fn supported_xiaomi_mimo_speech_models_are_tts_only() {
539 assert!(
540 SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS
541 .iter()
542 .all(|model| model.to_ascii_lowercase().contains("tts")),
543 "model-visible speech list must not include chat-only MiMo models"
544 );
545 assert!(SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS.contains(&"mimo-v2.5-tts"));
546 assert!(!SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS.contains(&"mimo-v2.5-pro"));
547 assert!(!SUPPORTED_XIAOMI_MIMO_SPEECH_MODELS.contains(&"mimo-v2.5"));
548 }
549
550 #[test]
551 fn configured_output_dir_is_used_for_default_tool_output() {
552 let tmp = tempfile::tempdir().expect("tempdir");
553 let context = ToolContext::new(tmp.path().to_path_buf());
554 let configured = tmp.path().join("speech-artifacts");
555
556 let output = resolve_speech_output_path(
557 &json!({"text": "hello"}),
558 &context,
559 None,
560 "pcm",
561 Some(&configured),
562 )
563 .expect("output path");
564
565 assert_eq!(output, configured.join("speech.pcm16"));
566 }
567
568 #[test]
569 fn displays_openai_compatible_base_url() {
570 assert_eq!(
571 openai_compatible_base_url("https://api.xiaomimimo.com"),
572 "https://api.xiaomimimo.com/v1"
573 );
574 assert_eq!(
575 openai_compatible_base_url("https://api.xiaomimimo.com/v1"),
576 "https://api.xiaomimimo.com/v1"
577 );
578 }
579
580 #[test]
581 fn speech_tool_is_auto_approved_but_not_read_only() {
582 let tool = SpeechTool::new("speech", None, None);
583 assert_eq!(tool.name(), "speech");
584 assert_eq!(tool.approval_requirement(), ApprovalRequirement::Auto);
585 assert!(!tool.is_read_only());
586 let schema = tool.input_schema();
587 assert!(schema.to_string().contains("mimo-v2.5-tts-voiceclone"));
588 assert!(schema.to_string().contains("pcm16"));
589 assert!(schema.to_string().contains("stream"));
590 }
591 }
592
592 lines RUST