| 1 | //! MCP Registry sync tool. |
| 2 | //! |
| 3 | //! `registry_sync` fetches the MCP Registry index, filters stdio servers, |
| 4 | //! caches locally, and returns a summary. The snapshot is reused while |
| 5 | //! fresh (`INCREMENTAL_INTERVAL_SECS`); refresh is incremental via |
| 6 | //! `updated_since`, with a full pagination only when the snapshot is |
| 7 | //! missing or older than `FULL_RESYNC_INTERVAL_SECS`. Downloads run in the |
| 8 | //! background; the cache file is replaced atomically and doubles as the |
| 9 | //! launch-metadata store for `start_registry_mcp_server` (a failed sync |
| 10 | //! leaves the previous snapshot untouched). |
| 11 | //! |
| 12 | //! Upstream contract (MCP Registry, preview — breaking changes possible): |
| 13 | //! * List operation `GET /v0.1/servers` (cursor / limit / search / version |
| 14 | //! / include_deleted params): |
| 15 | //! <https://registry.modelcontextprotocol.io/docs#/operations/list-servers-v0.1> |
| 16 | //! OpenAPI source: <https://github.com/modelcontextprotocol/registry/blob/main/docs/reference/api/openapi.yaml> |
| 17 | //! * Aggregator integration guide (pagination format, server status |
| 18 | //! lifecycle): |
| 19 | //! <https://github.com/modelcontextprotocol/registry/blob/main/docs/modelcontextprotocol-io/registry-aggregators.mdx> |
| 20 | |
| 21 | use std::collections::{HashMap, HashSet}; |
| 22 | use std::path::{Path, PathBuf}; |
| 23 | use std::sync::{Arc, OnceLock}; |
| 24 | |
| 25 | use anyhow::Result; |
| 26 | use chrono::{DateTime, Utc}; |
| 27 | use serde::{Deserialize, Serialize}; |
| 28 | use serde_json::{Value, json}; |
| 29 | use tokio::sync::{Mutex as AsyncMutex, MutexGuard}; |
| 30 | |
| 31 | use crate::mcp::McpPool; |
| 32 | use crate::tools::spec::{ |
| 33 | ApprovalRequirement, ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, |
| 34 | }; |
| 35 | use crate::utils::write_atomic; |
| 36 | |
| 37 | // === Registry API response types === |
| 38 | |
| 39 | #[derive(Deserialize)] |
| 40 | struct RegistryResponse { |
| 41 | servers: Vec<RegistryServerEntry>, |
| 42 | metadata: Option<RegistryMetadata>, |
| 43 | } |
| 44 | |
| 45 | #[derive(Deserialize)] |
| 46 | struct RegistryServerEntry { |
| 47 | server: RegistryServer, |
| 48 | // Registry-managed metadata. Carries the lifecycle `status` under the |
| 49 | // official extension key (see `RegistryOfficialMeta`); the |
| 50 | // publisher-provided subkey is deliberately not declared. |
| 51 | #[serde(rename = "_meta", default)] |
| 52 | meta: Option<RegistryResponseMeta>, |
| 53 | } |
| 54 | |
| 55 | impl RegistryServerEntry { |
| 56 | /// Lifecycle status reported by the official registry extension. |
| 57 | /// `"active"` (or an absent extension) keeps the entry; `"deprecated"` |
| 58 | /// and `"deleted"` retire it — the aggregator guide recommends dropping |
| 59 | /// `deleted` entries (moderation takedowns: spam/malware/illegal) from |
| 60 | /// downstream indexes, and we treat `deprecated` the same so the model |
| 61 | /// is only offered servers the publisher still stands behind. |
| 62 | /// <https://github.com/modelcontextprotocol/registry/blob/main/docs/modelcontextprotocol-io/registry-aggregators.mdx> |
| 63 | fn lifecycle_status(&self) -> Option<&str> { |
| 64 | self.meta |
| 65 | .as_ref() |
| 66 | .and_then(|m| m.official.as_ref()) |
| 67 | .and_then(|o| o.status.as_deref()) |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | #[derive(Deserialize)] |
| 72 | struct RegistryResponseMeta { |
| 73 | #[serde(rename = "io.modelcontextprotocol.registry/official", default)] |
| 74 | official: Option<RegistryOfficialMeta>, |
| 75 | } |
| 76 | |
| 77 | /// `status` is required upstream (enum `active | deprecated | deleted`); |
| 78 | /// kept Optional here so a missing extension never fails a page parse. |
| 79 | #[derive(Deserialize)] |
| 80 | struct RegistryOfficialMeta { |
| 81 | #[serde(default)] |
| 82 | status: Option<String>, |
| 83 | } |
| 84 | |
| 85 | #[derive(Deserialize)] |
| 86 | struct RegistryServer { |
| 87 | name: String, |
| 88 | description: String, |
| 89 | // `title`, `version`, `repository` are deliberately not declared — |
| 90 | // the cache no longer carries them (see `McpRegistryServerEntry`) |
| 91 | // and serde silently drops any extra fields, so we don't pay to |
| 92 | // validate or store data we'd immediately throw away. All three are |
| 93 | // optional in the upstream 2025-12 schema. |
| 94 | #[serde(default)] |
| 95 | packages: Option<Vec<RegistryPackage>>, |
| 96 | } |
| 97 | |
| 98 | #[derive(Deserialize)] |
| 99 | struct RegistryPackage { |
| 100 | #[serde(rename = "registryType")] |
| 101 | registry_type: String, |
| 102 | identifier: String, |
| 103 | // The upstream OCI entries (e.g. docker.io/foo/bar:1.2.3) omit the |
| 104 | // top-level `version` field because the tag is the version. Mirror that |
| 105 | // — Optional, with a fallback that parses the trailing `:tag` from the |
| 106 | // identifier when missing. |
| 107 | #[serde(default)] |
| 108 | version: Option<String>, |
| 109 | // The upstream 2025-12 schema dropped `runtimeHint` for nearly every |
| 110 | // entry (35/36 in the first page omit it; the runner is implied by |
| 111 | // `registryType`). Keep it Optional and fall back to a registry-type |
| 112 | // table when absent. |
| 113 | #[serde(rename = "runtimeHint", default)] |
| 114 | runtime_hint: Option<String>, |
| 115 | // The upstream schema now models `transport` as an object: |
| 116 | // `{"type": "stdio"}`. Older docs showed a bare string. Accept both |
| 117 | // so a future flip-back doesn't break us. |
| 118 | #[serde(deserialize_with = "deserialize_transport", default)] |
| 119 | transport: Option<String>, |
| 120 | #[serde(default)] |
| 121 | #[serde(rename = "packageArguments")] |
| 122 | package_arguments: Vec<RegistryArg>, |
| 123 | #[serde( |
| 124 | rename = "runtimeArguments", |
| 125 | deserialize_with = "deserialize_runtime_arguments", |
| 126 | default |
| 127 | )] |
| 128 | runtime_arguments: Vec<String>, |
| 129 | /// Registry-provided environment requirements are intentionally kept |
| 130 | /// transient. Runtime-discovered servers have no configuration channel |
| 131 | /// for secrets/API keys, so any package declaring environment variables |
| 132 | /// is ineligible and never reaches the on-disk cache. |
| 133 | #[serde(rename = "environmentVariables", default)] |
| 134 | environment_variables: Value, |
| 135 | } |
| 136 | |
| 137 | impl RegistryPackage { |
| 138 | fn declares_environment_variables(&self) -> bool { |
| 139 | match &self.environment_variables { |
| 140 | Value::Null => false, |
| 141 | Value::Array(values) => !values.is_empty(), |
| 142 | Value::Object(values) => !values.is_empty(), |
| 143 | // Fail closed if a future Registry schema uses an unexpected shape. |
| 144 | _ => true, |
| 145 | } |
| 146 | } |
| 147 | } |
| 148 | |
| 149 | /// Deserialize `transport` as either a bare string (`"stdio"`) or an object |
| 150 | /// (`{"type": "stdio"}`). The MCP Registry 2025-12 schema ships the object |
| 151 | /// shape; older/draft docs showed the bare string. We accept both. |
| 152 | fn deserialize_transport<'de, D>(deserializer: D) -> Result<Option<String>, D::Error> |
| 153 | where |
| 154 | D: serde::Deserializer<'de>, |
| 155 | { |
| 156 | #[derive(Deserialize)] |
| 157 | #[serde(untagged)] |
| 158 | enum OneOrString { |
| 159 | Bare(String), |
| 160 | Wrapped { |
| 161 | #[serde(rename = "type")] |
| 162 | r#type: String, |
| 163 | }, |
| 164 | } |
| 165 | |
| 166 | let opt: Option<OneOrString> = Option::deserialize(deserializer)?; |
| 167 | Ok(opt.map(|v| match v { |
| 168 | OneOrString::Bare(s) => s, |
| 169 | OneOrString::Wrapped { r#type } => r#type, |
| 170 | })) |
| 171 | } |
| 172 | |
| 173 | /// Deserialize `runtimeArguments` as either `Vec<String>` (old schema) |
| 174 | /// or `Vec<{value, name, default, type, ...}>` (2025-12 schema). In the |
| 175 | /// object case we derive a string value from the available fields: |
| 176 | /// - Named args (`type: "named"`): `"{name} {default}"` or just `name` |
| 177 | /// - Positional args (`type: "positional"`): `default` |
| 178 | /// - Legacy objects with `value`: use `value` directly |
| 179 | fn deserialize_runtime_arguments<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error> |
| 180 | where |
| 181 | D: serde::Deserializer<'de>, |
| 182 | { |
| 183 | #[derive(Deserialize)] |
| 184 | #[serde(untagged)] |
| 185 | enum StringOrArg { |
| 186 | Bare(String), |
| 187 | Wrapped { |
| 188 | #[serde(default)] |
| 189 | value: Option<String>, |
| 190 | #[serde(default)] |
| 191 | name: Option<String>, |
| 192 | #[serde(default)] |
| 193 | default: Option<String>, |
| 194 | }, |
| 195 | } |
| 196 | |
| 197 | let raw: Vec<StringOrArg> = Vec::deserialize(deserializer)?; |
| 198 | let mut args = Vec::new(); |
| 199 | for arg in raw { |
| 200 | match arg { |
| 201 | StringOrArg::Bare(value) => args.push(value), |
| 202 | StringOrArg::Wrapped { |
| 203 | value: Some(value), .. |
| 204 | } => args.push(value), |
| 205 | StringOrArg::Wrapped { name, default, .. } => { |
| 206 | if let Some(name) = name { |
| 207 | args.push(name); |
| 208 | } |
| 209 | if let Some(default) = default { |
| 210 | args.push(default); |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | } |
| 215 | Ok(args) |
| 216 | } |
| 217 | |
| 218 | /// Derive a runtime hint from `registryType` when the upstream omits one. |
| 219 | /// Kept small on purpose: only the runtimes we know how to launch. |
| 220 | fn default_runtime_hint(registry_type: &str) -> Option<&'static str> { |
| 221 | match registry_type { |
| 222 | "npm" => Some("npx"), |
| 223 | "pypi" => Some("uvx"), |
| 224 | _ => None, |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | #[derive(Deserialize)] |
| 229 | struct RegistryArg { |
| 230 | // Positional arguments ship without a name — only `value` and |
| 231 | // `type`. Named arguments (`{"name": "--foo", "value": "bar", ...}`) |
| 232 | // carry it. Accept both: when missing, downstream code uses `value` |
| 233 | // as the arg name. |
| 234 | #[serde(default)] |
| 235 | name: Option<String>, |
| 236 | description: Option<String>, |
| 237 | // Upstream allows omitting `isRequired`; default false per spec. |
| 238 | #[serde(rename = "isRequired", default)] |
| 239 | is_required: bool, |
| 240 | // Upstream `type` discriminator (`"positional"` / `"named"`). Drives |
| 241 | // cmd-format decisions downstream. Renamed because `type` is a |
| 242 | // reserved word in Rust. |
| 243 | #[serde(rename = "type", default)] |
| 244 | kind: Option<String>, |
| 245 | #[serde(default)] |
| 246 | value: Option<String>, |
| 247 | default: Option<String>, |
| 248 | // `format` dropped — never read by any consumer. |
| 249 | } |
| 250 | |
| 251 | // === Cached index types === |
| 252 | |
| 253 | #[derive(Deserialize)] |
| 254 | struct RegistryMetadata { |
| 255 | #[serde(rename = "nextCursor")] |
| 256 | next_cursor: Option<String>, |
| 257 | } |
| 258 | |
| 259 | // === Cached index types === |
| 260 | // |
| 261 | // The cache file (`~/.codewhale/mcp-index.json`) is the on-disk source of |
| 262 | // truth for Registry-discovered local MCP launch metadata. |
| 263 | |
| 264 | /// Bumped whenever the cache shape changes. Lets the loader detect an old |
| 265 | /// cache file and trigger a full resync instead of failing to deserialize. |
| 266 | pub const MCP_REGISTRY_CACHE_VERSION: u32 = 6; |
| 267 | |
| 268 | #[derive(Serialize, Deserialize, Clone)] |
| 269 | pub struct McpRegistryIndex { |
| 270 | pub version: u32, |
| 271 | pub count: usize, |
| 272 | pub servers: Vec<McpRegistryServerEntry>, |
| 273 | /// RFC3339 timestamp of the last successful sync. Absent or older |
| 274 | /// than `INCREMENTAL_INTERVAL_SECS` triggers the next refresh; |
| 275 | /// older than `FULL_RESYNC_INTERVAL_SECS` makes it a full resync. |
| 276 | #[serde(default)] |
| 277 | pub synced_at: Option<DateTime<Utc>>, |
| 278 | } |
| 279 | |
| 280 | /// One Registry catalog entry exposed to the model for contextual selection. |
| 281 | #[derive(Serialize, Deserialize, Clone)] |
| 282 | pub struct DigestEntry { |
| 283 | pub name: String, |
| 284 | pub description: String, |
| 285 | #[serde(default, skip_serializing_if = "Vec::is_empty")] |
| 286 | pub required_args: Vec<McpRegistryArgEntry>, |
| 287 | } |
| 288 | |
| 289 | /// One cached server entry used by discovery and structured startup. |
| 290 | /// |
| 291 | /// Everything else that the upstream Registry ships (`title`, repository, |
| 292 | /// `packages[]`, optional named args, |
| 293 | /// runtime_arguments at package level) is dropped. Fixed positional |
| 294 | /// `packageArguments` (e.g. an `mcp` subcommand) are folded into |
| 295 | /// `run_command` at render time rather than kept as fields. |
| 296 | #[derive(Serialize, Deserialize, Clone)] |
| 297 | pub struct McpRegistryServerEntry { |
| 298 | pub name: String, |
| 299 | pub description: String, |
| 300 | pub launch: McpLaunchSpec, |
| 301 | } |
| 302 | |
| 303 | /// Host-owned launch data for one zero-environment stdio server. |
| 304 | #[derive(Serialize, Deserialize, Clone)] |
| 305 | pub struct McpLaunchSpec { |
| 306 | /// Template for the run command. The literal substring `<ARGS>` is |
| 307 | /// replaced by host-rendered structured argument values. |
| 308 | pub run_command: String, |
| 309 | pub required_args: Vec<McpRegistryArgEntry>, |
| 310 | } |
| 311 | |
| 312 | /// One CLI argument required at install time. `is_required` was dropped |
| 313 | /// because the cache only stores required args (others are filtered out |
| 314 | /// during sync). `kind` carries the upstream `type` discriminator |
| 315 | /// (`"positional"` vs `"named"`) so the cmd builder can decide whether |
| 316 | /// to emit `--name value` or just `value`. |
| 317 | #[derive(Serialize, Deserialize, Clone)] |
| 318 | pub struct McpRegistryArgEntry { |
| 319 | pub name: String, |
| 320 | pub kind: Option<String>, |
| 321 | pub description: Option<String>, |
| 322 | pub default: Option<String>, |
| 323 | } |
| 324 | |
| 325 | // === Tool implementation === |
| 326 | |
| 327 | pub struct McpSyncRegistry { |
| 328 | cache_path_override: Option<PathBuf>, |
| 329 | } |
| 330 | |
| 331 | impl McpSyncRegistry { |
| 332 | /// Default instance; resolves the cache under `dirs::home_dir()`. |
| 333 | pub fn new() -> Self { |
| 334 | Self { |
| 335 | cache_path_override: None, |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | /// Test hook: pin the cache file to an explicit path. `dirs::home_dir()` |
| 340 | /// resolves the OS profile directory on Windows via SHGetKnownFolderPath, |
| 341 | /// which no environment variable can redirect, so tests that need a |
| 342 | /// hermetic cache inject the path directly on every platform. |
| 343 | #[cfg(test)] |
| 344 | pub fn with_cache_path(path: PathBuf) -> Self { |
| 345 | Self { |
| 346 | cache_path_override: Some(path), |
| 347 | } |
| 348 | } |
| 349 | |
| 350 | fn cache_path(&self) -> Result<PathBuf, ToolError> { |
| 351 | match &self.cache_path_override { |
| 352 | Some(path) => Ok(path.clone()), |
| 353 | None => dirs::home_dir() |
| 354 | .ok_or_else(|| ToolError::execution_failed("Cannot determine home directory")) |
| 355 | .map(|h| h.join(".codewhale").join("mcp-index.json")), |
| 356 | } |
| 357 | } |
| 358 | } |
| 359 | |
| 360 | const REGISTRY_API: &str = "https://registry.modelcontextprotocol.io/v0.1/servers"; |
| 361 | const PER_PAGE: usize = 100; |
| 362 | const REQUEST_TIMEOUT_SECS: u64 = 30; |
| 363 | /// Inter-page delay: the upstream stalls under request bursts. |
| 364 | const PAGE_PACING_MS: u64 = 500; |
| 365 | /// Freshness window: within it `registry_sync` serves the cache with zero |
| 366 | /// network requests; past it a background refresh starts. |
| 367 | const INCREMENTAL_INTERVAL_SECS: i64 = 24 * 60 * 60; |
| 368 | /// Age at which the background refresh falls back to a full pagination, |
| 369 | /// reconciling servers that vanished from the listing entirely. |
| 370 | const FULL_RESYNC_INTERVAL_SECS: i64 = 30 * 24 * 60 * 60; |
| 371 | /// Identifies the client (RFC 9110); matches the crate-wide convention in |
| 372 | /// `web/fetch.rs`. HTTP hygiene, not a fix for upstream stalls. |
| 373 | const USER_AGENT: &str = concat!( |
| 374 | "Mozilla/5.0 (compatible; codewhale/", |
| 375 | env!("CARGO_PKG_VERSION"), |
| 376 | "; +https://github.com/Hmbown/CodeWhale)" |
| 377 | ); |
| 378 | /// Bounded retries for one sync. The on-disk cache only changes at the |
| 379 | /// final atomic replace, so a failed fetch (HTTP/parse error) never |
| 380 | /// mutates state and retrying is side-effect free. |
| 381 | const MAX_SYNC_ATTEMPTS: usize = 3; |
| 382 | /// Connect budget for Registry-launched servers. The 10s global default is |
| 383 | /// meant for pre-installed servers; Registry packages are typically fetched |
| 384 | /// on first launch via npx/uvx, which routinely exceeds it. |
| 385 | const REGISTRY_CONNECT_TIMEOUT_SECS: u64 = 60; |
| 386 | |
| 387 | fn cache_path() -> Result<PathBuf, ToolError> { |
| 388 | dirs::home_dir() |
| 389 | .ok_or_else(|| ToolError::execution_failed("Cannot determine home directory")) |
| 390 | .map(|h| h.join(".codewhale").join("mcp-index.json")) |
| 391 | } |
| 392 | |
| 393 | fn read_cache(path: &Path) -> Option<McpRegistryIndex> { |
| 394 | let data = std::fs::read_to_string(path).ok()?; |
| 395 | serde_json::from_str(&data).ok() |
| 396 | } |
| 397 | |
| 398 | /// True when the snapshot is fresh enough to serve without a network |
| 399 | /// round-trip. Version mismatch or missing timestamp count as stale. |
| 400 | fn cache_is_fresh(cache: &McpRegistryIndex, now: DateTime<Utc>) -> bool { |
| 401 | if cache.version != MCP_REGISTRY_CACHE_VERSION { |
| 402 | return false; |
| 403 | } |
| 404 | cache.synced_at.is_some_and(|synced| { |
| 405 | now.signed_duration_since(synced).num_seconds() < INCREMENTAL_INTERVAL_SECS |
| 406 | }) |
| 407 | } |
| 408 | |
| 409 | /// Convert one fetched listing into launchable cache entries. Servers the |
| 410 | /// upstream marks `deleted`/`deprecated` are dropped (the aggregator guide |
| 411 | /// recommends removing `deleted` entries — moderation takedowns — from |
| 412 | /// downstream indexes), as is anything the structured launcher cannot run. |
| 413 | /// The cache is a full snapshot every sync, so this filtering is the whole |
| 414 | /// story: retired servers simply never enter the fresh index. |
| 415 | /// Status lives in registry-managed `_meta` |
| 416 | /// (`ServerResponse._meta["io.modelcontextprotocol.registry/official"] |
| 417 | /// .status`, enum `active | deprecated | deleted`). |
| 418 | /// <https://github.com/modelcontextprotocol/registry/blob/main/docs/reference/api/openapi.yaml> |
| 419 | fn viable_entries(entries: Vec<RegistryServerEntry>) -> Vec<McpRegistryServerEntry> { |
| 420 | entries |
| 421 | .into_iter() |
| 422 | .filter(|entry| { |
| 423 | !matches!( |
| 424 | entry.lifecycle_status(), |
| 425 | Some("deleted") | Some("deprecated") |
| 426 | ) |
| 427 | }) |
| 428 | .filter_map(|entry| server_to_entry(entry.server)) |
| 429 | .collect() |
| 430 | } |
| 431 | |
| 432 | /// Render the run command template. Positional `packageArguments` with a |
| 433 | /// fixed literal (or defaulted) value are part of the invocation itself — |
| 434 | /// e.g. the `mcp` subcommand in `npx -y agentic-mermaid@0.1.2 mcp` — so |
| 435 | /// they are folded into the command right after the package spec. |
| 436 | /// `<ARGS>` is the splice point for everything user-supplied: positional |
| 437 | /// placeholders plus each `required_args` entry, rendered as named or |
| 438 | /// positional arguments by the structured Registry launcher. |
| 439 | fn build_run_command( |
| 440 | runtime_hint: &str, |
| 441 | identifier: &str, |
| 442 | version: &str, |
| 443 | runtime_arguments: &[String], |
| 444 | package_arguments: &[RegistryArg], |
| 445 | ) -> String { |
| 446 | let runtime = runtime_hint; |
| 447 | let mut normalized_runtime_arguments = runtime_arguments.to_vec(); |
| 448 | if runtime_hint == "npx" |
| 449 | && !normalized_runtime_arguments |
| 450 | .iter() |
| 451 | .any(|argument| matches!(argument.as_str(), "-y" | "--yes")) |
| 452 | { |
| 453 | normalized_runtime_arguments.insert(0, "-y".to_string()); |
| 454 | } |
| 455 | let mid = normalized_runtime_arguments |
| 456 | .iter() |
| 457 | .map(|argument| shell_words::quote(argument)) |
| 458 | .collect::<Vec<_>>() |
| 459 | .join(" "); |
| 460 | let mid_with_space = if mid.is_empty() { |
| 461 | String::new() |
| 462 | } else { |
| 463 | format!("{mid} ") |
| 464 | }; |
| 465 | let (sep, tail) = match runtime_hint { |
| 466 | "npx" => ("@", version.to_string()), |
| 467 | "uvx" => ("==", version.to_string()), |
| 468 | _ => return String::new(), |
| 469 | }; |
| 470 | // Upstream positional packageArguments ship without a `name`; named |
| 471 | // args always carry one. A nameless arg with a literal `value` (or a |
| 472 | // `default`) is a fixed token of the invocation, not user input — |
| 473 | // dropping it renders a command that cannot start the server (the |
| 474 | // agentic-mermaid `mcp` subcommand bug). |
| 475 | let fixed: Vec<String> = package_arguments |
| 476 | .iter() |
| 477 | .filter(|a| !a.is_required) |
| 478 | .filter(|a| a.name.is_none()) |
| 479 | .filter_map(|a| a.value.as_deref().or(a.default.as_deref())) |
| 480 | .map(|value| shell_words::quote(value).into_owned()) |
| 481 | .collect(); |
| 482 | let fixed_str = if fixed.is_empty() { |
| 483 | String::new() |
| 484 | } else { |
| 485 | format!(" {}", fixed.join(" ")) |
| 486 | }; |
| 487 | let package_spec = format!("{identifier}{sep}{tail}"); |
| 488 | let package = shell_words::quote(&package_spec); |
| 489 | format!("{runtime} {mid_with_space}{package}{fixed_str} <ARGS>") |
| 490 | } |
| 491 | |
| 492 | fn build_launch_spec( |
| 493 | runtime_hint: &str, |
| 494 | identifier: &str, |
| 495 | version: &str, |
| 496 | pkg: &RegistryPackage, |
| 497 | ) -> McpLaunchSpec { |
| 498 | McpLaunchSpec { |
| 499 | run_command: build_run_command( |
| 500 | runtime_hint, |
| 501 | identifier, |
| 502 | version, |
| 503 | &pkg.runtime_arguments, |
| 504 | &pkg.package_arguments, |
| 505 | ), |
| 506 | required_args: pkg |
| 507 | .package_arguments |
| 508 | .iter() |
| 509 | .filter(|a| a.is_required) |
| 510 | .enumerate() |
| 511 | .map(|(index, a)| McpRegistryArgEntry { |
| 512 | // Positional args omit `name` upstream; fall back to the |
| 513 | // value so the cache still carries something the cmd |
| 514 | // builder can render. |
| 515 | name: a |
| 516 | .name |
| 517 | .clone() |
| 518 | .or_else(|| a.value.clone()) |
| 519 | .unwrap_or_else(|| format!("arg_{}", index + 1)), |
| 520 | kind: a.kind.clone(), |
| 521 | description: a.description.clone(), |
| 522 | default: a.default.clone(), |
| 523 | }) |
| 524 | .collect(), |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | fn server_to_entry(server: RegistryServer) -> Option<McpRegistryServerEntry> { |
| 529 | // We only need the FIRST viable stdio package per server for |
| 530 | // launch metadata; everything beyond it would just duplicate |
| 531 | // info. Filter to stdio, resolve runtime_hint + version, and stop |
| 532 | // at the first hit. |
| 533 | let first_pkg = server |
| 534 | .packages |
| 535 | .unwrap_or_default() |
| 536 | .into_iter() |
| 537 | .filter(|p| p.transport.as_deref() == Some("stdio")) |
| 538 | .filter(|p| !p.declares_environment_variables()) |
| 539 | // The automatic launcher currently has deterministic install/run |
| 540 | // semantics for package-manager-backed npm and PyPI entries only. |
| 541 | .filter(|p| matches!(p.registry_type.as_str(), "npm" | "pypi")) |
| 542 | .find_map(|p| { |
| 543 | let expected_hint = default_runtime_hint(&p.registry_type)?; |
| 544 | let hint = p |
| 545 | .runtime_hint |
| 546 | .clone() |
| 547 | .unwrap_or_else(|| expected_hint.to_string()); |
| 548 | if hint != expected_hint { |
| 549 | return None; |
| 550 | } |
| 551 | let version = p.version.clone()?; |
| 552 | Some((p, hint, version)) |
| 553 | }); |
| 554 | |
| 555 | let (pkg, hint, version) = first_pkg?; |
| 556 | |
| 557 | Some(McpRegistryServerEntry { |
| 558 | name: server.name, |
| 559 | description: server.description, |
| 560 | launch: build_launch_spec(&hint, &pkg.identifier, &version, &pkg), |
| 561 | }) |
| 562 | } |
| 563 | |
| 564 | /// Prompt attached to every `registry_sync` result. |
| 565 | /// |
| 566 | /// A scored list is not a verdict. This used to read "you must call |
| 567 | /// start_registry_mcp_server ... before using shell commands, local programs, |
| 568 | /// custom code, or a manual implementation", which turned any near-miss row |
| 569 | /// into an obligation: a turn that only had to write an HTML file and read a |
| 570 | /// fixture matched a browser-automation server and spent itself on starting |
| 571 | /// that server. The result now describes the list and leaves the choice with |
| 572 | /// the model, while keeping the one instruction the host actually depends on — |
| 573 | /// start an entry through `start_registry_mcp_server`, never by running its |
| 574 | /// package command through the shell. |
| 575 | const REGISTRY_FIRST_PROMPT: &str = concat!( |
| 576 | "These are the top scored matches for your query from the local Registry ", |
| 577 | "snapshot; the full catalog stays on the host. A match is only worth ", |
| 578 | "starting when it covers a capability you do not already have — if an ", |
| 579 | "available tool, a project script, or a few lines of local code already ", |
| 580 | "does the job, use that and ignore these results. To start one, call ", |
| 581 | "start_registry_mcp_server with its exact name; never install or run its ", |
| 582 | "package command through the shell. If nothing here covers the missing ", |
| 583 | "capability, refine the query once, then continue with local tools.", |
| 584 | ); |
| 585 | |
| 586 | /// Host-side cap on model-visible Registry matches. The complete catalog |
| 587 | /// stays on disk; only this many matched entries ever reach the model. |
| 588 | const MAX_REGISTRY_MATCHES: usize = 8; |
| 589 | |
| 590 | #[derive(Serialize)] |
| 591 | struct RegistryCatalogResult { |
| 592 | instruction: &'static str, |
| 593 | /// Total entries in the on-disk catalog (reported, never shipped). |
| 594 | total: usize, |
| 595 | query: String, |
| 596 | servers: Vec<DigestEntry>, |
| 597 | } |
| 598 | |
| 599 | fn catalog_from_cache(cache: &McpRegistryIndex, query: &str) -> RegistryCatalogResult { |
| 600 | let servers = search_registry_entries(&cache.servers, query, MAX_REGISTRY_MATCHES); |
| 601 | RegistryCatalogResult { |
| 602 | instruction: REGISTRY_FIRST_PROMPT, |
| 603 | total: cache.servers.len(), |
| 604 | query: query.to_string(), |
| 605 | servers, |
| 606 | } |
| 607 | } |
| 608 | |
| 609 | /// Deterministic host-side scoring: name hits outrank description hits, |
| 610 | /// exact name match outranks substring, ties break alphabetically. The model |
| 611 | /// never sees the un-matched remainder of the catalog. |
| 612 | fn search_registry_entries( |
| 613 | entries: &[McpRegistryServerEntry], |
| 614 | query: &str, |
| 615 | limit: usize, |
| 616 | ) -> Vec<DigestEntry> { |
| 617 | let terms = query |
| 618 | .split_whitespace() |
| 619 | .map(str::to_ascii_lowercase) |
| 620 | .filter(|term| !term.is_empty()) |
| 621 | .collect::<Vec<_>>(); |
| 622 | let mut scored = entries |
| 623 | .iter() |
| 624 | .map(|server| { |
| 625 | let name = server.name.to_ascii_lowercase(); |
| 626 | let description = server.description.to_ascii_lowercase(); |
| 627 | let mut score = 0u32; |
| 628 | for term in &terms { |
| 629 | if name == *term { |
| 630 | score = score.saturating_add(100); |
| 631 | } else if name.contains(term) { |
| 632 | score = score.saturating_add(50); |
| 633 | } |
| 634 | if description.contains(term) { |
| 635 | score = score.saturating_add(10); |
| 636 | } |
| 637 | } |
| 638 | (score, server) |
| 639 | }) |
| 640 | .filter(|(score, _)| *score > 0) |
| 641 | .collect::<Vec<_>>(); |
| 642 | scored.sort_by(|(a_score, a), (b_score, b)| { |
| 643 | b_score.cmp(a_score).then_with(|| a.name.cmp(&b.name)) |
| 644 | }); |
| 645 | scored |
| 646 | .into_iter() |
| 647 | .take(limit) |
| 648 | .map(|(_, server)| DigestEntry { |
| 649 | name: server.name.clone(), |
| 650 | description: server.description.clone(), |
| 651 | required_args: server.launch.required_args.clone(), |
| 652 | }) |
| 653 | .collect() |
| 654 | } |
| 655 | |
| 656 | /// Always fast: serve the local snapshot and start a background download |
| 657 | /// only when it is missing or stale. |
| 658 | async fn load_registry_catalog( |
| 659 | path: &Path, |
| 660 | query: &str, |
| 661 | ) -> Result<RegistryCatalogResult, ToolError> { |
| 662 | let existing = read_cache(path); |
| 663 | let fresh = existing |
| 664 | .as_ref() |
| 665 | .is_some_and(|cache| cache_is_fresh(cache, Utc::now())); |
| 666 | if !fresh { |
| 667 | spawn_background_sync(path); |
| 668 | } |
| 669 | Ok(catalog_for_snapshot(existing, query)) |
| 670 | } |
| 671 | |
| 672 | /// Decide what `registry_sync` returns: the scored top matches for the |
| 673 | /// query from the cached entries (they pin their own package versions, so |
| 674 | /// snapshot age is irrelevant), or an empty match set when no snapshot |
| 675 | /// exists yet. |
| 676 | fn catalog_for_snapshot(existing: Option<McpRegistryIndex>, query: &str) -> RegistryCatalogResult { |
| 677 | match existing { |
| 678 | Some(cache) => catalog_from_cache(&cache, query), |
| 679 | // No snapshot yet: an empty match set has no candidates to act on; |
| 680 | // a background sync has already been queued. |
| 681 | None => RegistryCatalogResult { |
| 682 | instruction: "", |
| 683 | total: 0, |
| 684 | query: query.to_string(), |
| 685 | servers: Vec::new(), |
| 686 | }, |
| 687 | } |
| 688 | } |
| 689 | |
| 690 | /// Process-wide lock so at most one background sync runs at a time; |
| 691 | /// released when the sync settles, so the next call can retry. |
| 692 | fn try_acquire_sync_permit() -> Option<MutexGuard<'static, ()>> { |
| 693 | static SYNC_GUARD: OnceLock<AsyncMutex<()>> = OnceLock::new(); |
| 694 | SYNC_GUARD |
| 695 | .get_or_init(|| AsyncMutex::new(())) |
| 696 | .try_lock() |
| 697 | .ok() |
| 698 | } |
| 699 | |
| 700 | /// Start the download in the background. Returns false when one is |
| 701 | /// already running. |
| 702 | fn spawn_background_sync(path: &Path) -> bool { |
| 703 | let Some(permit) = try_acquire_sync_permit() else { |
| 704 | return false; |
| 705 | }; |
| 706 | let path = path.to_path_buf(); |
| 707 | tokio::spawn(async move { |
| 708 | if let Err(error) = sync_once(&path).await { |
| 709 | tracing::warn!("background Registry sync failed: {error}"); |
| 710 | } |
| 711 | drop(permit); |
| 712 | }); |
| 713 | true |
| 714 | } |
| 715 | |
| 716 | /// Paginate the listing; with `updated_since`, only servers updated after |
| 717 | /// it. The filter MUST be repeated on every page — the cursor alone does |
| 718 | /// not carry it (verified against the live API). Never writes anything. |
| 719 | async fn fetch_registry_entries( |
| 720 | client: &reqwest::Client, |
| 721 | updated_since: Option<DateTime<Utc>>, |
| 722 | ) -> Result<Vec<RegistryServerEntry>, ToolError> { |
| 723 | let mut all_entries: Vec<RegistryServerEntry> = Vec::new(); |
| 724 | let mut cursor: Option<String> = None; |
| 725 | loop { |
| 726 | let mut url = format!("{REGISTRY_API}?version=latest&limit={PER_PAGE}"); |
| 727 | if let Some(since) = updated_since { |
| 728 | url.push_str(&format!( |
| 729 | "&updated_since={}", |
| 730 | urlencoding::encode(&since.to_rfc3339()) |
| 731 | )); |
| 732 | } |
| 733 | if let Some(ref c) = cursor { |
| 734 | url.push_str(&format!("&cursor={}", urlencoding::encode(c))); |
| 735 | } |
| 736 | let resp = client |
| 737 | .get(&url) |
| 738 | .send() |
| 739 | .await |
| 740 | .and_then(reqwest::Response::error_for_status) |
| 741 | .map_err(|e| ToolError::execution_failed(format!("Registry API: {e}")))?; |
| 742 | let text = resp |
| 743 | .text() |
| 744 | .await |
| 745 | .map_err(|e| ToolError::execution_failed(format!("Registry body: {e}")))?; |
| 746 | let body: RegistryResponse = serde_json::from_str(&text) |
| 747 | .map_err(|e| ToolError::execution_failed(format!("Registry JSON parse: {e}")))?; |
| 748 | all_entries.extend(body.servers); |
| 749 | cursor = body.metadata.and_then(|m| m.next_cursor); |
| 750 | if cursor.is_none() { |
| 751 | break; |
| 752 | } |
| 753 | // Pace the next request so a burst of page fetches does not stall |
| 754 | // the upstream (and, with many clients, each other). |
| 755 | tokio::time::sleep(std::time::Duration::from_millis(PAGE_PACING_MS)).await; |
| 756 | } |
| 757 | Ok(all_entries) |
| 758 | } |
| 759 | |
| 760 | /// Refresh strategy: full listing, or `updated_since` delta from the |
| 761 | /// snapshot's last sync (full when the snapshot is missing, legacy, or |
| 762 | /// older than `FULL_RESYNC_INTERVAL_SECS`). |
| 763 | #[derive(Debug, PartialEq)] |
| 764 | enum SyncStrategy { |
| 765 | Full, |
| 766 | Incremental { since: DateTime<Utc> }, |
| 767 | } |
| 768 | |
| 769 | fn sync_strategy(cache: Option<&McpRegistryIndex>, now: DateTime<Utc>) -> SyncStrategy { |
| 770 | match cache { |
| 771 | None => SyncStrategy::Full, |
| 772 | Some(cache) if cache.version != MCP_REGISTRY_CACHE_VERSION => SyncStrategy::Full, |
| 773 | Some(cache) => match cache.synced_at { |
| 774 | None => SyncStrategy::Full, |
| 775 | Some(synced) |
| 776 | if now.signed_duration_since(synced).num_seconds() >= FULL_RESYNC_INTERVAL_SECS => |
| 777 | { |
| 778 | SyncStrategy::Full |
| 779 | } |
| 780 | Some(synced) => SyncStrategy::Incremental { since: synced }, |
| 781 | }, |
| 782 | } |
| 783 | } |
| 784 | |
| 785 | /// Merge an incremental listing into the snapshot (in memory). Delta |
| 786 | /// entries replace by name; retired or no-longer-launchable servers are |
| 787 | /// dropped; everything else keeps its cached copy. |
| 788 | fn merge_incremental_entries( |
| 789 | base: &[McpRegistryServerEntry], |
| 790 | entries: Vec<RegistryServerEntry>, |
| 791 | ) -> Vec<McpRegistryServerEntry> { |
| 792 | let mut merged: HashMap<String, McpRegistryServerEntry> = base |
| 793 | .iter() |
| 794 | .map(|entry| (entry.name.clone(), entry.clone())) |
| 795 | .collect(); |
| 796 | for entry in entries { |
| 797 | let name = entry.server.name.clone(); |
| 798 | if matches!( |
| 799 | entry.lifecycle_status(), |
| 800 | Some("deleted") | Some("deprecated") |
| 801 | ) { |
| 802 | merged.remove(&name); |
| 803 | continue; |
| 804 | } |
| 805 | match server_to_entry(entry.server) { |
| 806 | Some(updated) => { |
| 807 | merged.insert(name, updated); |
| 808 | } |
| 809 | None => { |
| 810 | merged.remove(&name); |
| 811 | } |
| 812 | } |
| 813 | } |
| 814 | merged.into_values().collect() |
| 815 | } |
| 816 | |
| 817 | /// Fetch the index (full or incremental), assemble the next snapshot in |
| 818 | /// memory, and atomically replace the cache file. Retried on failure; |
| 819 | /// the old snapshot survives any failed sync. |
| 820 | async fn sync_once(path: &Path) -> Result<(), ToolError> { |
| 821 | let client = crate::tls::reqwest_client_builder() |
| 822 | .user_agent(USER_AGENT) |
| 823 | .timeout(std::time::Duration::from_secs(REQUEST_TIMEOUT_SECS)) |
| 824 | .build() |
| 825 | .map_err(|e| ToolError::execution_failed(format!("HTTP client: {e}")))?; |
| 826 | |
| 827 | let now = Utc::now(); |
| 828 | let cached = read_cache(path); |
| 829 | let strategy = sync_strategy(cached.as_ref(), now); |
| 830 | let updated_since = match strategy { |
| 831 | SyncStrategy::Full => None, |
| 832 | SyncStrategy::Incremental { since } => Some(since), |
| 833 | }; |
| 834 | |
| 835 | let mut last_error: Option<ToolError> = None; |
| 836 | let mut servers: Option<Vec<McpRegistryServerEntry>> = None; |
| 837 | for _attempt in 0..MAX_SYNC_ATTEMPTS { |
| 838 | match fetch_registry_entries(&client, updated_since).await { |
| 839 | Ok(entries) => { |
| 840 | servers = Some(match strategy { |
| 841 | SyncStrategy::Full => viable_entries(entries), |
| 842 | SyncStrategy::Incremental { .. } => merge_incremental_entries( |
| 843 | &cached.expect("incremental needs a cache").servers, |
| 844 | entries, |
| 845 | ), |
| 846 | }); |
| 847 | break; |
| 848 | } |
| 849 | Err(error) => last_error = Some(error), |
| 850 | } |
| 851 | } |
| 852 | let servers = servers.ok_or_else(|| { |
| 853 | last_error.unwrap_or_else(|| ToolError::execution_failed("Registry sync failed")) |
| 854 | })?; |
| 855 | let index = McpRegistryIndex { |
| 856 | version: MCP_REGISTRY_CACHE_VERSION, |
| 857 | count: servers.len(), |
| 858 | servers, |
| 859 | synced_at: Some(now), |
| 860 | }; |
| 861 | // Blocking filesystem work (including `write_atomic`'s publish-retry) |
| 862 | // runs on the blocking pool — tool handlers execute on the Tokio |
| 863 | // runtime (blocking-call convention, #6149). |
| 864 | let json_str = serde_json::to_string_pretty(&index) |
| 865 | .map_err(|e| ToolError::execution_failed(format!("Serialize: {e}")))?; |
| 866 | let path = path.to_path_buf(); |
| 867 | tokio::task::spawn_blocking(move || { |
| 868 | if let Some(parent) = path.parent() { |
| 869 | std::fs::create_dir_all(parent) |
| 870 | .map_err(|e| ToolError::execution_failed(format!("Create cache dir: {e}")))?; |
| 871 | } |
| 872 | write_atomic(&path, json_str.as_bytes()) |
| 873 | .map_err(|e| ToolError::execution_failed(format!("Write cache: {e}"))) |
| 874 | }) |
| 875 | .await |
| 876 | .map_err(|e| ToolError::execution_failed(format!("Write cache task: {e}")))??; |
| 877 | Ok(()) |
| 878 | } |
| 879 | |
| 880 | #[async_trait::async_trait] |
| 881 | impl ToolSpec for McpSyncRegistry { |
| 882 | fn name(&self) -> &str { |
| 883 | "registry_sync" |
| 884 | } |
| 885 | |
| 886 | fn description(&self) -> &str { |
| 887 | "Search installable local MCP servers for a capability this session \ |
| 888 | does not already have, and return at most eight scored matches; the \ |
| 889 | full Registry index stays host-side. Describe the missing capability \ |
| 890 | in the query. The index contains only stdio packages that declare no \ |
| 891 | environment variables or API keys. Use this when an available tool, a \ |
| 892 | project script, or ordinary local code cannot do the job — not before \ |
| 893 | ordinary work. To use a match, call start_registry_mcp_server with its \ |
| 894 | exact name; do not run its package command through the shell." |
| 895 | } |
| 896 | |
| 897 | fn input_schema(&self) -> Value { |
| 898 | json!({ |
| 899 | "type": "object", |
| 900 | "properties": { |
| 901 | "query": { |
| 902 | "type": "string", |
| 903 | "description": "Required. The specialized capability to \ |
| 904 | search for, e.g. 'convert PDF to markdown' or 'postgres \ |
| 905 | database access'. Matched server names and descriptions \ |
| 906 | are scored host-side; at most eight matches return." |
| 907 | } |
| 908 | }, |
| 909 | "required": ["query"], |
| 910 | "additionalProperties": false |
| 911 | }) |
| 912 | } |
| 913 | |
| 914 | fn capabilities(&self) -> Vec<ToolCapability> { |
| 915 | vec![ToolCapability::Network] |
| 916 | } |
| 917 | |
| 918 | fn approval_requirement(&self) -> ApprovalRequirement { |
| 919 | ApprovalRequirement::Auto |
| 920 | } |
| 921 | |
| 922 | fn supports_parallel(&self) -> bool { |
| 923 | true |
| 924 | } |
| 925 | |
| 926 | async fn execute(&self, input: Value, _ctx: &ToolContext) -> Result<ToolResult, ToolError> { |
| 927 | let query = input |
| 928 | .get("query") |
| 929 | .and_then(Value::as_str) |
| 930 | .map(str::trim) |
| 931 | .filter(|query| !query.is_empty()) |
| 932 | .ok_or_else(|| { |
| 933 | ToolError::invalid_input( |
| 934 | "registry_sync requires a non-empty 'query' describing the \ |
| 935 | specialized capability to search for.", |
| 936 | ) |
| 937 | })?; |
| 938 | let path = self.cache_path()?; |
| 939 | let result = load_registry_catalog(&path, query).await?; |
| 940 | let json = serde_json::to_string(&result) |
| 941 | .map_err(|e| ToolError::execution_failed(format!("Serialize: {e}")))?; |
| 942 | Ok(ToolResult::success(json)) |
| 943 | } |
| 944 | } |
| 945 | |
| 946 | /// Start one zero-environment stdio server selected from the Registry cache. |
| 947 | /// The model supplies a Registry identity and structured CLI values; the host |
| 948 | /// owns command construction, so no arbitrary shell command or environment |
| 949 | /// channel is exposed by the discovery flow. |
| 950 | pub struct StartRegistryMcpServer { |
| 951 | pool: Arc<AsyncMutex<McpPool>>, |
| 952 | } |
| 953 | |
| 954 | impl StartRegistryMcpServer { |
| 955 | pub fn new(pool: Arc<AsyncMutex<McpPool>>) -> Self { |
| 956 | Self { pool } |
| 957 | } |
| 958 | } |
| 959 | |
| 960 | #[async_trait::async_trait] |
| 961 | impl ToolSpec for StartRegistryMcpServer { |
| 962 | fn name(&self) -> &str { |
| 963 | "start_registry_mcp_server" |
| 964 | } |
| 965 | |
| 966 | fn description(&self) -> &str { |
| 967 | "Install and start a local stdio MCP server previously returned by \ |
| 968 | registry_sync. Only Registry packages that declare no environment \ |
| 969 | variables are eligible. Pass the exact registry_name and, when the \ |
| 970 | discovery result lists required_args, provide their values in the \ |
| 971 | structured arguments object. The connected server's complete tool \ |
| 972 | schemas become callable in the same turn." |
| 973 | } |
| 974 | |
| 975 | fn input_schema(&self) -> Value { |
| 976 | json!({ |
| 977 | "type": "object", |
| 978 | "properties": { |
| 979 | "registry_name": { |
| 980 | "type": "string", |
| 981 | "description": "Exact server name returned by registry_sync" |
| 982 | }, |
| 983 | "arguments": { |
| 984 | "type": "object", |
| 985 | "additionalProperties": { "type": "string" }, |
| 986 | "description": "Values keyed by required_args[].name; omit when none are required" |
| 987 | } |
| 988 | }, |
| 989 | "required": ["registry_name"] |
| 990 | }) |
| 991 | } |
| 992 | |
| 993 | fn capabilities(&self) -> Vec<ToolCapability> { |
| 994 | vec![ToolCapability::Network, ToolCapability::ExecutesCode] |
| 995 | } |
| 996 | |
| 997 | fn approval_requirement(&self) -> ApprovalRequirement { |
| 998 | ApprovalRequirement::Required |
| 999 | } |
| 1000 | |
| 1001 | async fn execute(&self, input: Value, ctx: &ToolContext) -> Result<ToolResult, ToolError> { |
| 1002 | let registry_name = input |
| 1003 | .get("registry_name") |
| 1004 | .and_then(Value::as_str) |
| 1005 | .ok_or_else(|| ToolError::invalid_input("missing required field: registry_name"))?; |
| 1006 | let supplied: HashMap<String, String> = match input.get("arguments") { |
| 1007 | Some(value) => serde_json::from_value(value.clone()) |
| 1008 | .map_err(|error| ToolError::invalid_input(format!("invalid arguments: {error}")))?, |
| 1009 | None => HashMap::new(), |
| 1010 | }; |
| 1011 | |
| 1012 | let path = cache_path()?; |
| 1013 | let cache = read_cache(&path).ok_or_else(|| { |
| 1014 | ToolError::execution_failed(format!( |
| 1015 | "no current Registry cache at {}; run registry_sync first", |
| 1016 | path.display() |
| 1017 | )) |
| 1018 | })?; |
| 1019 | if cache.version != MCP_REGISTRY_CACHE_VERSION { |
| 1020 | return Err(ToolError::execution_failed(format!( |
| 1021 | "cache at {} is from an older schema version; run registry_sync first", |
| 1022 | path.display() |
| 1023 | ))); |
| 1024 | } |
| 1025 | let entry = cache |
| 1026 | .servers |
| 1027 | .iter() |
| 1028 | .find(|server| server.name == registry_name) |
| 1029 | .ok_or_else(|| ToolError::invalid_input("registry_name is not present in the cache"))?; |
| 1030 | |
| 1031 | let expected: HashSet<&str> = entry |
| 1032 | .launch |
| 1033 | .required_args |
| 1034 | .iter() |
| 1035 | .map(|arg| arg.name.as_str()) |
| 1036 | .collect(); |
| 1037 | if let Some(unknown) = supplied |
| 1038 | .keys() |
| 1039 | .find(|name| !expected.contains(name.as_str())) |
| 1040 | { |
| 1041 | return Err(ToolError::invalid_input(format!( |
| 1042 | "unknown argument '{unknown}' for {registry_name}" |
| 1043 | ))); |
| 1044 | } |
| 1045 | |
| 1046 | let mut rendered_args = Vec::new(); |
| 1047 | for argument in &entry.launch.required_args { |
| 1048 | let value = supplied |
| 1049 | .get(&argument.name) |
| 1050 | .cloned() |
| 1051 | .or_else(|| argument.default.clone()) |
| 1052 | .ok_or_else(|| { |
| 1053 | ToolError::invalid_input(format!( |
| 1054 | "missing required argument '{}' for {registry_name}", |
| 1055 | argument.name |
| 1056 | )) |
| 1057 | })?; |
| 1058 | if matches!(argument.kind.as_deref(), Some("named")) && !argument.name.is_empty() { |
| 1059 | rendered_args.push(shell_words::quote(&argument.name).into_owned()); |
| 1060 | } |
| 1061 | rendered_args.push(shell_words::quote(&value).into_owned()); |
| 1062 | } |
| 1063 | |
| 1064 | let command = entry |
| 1065 | .launch |
| 1066 | .run_command |
| 1067 | .replace("<ARGS>", &rendered_args.join(" ")); |
| 1068 | let delegated = json!({ |
| 1069 | "server": command.trim(), |
| 1070 | "name": registry_name, |
| 1071 | // Registry packages cold-start through npx/uvx downloads; the |
| 1072 | // 10s default connect budget is routinely exceeded on first |
| 1073 | // launch. This override is host-supplied only — it is not part |
| 1074 | // of the model-facing schema of either tool. |
| 1075 | "connect_timeout": REGISTRY_CONNECT_TIMEOUT_SECS, |
| 1076 | }); |
| 1077 | crate::tools::runtime_mcp::StartRuntimeMcpServer::new(Arc::clone(&self.pool)) |
| 1078 | .execute(delegated, ctx) |
| 1079 | .await |
| 1080 | } |
| 1081 | } |
| 1082 | |
| 1083 | #[cfg(test)] |
| 1084 | mod tests { |
| 1085 | use super::*; |
| 1086 | |
| 1087 | #[test] |
| 1088 | fn server_no_packages_filtered() { |
| 1089 | let server = RegistryServer { |
| 1090 | name: "test".into(), |
| 1091 | description: "desc".into(), |
| 1092 | packages: None, |
| 1093 | }; |
| 1094 | assert!(server_to_entry(server).is_none()); |
| 1095 | } |
| 1096 | |
| 1097 | #[test] |
| 1098 | fn server_remote_only_filtered() { |
| 1099 | let server = RegistryServer { |
| 1100 | name: "test".into(), |
| 1101 | description: "desc".into(), |
| 1102 | packages: Some(vec![RegistryPackage { |
| 1103 | registry_type: "npm".into(), |
| 1104 | identifier: "@test/pkg".into(), |
| 1105 | version: Some("1.0.0".into()), |
| 1106 | runtime_hint: Some("npx".into()), |
| 1107 | transport: Some("streamable-http".into()), |
| 1108 | package_arguments: vec![], |
| 1109 | runtime_arguments: vec![], |
| 1110 | environment_variables: Value::Null, |
| 1111 | }]), |
| 1112 | }; |
| 1113 | assert!(server_to_entry(server).is_none()); |
| 1114 | } |
| 1115 | |
| 1116 | #[test] |
| 1117 | fn server_stdio_kept() { |
| 1118 | let server = RegistryServer { |
| 1119 | name: "test".into(), |
| 1120 | description: "desc".into(), |
| 1121 | packages: Some(vec![RegistryPackage { |
| 1122 | registry_type: "npm".into(), |
| 1123 | identifier: "@test/pkg".into(), |
| 1124 | version: Some("1.0.0".into()), |
| 1125 | runtime_hint: Some("npx".into()), |
| 1126 | transport: Some("stdio".into()), |
| 1127 | package_arguments: vec![], |
| 1128 | runtime_arguments: vec!["-y".into()], |
| 1129 | environment_variables: Value::Null, |
| 1130 | }]), |
| 1131 | }; |
| 1132 | let entry = server_to_entry(server).unwrap(); |
| 1133 | assert_eq!(entry.launch.run_command, "npx -y @test/pkg@1.0.0 <ARGS>"); |
| 1134 | } |
| 1135 | |
| 1136 | #[test] |
| 1137 | fn server_declaring_environment_variables_is_filtered() { |
| 1138 | let server = RegistryServer { |
| 1139 | name: "needs-secret".into(), |
| 1140 | description: "requires an API key".into(), |
| 1141 | packages: Some(vec![RegistryPackage { |
| 1142 | registry_type: "npm".into(), |
| 1143 | identifier: "@test/secret-server".into(), |
| 1144 | version: Some("1.0.0".into()), |
| 1145 | runtime_hint: Some("npx".into()), |
| 1146 | transport: Some("stdio".into()), |
| 1147 | package_arguments: vec![], |
| 1148 | runtime_arguments: vec!["-y".into()], |
| 1149 | environment_variables: json!([{ "name": "API_KEY", "isRequired": true }]), |
| 1150 | }]), |
| 1151 | }; |
| 1152 | assert!(server_to_entry(server).is_none()); |
| 1153 | } |
| 1154 | |
| 1155 | #[test] |
| 1156 | fn fixed_positional_package_arguments_render_into_run_command() { |
| 1157 | // Regression for the agentic-mermaid launch failure: upstream |
| 1158 | // declares `packageArguments: [{"value": "mcp", "type": |
| 1159 | // "positional"}]` (no `isRequired`), and dropping that token |
| 1160 | // produced `npx -y agentic-mermaid@0.1.2 <ARGS>` — which starts |
| 1161 | // the package's default entrypoint, not the MCP server. |
| 1162 | let server = RegistryServer { |
| 1163 | name: "io.github.adewale/agentic-mermaid".into(), |
| 1164 | description: "Render Mermaid diagrams through MCP.".into(), |
| 1165 | packages: Some(vec![RegistryPackage { |
| 1166 | registry_type: "npm".into(), |
| 1167 | identifier: "agentic-mermaid".into(), |
| 1168 | version: Some("0.1.2".into()), |
| 1169 | runtime_hint: Some("npx".into()), |
| 1170 | transport: Some("stdio".into()), |
| 1171 | package_arguments: vec![RegistryArg { |
| 1172 | name: None, |
| 1173 | description: None, |
| 1174 | is_required: false, |
| 1175 | kind: Some("positional".into()), |
| 1176 | value: Some("mcp".into()), |
| 1177 | default: None, |
| 1178 | }], |
| 1179 | runtime_arguments: vec!["-y".into()], |
| 1180 | environment_variables: Value::Null, |
| 1181 | }]), |
| 1182 | }; |
| 1183 | let entry = server_to_entry(server).unwrap(); |
| 1184 | assert_eq!( |
| 1185 | entry.launch.run_command, |
| 1186 | "npx -y agentic-mermaid@0.1.2 mcp <ARGS>" |
| 1187 | ); |
| 1188 | assert!(entry.launch.required_args.is_empty()); |
| 1189 | } |
| 1190 | |
| 1191 | #[test] |
| 1192 | fn placeholder_positional_package_argument_stays_in_required_args() { |
| 1193 | // The flip side: a positional arg with neither `value` nor |
| 1194 | // `default` is user input (e.g. an allowed directory), not a |
| 1195 | // fixed token — it must NOT leak into the rendered command. |
| 1196 | let server = RegistryServer { |
| 1197 | name: "test".into(), |
| 1198 | description: "desc".into(), |
| 1199 | packages: Some(vec![RegistryPackage { |
| 1200 | registry_type: "npm".into(), |
| 1201 | identifier: "@test/fs".into(), |
| 1202 | version: Some("1.0.0".into()), |
| 1203 | runtime_hint: Some("npx".into()), |
| 1204 | transport: Some("stdio".into()), |
| 1205 | package_arguments: vec![RegistryArg { |
| 1206 | name: None, |
| 1207 | description: Some("Directory to expose".into()), |
| 1208 | is_required: true, |
| 1209 | kind: Some("positional".into()), |
| 1210 | value: None, |
| 1211 | default: None, |
| 1212 | }], |
| 1213 | runtime_arguments: vec!["-y".into()], |
| 1214 | environment_variables: Value::Null, |
| 1215 | }]), |
| 1216 | }; |
| 1217 | let entry = server_to_entry(server).unwrap(); |
| 1218 | assert_eq!(entry.launch.run_command, "npx -y @test/fs@1.0.0 <ARGS>"); |
| 1219 | assert_eq!(entry.launch.required_args.len(), 1); |
| 1220 | } |
| 1221 | |
| 1222 | #[test] |
| 1223 | fn fixed_argument_with_spaces_preserves_one_process_argument() { |
| 1224 | let command = build_run_command( |
| 1225 | "npx", |
| 1226 | "@test/fs", |
| 1227 | "1.0.0", |
| 1228 | &[], |
| 1229 | &[RegistryArg { |
| 1230 | name: None, |
| 1231 | description: None, |
| 1232 | is_required: false, |
| 1233 | kind: Some("positional".into()), |
| 1234 | value: Some("/tmp/a folder".into()), |
| 1235 | default: None, |
| 1236 | }], |
| 1237 | ); |
| 1238 | let parsed = shell_words::split(command.replace("<ARGS>", "").trim()).unwrap(); |
| 1239 | assert_eq!(parsed.last().map(String::as_str), Some("/tmp/a folder")); |
| 1240 | } |
| 1241 | |
| 1242 | #[test] |
| 1243 | fn server_without_explicit_stdio_transport_is_filtered() { |
| 1244 | let server = RegistryServer { |
| 1245 | name: "test".into(), |
| 1246 | description: "desc".into(), |
| 1247 | packages: Some(vec![RegistryPackage { |
| 1248 | registry_type: "npm".into(), |
| 1249 | identifier: "@test/pkg".into(), |
| 1250 | version: Some("1.0.0".into()), |
| 1251 | runtime_hint: Some("npx".into()), |
| 1252 | transport: None, |
| 1253 | package_arguments: vec![], |
| 1254 | runtime_arguments: vec![], |
| 1255 | environment_variables: Value::Null, |
| 1256 | }]), |
| 1257 | }; |
| 1258 | assert!(server_to_entry(server).is_none()); |
| 1259 | } |
| 1260 | |
| 1261 | #[test] |
| 1262 | fn unsupported_registry_runtime_is_not_advertised() { |
| 1263 | let server = RegistryServer { |
| 1264 | name: "container-only".into(), |
| 1265 | description: "OCI stdio server".into(), |
| 1266 | packages: Some(vec![RegistryPackage { |
| 1267 | registry_type: "oci".into(), |
| 1268 | identifier: "docker.io/example/server:1.0.0".into(), |
| 1269 | version: None, |
| 1270 | runtime_hint: Some("docker".into()), |
| 1271 | transport: Some("stdio".into()), |
| 1272 | package_arguments: vec![], |
| 1273 | runtime_arguments: vec![], |
| 1274 | environment_variables: Value::Null, |
| 1275 | }]), |
| 1276 | }; |
| 1277 | assert!(server_to_entry(server).is_none()); |
| 1278 | } |
| 1279 | |
| 1280 | #[test] |
| 1281 | fn registry_type_and_runtime_must_match() { |
| 1282 | let server = RegistryServer { |
| 1283 | name: "mismatched".into(), |
| 1284 | description: "invalid npm runner".into(), |
| 1285 | packages: Some(vec![RegistryPackage { |
| 1286 | registry_type: "npm".into(), |
| 1287 | identifier: "example".into(), |
| 1288 | version: Some("1.0.0".into()), |
| 1289 | runtime_hint: Some("uvx".into()), |
| 1290 | transport: Some("stdio".into()), |
| 1291 | package_arguments: vec![], |
| 1292 | runtime_arguments: vec![], |
| 1293 | environment_variables: Value::Null, |
| 1294 | }]), |
| 1295 | }; |
| 1296 | assert!(server_to_entry(server).is_none()); |
| 1297 | } |
| 1298 | |
| 1299 | /// End-to-end smoke test: cold-start `McpSyncRegistry::execute()` |
| 1300 | /// against the live Registry, wait for the background download to |
| 1301 | /// land, then verify the cache file and the next payload. |
| 1302 | /// |
| 1303 | /// Ignored: needs network + minutes of wall clock (page pacing, flaky |
| 1304 | /// upstream). Run manually with: |
| 1305 | /// cargo test -p codewhale-tui --bin codewhale-tui --locked \ |
| 1306 | /// execute_writes_cache_file_and_returns_summary -- --ignored --nocapture |
| 1307 | /// |
| 1308 | /// The cache path is injected into a tempdir so the real cache is |
| 1309 | /// untouched (works on every platform). |
| 1310 | #[tokio::test] |
| 1311 | #[ignore = "requires network access to the public MCP Registry; \ |
| 1312 | run with `cargo test -- --ignored`"] |
| 1313 | async fn execute_writes_cache_file_and_returns_summary() { |
| 1314 | use crate::tools::spec::ToolContext; |
| 1315 | |
| 1316 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1317 | // Persist the tempdir so we can inspect the cache file after the |
| 1318 | // test returns. `TempDir::keep` (newer API) disables Drop's |
| 1319 | // cleanup so the directory leaks — acceptable for a manual-run |
| 1320 | // integration smoke test that intentionally outlives its scope. |
| 1321 | let tmp_path = tmp.keep(); |
| 1322 | let cache_path = tmp_path.join(".codewhale").join("mcp-index.json"); |
| 1323 | |
| 1324 | let ctx = ToolContext::new(tmp_path.clone()); |
| 1325 | |
| 1326 | let input = json!({ "query": "filesystem file access" }); |
| 1327 | |
| 1328 | let result = McpSyncRegistry::with_cache_path(cache_path.clone()) |
| 1329 | .execute(input, &ctx) |
| 1330 | .await |
| 1331 | .expect("execute() should not error against the live Registry"); |
| 1332 | |
| 1333 | assert!( |
| 1334 | result.success, |
| 1335 | "execute returned non-success: content={}", |
| 1336 | result.content |
| 1337 | ); |
| 1338 | // Cold start: empty catalog returned immediately, download in the |
| 1339 | // background. |
| 1340 | let first_payload: serde_json::Value = |
| 1341 | serde_json::from_str(&result.content).expect("result content must parse"); |
| 1342 | assert_eq!(first_payload["total"], 0, "no cache ⇒ empty catalog"); |
| 1343 | |
| 1344 | // Poll for the background download to land, then assert on the |
| 1345 | // final snapshot (page pacing + upstream stalls take minutes). |
| 1346 | let deadline = std::time::Instant::now() + std::time::Duration::from_secs(600); |
| 1347 | let cache = loop { |
| 1348 | if let Ok(raw) = std::fs::read_to_string(&cache_path) |
| 1349 | && let Ok(cache) = serde_json::from_str::<McpRegistryIndex>(&raw) |
| 1350 | { |
| 1351 | break cache; |
| 1352 | } |
| 1353 | assert!( |
| 1354 | std::time::Instant::now() < deadline, |
| 1355 | "cache file did not appear at {:?} within 600s", |
| 1356 | cache_path |
| 1357 | ); |
| 1358 | tokio::time::sleep(std::time::Duration::from_millis(500)).await; |
| 1359 | }; |
| 1360 | |
| 1361 | // The Registry has hundreds of stdio servers as of 2026; expect at |
| 1362 | // least 1 from a single page. If this fails the upstream either |
| 1363 | // removed all stdio entries or our filter is wrong. |
| 1364 | assert!( |
| 1365 | cache.count > 0, |
| 1366 | "expected at least 1 stdio server, got count={}", |
| 1367 | cache.count |
| 1368 | ); |
| 1369 | assert_eq!( |
| 1370 | cache.servers.len(), |
| 1371 | cache.count, |
| 1372 | "cache.count must equal cache.servers.len()" |
| 1373 | ); |
| 1374 | for entry in &cache.servers { |
| 1375 | assert!(!entry.name.is_empty(), "server entry has empty name"); |
| 1376 | assert!( |
| 1377 | entry.launch.run_command.ends_with("<ARGS>"), |
| 1378 | "kept entry {} run_command should end with <ARGS>; got: {}", |
| 1379 | entry.name, |
| 1380 | entry.launch.run_command |
| 1381 | ); |
| 1382 | } |
| 1383 | |
| 1384 | // After the background download lands, the next call serves the |
| 1385 | // fresh snapshot and its payload must match the cache file. |
| 1386 | let result = McpSyncRegistry::with_cache_path(cache_path.clone()) |
| 1387 | .execute(json!({ "query": "filesystem file access" }), &ctx) |
| 1388 | .await |
| 1389 | .expect("execute() should succeed once the cache is fresh"); |
| 1390 | let payload: serde_json::Value = |
| 1391 | serde_json::from_str(&result.content).expect("result content must parse"); |
| 1392 | assert_eq!(payload["total"].as_u64(), Some(cache.count as u64)); |
| 1393 | let matches = payload["servers"].as_array().map(Vec::len); |
| 1394 | assert!( |
| 1395 | matches.is_some_and(|len| len <= 8), |
| 1396 | "model-visible matches must stay bounded: {matches:?}" |
| 1397 | ); |
| 1398 | assert!( |
| 1399 | matches.is_some_and(|len| len >= 1), |
| 1400 | "the filesystem fixture must match the filesystem query" |
| 1401 | ); |
| 1402 | } |
| 1403 | |
| 1404 | fn make_test_cache() -> McpRegistryIndex { |
| 1405 | let server = McpRegistryServerEntry { |
| 1406 | name: "io.modelcontextprotocol/filesystem".into(), |
| 1407 | description: "Read/write local files with sandboxed paths".into(), |
| 1408 | launch: McpLaunchSpec { |
| 1409 | run_command: "npx -y @modelcontextprotocol/server-filesystem@1.0.0 <ARGS>".into(), |
| 1410 | required_args: vec![], |
| 1411 | }, |
| 1412 | }; |
| 1413 | McpRegistryIndex { |
| 1414 | version: MCP_REGISTRY_CACHE_VERSION, |
| 1415 | count: 1, |
| 1416 | servers: vec![server], |
| 1417 | synced_at: Some(Utc::now()), |
| 1418 | } |
| 1419 | } |
| 1420 | |
| 1421 | #[test] |
| 1422 | fn catalog_search_reports_total_and_returns_only_matches() { |
| 1423 | let cache = make_test_cache(); |
| 1424 | let catalog = catalog_from_cache(&cache, "filesystem"); |
| 1425 | assert_eq!(catalog.total, 1); |
| 1426 | assert_eq!(catalog.servers.len(), 1); |
| 1427 | assert_eq!( |
| 1428 | catalog.servers[0].name, |
| 1429 | "io.modelcontextprotocol/filesystem" |
| 1430 | ); |
| 1431 | assert_eq!( |
| 1432 | catalog.servers[0].description, |
| 1433 | "Read/write local files with sandboxed paths" |
| 1434 | ); |
| 1435 | // A query that does not match still reports the total, but returns |
| 1436 | // an empty match set — never the full catalog. |
| 1437 | let miss = catalog_from_cache(&cache, "database"); |
| 1438 | assert_eq!(miss.total, 1); |
| 1439 | assert!(miss.servers.is_empty()); |
| 1440 | } |
| 1441 | |
| 1442 | /// A scored row must not read as an obligation. The prompt this result |
| 1443 | /// carries used to say the model "must call start_registry_mcp_server ... |
| 1444 | /// before using shell commands, local programs, custom code, or a manual |
| 1445 | /// implementation", which is how a turn that only needed to write an HTML |
| 1446 | /// file and read a fixture ended up trying to start a browser server. |
| 1447 | #[test] |
| 1448 | fn result_prompt_does_not_oblige_starting_a_match() { |
| 1449 | let catalog = catalog_from_cache(&make_test_cache(), "filesystem"); |
| 1450 | let instruction = catalog.instruction; |
| 1451 | |
| 1452 | assert!(!instruction.contains("must call")); |
| 1453 | assert!(!instruction.contains("custom code")); |
| 1454 | assert!(!instruction.contains("manual implementation")); |
| 1455 | |
| 1456 | // What the result is still responsible for saying: the matches are |
| 1457 | // only worth starting for a capability the session lacks, and a |
| 1458 | // package is started through the approved host path, never the shell. |
| 1459 | assert!(instruction.contains("a capability you do not already have")); |
| 1460 | assert!(instruction.contains("start_registry_mcp_server with its exact name")); |
| 1461 | assert!(instruction.contains("never install or run its package command through the shell")); |
| 1462 | } |
| 1463 | |
| 1464 | /// Parse one `ServerResponse` JSON object the way the upstream list |
| 1465 | /// endpoint ships it. Lifecycle status travels in registry-managed |
| 1466 | /// `_meta` (`io.modelcontextprotocol.registry/official`), as a |
| 1467 | /// SIBLING of `server` — not inside the server body — and this |
| 1468 | /// helper keeps the tests honest about that path. |
| 1469 | /// <https://github.com/modelcontextprotocol/registry/blob/main/docs/reference/api/openapi.yaml> |
| 1470 | fn parse_server_entry(server_json: Value, status: Option<&str>) -> RegistryServerEntry { |
| 1471 | let mut response = json!({ "server": server_json }); |
| 1472 | if let Some(status) = status { |
| 1473 | response["_meta"] = |
| 1474 | json!({ "io.modelcontextprotocol.registry/official": { "status": status } }); |
| 1475 | } |
| 1476 | serde_json::from_value(response).expect("ServerResponse must deserialize") |
| 1477 | } |
| 1478 | |
| 1479 | #[test] |
| 1480 | fn viable_entries_keeps_only_active_launchable_servers() { |
| 1481 | let launchable = |name: &str, status: Option<&str>| { |
| 1482 | parse_server_entry( |
| 1483 | json!({ |
| 1484 | "name": name, |
| 1485 | "description": "d", |
| 1486 | "packages": [{ |
| 1487 | "registryType": "npm", |
| 1488 | "identifier": "@test/pkg", |
| 1489 | "version": "1.0.0", |
| 1490 | "runtimeHint": "npx", |
| 1491 | "transport": "stdio", |
| 1492 | "packageArguments": [], |
| 1493 | "runtimeArguments": [], |
| 1494 | "environmentVariables": null |
| 1495 | }] |
| 1496 | }), |
| 1497 | status, |
| 1498 | ) |
| 1499 | }; |
| 1500 | let entries = vec![ |
| 1501 | launchable("a/active", Some("active")), |
| 1502 | launchable("b/deprecated", Some("deprecated")), |
| 1503 | launchable("c/deleted", Some("deleted")), |
| 1504 | // No `_meta` at all: treated as active. |
| 1505 | launchable("d/no-meta", None), |
| 1506 | // Missing a launchable package: dropped by the launcher filter. |
| 1507 | parse_server_entry(json!({ "name": "e/no-pkg", "description": "d" }), None), |
| 1508 | ]; |
| 1509 | |
| 1510 | let viable = viable_entries(entries); |
| 1511 | |
| 1512 | let names: Vec<&str> = viable.iter().map(|s| s.name.as_str()).collect(); |
| 1513 | assert_eq!(names, ["a/active", "d/no-meta"]); |
| 1514 | } |
| 1515 | |
| 1516 | /// Minimal cache entry — only `name`/`description`/launch matter to the |
| 1517 | /// catalog conversion tests. |
| 1518 | fn cache_entry(name: &str, description: &str) -> McpRegistryServerEntry { |
| 1519 | McpRegistryServerEntry { |
| 1520 | name: name.into(), |
| 1521 | description: description.into(), |
| 1522 | launch: McpLaunchSpec { |
| 1523 | run_command: "npx pkg@1.0.0 <ARGS>".into(), |
| 1524 | required_args: vec![], |
| 1525 | }, |
| 1526 | } |
| 1527 | } |
| 1528 | |
| 1529 | #[test] |
| 1530 | fn catalog_search_caps_model_visible_matches() { |
| 1531 | let servers = (0..12) |
| 1532 | .map(|index| cache_entry(&format!("example/file-{index}"), "file server")) |
| 1533 | .collect::<Vec<_>>(); |
| 1534 | let cache = McpRegistryIndex { |
| 1535 | version: MCP_REGISTRY_CACHE_VERSION, |
| 1536 | count: servers.len(), |
| 1537 | servers, |
| 1538 | synced_at: Some(Utc::now()), |
| 1539 | }; |
| 1540 | let result = catalog_from_cache(&cache, "file"); |
| 1541 | assert_eq!(result.total, 12); |
| 1542 | assert!( |
| 1543 | result.servers.len() <= MAX_REGISTRY_MATCHES, |
| 1544 | "model-visible matches stay bounded" |
| 1545 | ); |
| 1546 | } |
| 1547 | |
| 1548 | #[test] |
| 1549 | fn cache_is_fresh_honors_ttl_version_and_missing_timestamp() { |
| 1550 | let now = Utc::now(); |
| 1551 | let base = make_test_cache(); |
| 1552 | |
| 1553 | // Within the window: fresh. |
| 1554 | let mut fresh = base.clone(); |
| 1555 | fresh.synced_at = Some(now - chrono::Duration::minutes(30)); |
| 1556 | assert!(cache_is_fresh(&fresh, now)); |
| 1557 | |
| 1558 | // At/over the incremental window boundary: stale. |
| 1559 | let mut expired = base.clone(); |
| 1560 | expired.synced_at = Some(now - chrono::Duration::seconds(INCREMENTAL_INTERVAL_SECS + 1)); |
| 1561 | assert!(!cache_is_fresh(&expired, now)); |
| 1562 | |
| 1563 | // Legacy cache without a timestamp: stale (triggers one resync). |
| 1564 | let mut legacy = base.clone(); |
| 1565 | legacy.synced_at = None; |
| 1566 | assert!(!cache_is_fresh(&legacy, now)); |
| 1567 | |
| 1568 | // Schema version mismatch: always stale. |
| 1569 | let mut old_schema = base.clone(); |
| 1570 | old_schema.version = MCP_REGISTRY_CACHE_VERSION - 1; |
| 1571 | old_schema.synced_at = Some(now); |
| 1572 | assert!(!cache_is_fresh(&old_schema, now)); |
| 1573 | |
| 1574 | // Clock skew: a future-stamped cache stays fresh. |
| 1575 | let mut future = base.clone(); |
| 1576 | future.synced_at = Some(now + chrono::Duration::hours(2)); |
| 1577 | assert!(cache_is_fresh(&future, now)); |
| 1578 | } |
| 1579 | |
| 1580 | #[test] |
| 1581 | fn catalog_for_snapshot_returns_empty_catalog_when_no_cache() { |
| 1582 | let result = catalog_for_snapshot(None, "anything"); |
| 1583 | assert_eq!(result.total, 0); |
| 1584 | assert!(result.servers.is_empty()); |
| 1585 | } |
| 1586 | |
| 1587 | #[test] |
| 1588 | fn catalog_for_snapshot_serves_cached_entries_without_flags() { |
| 1589 | let result = catalog_for_snapshot(Some(make_test_cache()), "filesystem"); |
| 1590 | assert_eq!(result.total, 1); |
| 1591 | assert_eq!(result.servers.len(), 1); |
| 1592 | } |
| 1593 | |
| 1594 | #[test] |
| 1595 | fn catalog_for_snapshot_serves_old_snapshot_as_is() { |
| 1596 | // Snapshot age is invisible to the model: every entry pins its own |
| 1597 | // package version, so even a month-old snapshot is served as-is. |
| 1598 | let mut cache = make_test_cache(); |
| 1599 | cache.synced_at = Some(Utc::now() - chrono::Duration::days(31)); |
| 1600 | let result = catalog_for_snapshot(Some(cache), "filesystem"); |
| 1601 | assert_eq!(result.total, 1); |
| 1602 | assert_eq!(result.servers.len(), 1); |
| 1603 | } |
| 1604 | |
| 1605 | #[test] |
| 1606 | fn sync_strategy_decides_full_vs_incremental() { |
| 1607 | let now = Utc::now(); |
| 1608 | |
| 1609 | // No cache at all: full. |
| 1610 | assert_eq!(sync_strategy(None, now), SyncStrategy::Full); |
| 1611 | |
| 1612 | // Legacy schema version: full. |
| 1613 | let mut old = make_test_cache(); |
| 1614 | old.version = MCP_REGISTRY_CACHE_VERSION - 1; |
| 1615 | assert_eq!(sync_strategy(Some(&old), now), SyncStrategy::Full); |
| 1616 | |
| 1617 | // Missing timestamp: full. |
| 1618 | let mut no_ts = make_test_cache(); |
| 1619 | no_ts.synced_at = None; |
| 1620 | assert_eq!(sync_strategy(Some(&no_ts), now), SyncStrategy::Full); |
| 1621 | |
| 1622 | // Older than the full-resync window: full (reconciliation — an |
| 1623 | // incremental delta cannot observe vanished servers). |
| 1624 | let mut ancient = make_test_cache(); |
| 1625 | ancient.synced_at = Some(now - chrono::Duration::days(31)); |
| 1626 | assert_eq!(sync_strategy(Some(&ancient), now), SyncStrategy::Full); |
| 1627 | |
| 1628 | // Past the incremental window but inside the full-resync window: |
| 1629 | // incremental from the last sync. |
| 1630 | let mut stale = make_test_cache(); |
| 1631 | stale.synced_at = Some(now - chrono::Duration::days(2)); |
| 1632 | assert_eq!( |
| 1633 | sync_strategy(Some(&stale), now), |
| 1634 | SyncStrategy::Incremental { |
| 1635 | since: stale.synced_at.expect("set above") |
| 1636 | } |
| 1637 | ); |
| 1638 | } |
| 1639 | |
| 1640 | /// One launchable delta entry (active stdio npm package); `status` |
| 1641 | /// None means no `_meta` (treated as active). |
| 1642 | fn delta_entry(name: &str, status: Option<&str>) -> RegistryServerEntry { |
| 1643 | parse_server_entry( |
| 1644 | json!({ |
| 1645 | "name": name, |
| 1646 | "description": "d", |
| 1647 | "packages": [{ |
| 1648 | "registryType": "npm", |
| 1649 | "identifier": "@test/pkg", |
| 1650 | "version": "2.0.0", |
| 1651 | "runtimeHint": "npx", |
| 1652 | "transport": "stdio", |
| 1653 | "packageArguments": [], |
| 1654 | "runtimeArguments": [], |
| 1655 | "environmentVariables": null |
| 1656 | }] |
| 1657 | }), |
| 1658 | status, |
| 1659 | ) |
| 1660 | } |
| 1661 | |
| 1662 | #[test] |
| 1663 | fn merge_incremental_entries_updates_inserts_and_removes() { |
| 1664 | let base = vec![ |
| 1665 | cache_entry("a/unchanged", "unchanged"), |
| 1666 | cache_entry("b/updated", "old description"), |
| 1667 | cache_entry("c/deleted", "will be removed"), |
| 1668 | cache_entry("d/deprecated", "will be removed"), |
| 1669 | cache_entry("e/unlaunchable", "will be removed"), |
| 1670 | ]; |
| 1671 | let entries = vec![ |
| 1672 | // Replace the cached copy by name. |
| 1673 | delta_entry("b/updated", Some("active")), |
| 1674 | // Insert a brand-new server. |
| 1675 | delta_entry("f/new", Some("active")), |
| 1676 | // Retired servers are dropped from the snapshot. |
| 1677 | delta_entry("c/deleted", Some("deleted")), |
| 1678 | delta_entry("d/deprecated", Some("deprecated")), |
| 1679 | // Present but no longer launchable (npm package whose runner |
| 1680 | // no longer matches): the stale cached copy is dropped too. |
| 1681 | parse_server_entry( |
| 1682 | json!({ |
| 1683 | "name": "e/unlaunchable", |
| 1684 | "description": "d", |
| 1685 | "packages": [{ |
| 1686 | "registryType": "npm", |
| 1687 | "identifier": "@test/pkg", |
| 1688 | "version": "2.0.0", |
| 1689 | "runtimeHint": "uvx", |
| 1690 | "transport": "stdio", |
| 1691 | "packageArguments": [], |
| 1692 | "runtimeArguments": [], |
| 1693 | "environmentVariables": null |
| 1694 | }] |
| 1695 | }), |
| 1696 | Some("active"), |
| 1697 | ), |
| 1698 | ]; |
| 1699 | |
| 1700 | let merged = merge_incremental_entries(&base, entries); |
| 1701 | let by_name: HashMap<&str, &McpRegistryServerEntry> = merged |
| 1702 | .iter() |
| 1703 | .map(|entry| (entry.name.as_str(), entry)) |
| 1704 | .collect(); |
| 1705 | |
| 1706 | assert_eq!(merged.len(), 3); |
| 1707 | assert!( |
| 1708 | by_name.contains_key("a/unchanged"), |
| 1709 | "entries absent from the delta must keep their cached copy" |
| 1710 | ); |
| 1711 | assert!( |
| 1712 | by_name.contains_key("f/new"), |
| 1713 | "a new server in the delta must be inserted" |
| 1714 | ); |
| 1715 | assert_eq!( |
| 1716 | by_name["b/updated"].description, "d", |
| 1717 | "an updated entry must replace the cached copy" |
| 1718 | ); |
| 1719 | assert!(!by_name.contains_key("c/deleted")); |
| 1720 | assert!(!by_name.contains_key("d/deprecated")); |
| 1721 | assert!( |
| 1722 | !by_name.contains_key("e/unlaunchable"), |
| 1723 | "an entry that lost its launchable package must be dropped" |
| 1724 | ); |
| 1725 | } |
| 1726 | |
| 1727 | /// The sync permit is exclusive while held (no duplicate background |
| 1728 | /// downloads) and reusable after release (failed syncs get retried). |
| 1729 | /// Direct acceptance: a 4,786-entry fixture must never expose more |
| 1730 | /// than eight model-visible matches, and the serialized payload stays |
| 1731 | /// bounded regardless of catalog size. |
| 1732 | #[test] |
| 1733 | fn huge_catalog_search_stays_bounded_at_eight_matches() { |
| 1734 | let entries = (0..4_786) |
| 1735 | .map(|index| McpRegistryServerEntry { |
| 1736 | name: format!("com.example/server-{index}"), |
| 1737 | description: format!("Specialized capability server number {index} for conversion"), |
| 1738 | launch: McpLaunchSpec { |
| 1739 | run_command: "npx -y com.example/server <ARGS>".into(), |
| 1740 | required_args: vec![], |
| 1741 | }, |
| 1742 | }) |
| 1743 | .collect::<Vec<_>>(); |
| 1744 | |
| 1745 | let matches = search_registry_entries(&entries, "conversion server", MAX_REGISTRY_MATCHES); |
| 1746 | assert_eq!(matches.len(), MAX_REGISTRY_MATCHES, "cap at eight matches"); |
| 1747 | |
| 1748 | let payload = serde_json::to_string(&matches).expect("serialize matches"); |
| 1749 | assert!( |
| 1750 | payload.len() < 16_000, |
| 1751 | "bounded payload, got {} bytes", |
| 1752 | payload.len() |
| 1753 | ); |
| 1754 | |
| 1755 | // A query that matches nothing returns an empty set — never the |
| 1756 | // whole catalog. |
| 1757 | let none = search_registry_entries(&entries, "nothing matches this", MAX_REGISTRY_MATCHES); |
| 1758 | assert!(none.is_empty()); |
| 1759 | } |
| 1760 | |
| 1761 | #[test] |
| 1762 | fn search_ranks_exact_name_over_substring_and_ties_break_alphabetically() { |
| 1763 | let entries = [ |
| 1764 | entry("b/convert", "converts documents"), |
| 1765 | entry("a/convert-pro", "converts documents better"), |
| 1766 | entry( |
| 1767 | "zzz-unrelated", |
| 1768 | "but mentions convert deep in a long description", |
| 1769 | ), |
| 1770 | ]; |
| 1771 | let matches = search_registry_entries(&entries, "convert", MAX_REGISTRY_MATCHES); |
| 1772 | let names = matches.iter().map(|m| m.name.as_str()).collect::<Vec<_>>(); |
| 1773 | assert_eq!(names, vec!["a/convert-pro", "b/convert", "zzz-unrelated"]); |
| 1774 | } |
| 1775 | |
| 1776 | #[test] |
| 1777 | fn empty_query_matches_nothing_instead_of_everything() { |
| 1778 | let entries = [entry("a/server", "does things")]; |
| 1779 | assert!(search_registry_entries(&entries, " ", MAX_REGISTRY_MATCHES).is_empty()); |
| 1780 | } |
| 1781 | |
| 1782 | fn entry(name: &str, description: &str) -> McpRegistryServerEntry { |
| 1783 | McpRegistryServerEntry { |
| 1784 | name: name.into(), |
| 1785 | description: description.into(), |
| 1786 | launch: McpLaunchSpec { |
| 1787 | run_command: "npx -y pkg <ARGS>".into(), |
| 1788 | required_args: vec![], |
| 1789 | }, |
| 1790 | } |
| 1791 | } |
| 1792 | |
| 1793 | #[tokio::test] |
| 1794 | async fn background_sync_permit_is_exclusive_until_released() { |
| 1795 | let first = try_acquire_sync_permit(); |
| 1796 | assert!(first.is_some(), "first acquisition must succeed"); |
| 1797 | assert!( |
| 1798 | try_acquire_sync_permit().is_none(), |
| 1799 | "second acquisition must fail while the first is held" |
| 1800 | ); |
| 1801 | drop(first); |
| 1802 | let again = try_acquire_sync_permit(); |
| 1803 | assert!(again.is_some(), "permit must be reusable after release"); |
| 1804 | drop(again); |
| 1805 | } |
| 1806 | |
| 1807 | /// With a fresh snapshot on disk, `registry_sync` must serve it |
| 1808 | /// without touching the network: the fixture is a single synthetic |
| 1809 | /// server, so any live sync would return a different catalog and fail |
| 1810 | /// the assertion. The cache path is injected explicitly because |
| 1811 | /// `dirs::home_dir()` resolves the OS profile directory on Windows via |
| 1812 | /// SHGetKnownFolderPath — no environment variable can redirect it. |
| 1813 | #[tokio::test] |
| 1814 | async fn fresh_cache_serves_catalog_without_network() { |
| 1815 | use crate::tools::spec::ToolContext; |
| 1816 | |
| 1817 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1818 | let cache_dir = tmp.path().join(".codewhale"); |
| 1819 | std::fs::create_dir_all(&cache_dir).expect("create cache dir"); |
| 1820 | let cache_file = cache_dir.join("mcp-index.json"); |
| 1821 | let index = make_test_cache(); |
| 1822 | std::fs::write( |
| 1823 | &cache_file, |
| 1824 | serde_json::to_string_pretty(&index).expect("serialize fixture"), |
| 1825 | ) |
| 1826 | .expect("write fixture cache"); |
| 1827 | |
| 1828 | let ctx = ToolContext::new(tmp.path().to_path_buf()); |
| 1829 | let result = McpSyncRegistry::with_cache_path(cache_file) |
| 1830 | .execute(json!({ "query": "filesystem" }), &ctx) |
| 1831 | .await |
| 1832 | .expect("fresh cache must serve without network"); |
| 1833 | |
| 1834 | let payload: serde_json::Value = |
| 1835 | serde_json::from_str(&result.content).expect("result content must parse"); |
| 1836 | assert_eq!(payload["total"], 1, "fixture catalog is reported in full"); |
| 1837 | assert_eq!( |
| 1838 | payload["servers"][0]["name"], "io.modelcontextprotocol/filesystem", |
| 1839 | "cache-first path must return the cached entry, not a live sync" |
| 1840 | ); |
| 1841 | } |
| 1842 | |
| 1843 | /// Manual smoke run: cold-start against the live Registry, wait for |
| 1844 | /// the background download, then print the final payload to stdout. |
| 1845 | /// Run with: |
| 1846 | /// cargo test -p codewhale-tui --bin codewhale-tui --locked \ |
| 1847 | /// execute_and_print_catalog_for_manual_inspection -- --ignored --nocapture |
| 1848 | #[tokio::test] |
| 1849 | #[ignore = "manual smoke run; prints the tool result to stdout \ |
| 1850 | (requires network access to registry.modelcontextprotocol.io)"] |
| 1851 | // The module tree denies `print_stderr` (scroll-demon guard, #1085) so |
| 1852 | // TUI runtime code can never leak into ratatui's buffer. This test is |
| 1853 | // the deliberate exception: it only runs manually (`--ignored`) and its |
| 1854 | // entire purpose is printing the payload for operator inspection. |
| 1855 | #[allow(clippy::print_stderr)] |
| 1856 | async fn execute_and_print_catalog_for_manual_inspection() { |
| 1857 | use crate::tools::spec::ToolContext; |
| 1858 | |
| 1859 | let tmp = tempfile::tempdir().expect("tempdir"); |
| 1860 | // Persist the tempdir past the test so the cache file survives and |
| 1861 | // can be inspected from the shell after the test returns. |
| 1862 | let tmp_path = tmp.keep(); |
| 1863 | let cache_path = tmp_path.join(".codewhale").join("mcp-index.json"); |
| 1864 | |
| 1865 | let ctx = ToolContext::new(tmp_path.clone()); |
| 1866 | |
| 1867 | let input = json!({ "query": "filesystem file access" }); |
| 1868 | |
| 1869 | let result = McpSyncRegistry::with_cache_path(cache_path.clone()) |
| 1870 | .execute(input, &ctx) |
| 1871 | .await |
| 1872 | .expect("execute() should not error against the live Registry"); |
| 1873 | |
| 1874 | // The cold-start call returns immediately (empty catalog + |
| 1875 | // background download). Wait for the download to land, then call |
| 1876 | // again and print the final payload the model would receive. |
| 1877 | eprintln!("cold-start payload (background download flagged):"); |
| 1878 | eprintln!("{}", result.content); |
| 1879 | eprintln!("waiting for the background download to land..."); |
| 1880 | let deadline = std::time::Instant::now() + std::time::Duration::from_secs(600); |
| 1881 | while !cache_path.exists() { |
| 1882 | assert!( |
| 1883 | std::time::Instant::now() < deadline, |
| 1884 | "cache file did not appear at {:?} within 600s", |
| 1885 | cache_path |
| 1886 | ); |
| 1887 | tokio::time::sleep(std::time::Duration::from_millis(500)).await; |
| 1888 | } |
| 1889 | |
| 1890 | let result = McpSyncRegistry::with_cache_path(cache_path.clone()) |
| 1891 | .execute(json!({}), &ctx) |
| 1892 | .await |
| 1893 | .expect("execute() should succeed once the cache is fresh"); |
| 1894 | |
| 1895 | eprintln!("\n=== registry_sync output ==="); |
| 1896 | eprintln!("tool: registry_sync"); |
| 1897 | eprintln!( |
| 1898 | "status: {}", |
| 1899 | if result.success { "ok" } else { "fail" } |
| 1900 | ); |
| 1901 | eprintln!("cache_path: {}", cache_path.display()); |
| 1902 | eprintln!("cache_exists: {}", cache_path.exists()); |
| 1903 | if cache_path.exists() { |
| 1904 | match std::fs::metadata(&cache_path) { |
| 1905 | Ok(meta) => eprintln!("cache_size_bytes: {}", meta.len()), |
| 1906 | Err(e) => eprintln!("cache_stat_error: {e}"), |
| 1907 | } |
| 1908 | } |
| 1909 | eprintln!("--- catalog payload (what the model sees) ---"); |
| 1910 | eprintln!("{}", result.content); |
| 1911 | eprintln!("=== end ===\n"); |
| 1912 | } |
| 1913 | } |
| 1914 |