| 1 | //! Persistence for locally-added marketplace catalogs (#5311). |
| 2 | //! |
| 3 | //! Catalogs live in one sibling JSON file next to the plugin registry's |
| 4 | //! `state.json`, under the same Codewhale-owned `plugins/` root, and reuse the |
| 5 | //! registry's audited persistence machinery: private parent directory, atomic |
| 6 | //! temp-file publication, no-follow opens, and an `fd_lock` sidecar lock so a |
| 7 | //! concurrent TUI cannot interleave `add`/`remove` with a read. |
| 8 | //! |
| 9 | //! This is deliberately not a second database — it is the same store pattern |
| 10 | //! with a different schema, and it hard-fails closed on a malformed file |
| 11 | //! rather than rewriting it. |
| 12 | |
| 13 | use std::collections::BTreeMap; |
| 14 | use std::io::Read; |
| 15 | use std::path::{Path, PathBuf}; |
| 16 | |
| 17 | use serde::{Deserialize, Serialize}; |
| 18 | |
| 19 | use super::types::{MarketplaceCatalog, MarketplaceCatalogId}; |
| 20 | use crate::plugins::registry::{ |
| 21 | ensure_private_plugin_state_directory, harden_plugin_state_file, open_existing_regular_file, |
| 22 | open_state_lock, path_entry_exists, save_state_with_hardener, state_lock_path, |
| 23 | validate_existing_plugin_state_parent, |
| 24 | }; |
| 25 | |
| 26 | const MARKETPLACE_SCHEMA_VERSION: u32 = 1; |
| 27 | const MARKETPLACE_STATE_FILE: &str = "marketplaces.json"; |
| 28 | |
| 29 | /// One stored catalog: the parsed result plus where the document was read |
| 30 | /// from, so relative sources resolve against the catalog's own location at |
| 31 | /// install time. |
| 32 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 33 | pub struct StoredMarketplaceCatalog { |
| 34 | pub added_at: String, |
| 35 | /// Absolute path the catalog document was read from. |
| 36 | pub source_path: String, |
| 37 | pub catalog: MarketplaceCatalog, |
| 38 | } |
| 39 | |
| 40 | #[derive(Debug, Clone, Serialize, Deserialize)] |
| 41 | pub struct MarketplaceState { |
| 42 | schema_version: u32, |
| 43 | #[serde(default)] |
| 44 | first_party_removed: bool, |
| 45 | #[serde(default)] |
| 46 | catalogs: BTreeMap<String, StoredMarketplaceCatalog>, |
| 47 | } |
| 48 | |
| 49 | impl Default for MarketplaceState { |
| 50 | fn default() -> Self { |
| 51 | Self { |
| 52 | schema_version: MARKETPLACE_SCHEMA_VERSION, |
| 53 | first_party_removed: false, |
| 54 | catalogs: BTreeMap::new(), |
| 55 | } |
| 56 | } |
| 57 | } |
| 58 | |
| 59 | impl MarketplaceState { |
| 60 | #[must_use] |
| 61 | pub fn catalogs(&self) -> &BTreeMap<String, StoredMarketplaceCatalog> { |
| 62 | &self.catalogs |
| 63 | } |
| 64 | |
| 65 | pub fn get(&self, name: &str) -> Option<&StoredMarketplaceCatalog> { |
| 66 | self.catalogs.get(name) |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | /// Handle on the sibling catalog store. `None` from [`MarketplaceStore::open`] |
| 71 | /// means the registry has no persistence root at all (fail-closed registry); |
| 72 | /// callers report that honestly instead of inventing a location. |
| 73 | pub struct MarketplaceStore { |
| 74 | path: PathBuf, |
| 75 | } |
| 76 | |
| 77 | impl MarketplaceStore { |
| 78 | /// Open the store that siblings the registry's `state.json`. |
| 79 | #[must_use] |
| 80 | pub fn open(state_path: Option<&Path>) -> Option<Self> { |
| 81 | let parent = state_path?.parent()?.to_path_buf(); |
| 82 | Some(Self { |
| 83 | path: parent.join(MARKETPLACE_STATE_FILE), |
| 84 | }) |
| 85 | } |
| 86 | |
| 87 | #[must_use] |
| 88 | pub fn path(&self) -> &Path { |
| 89 | &self.path |
| 90 | } |
| 91 | |
| 92 | /// Read all catalogs. Never writes; never creates the file or its lock. |
| 93 | pub fn load(&self) -> Result<MarketplaceState, String> { |
| 94 | validate_existing_plugin_state_parent(&self.path)?; |
| 95 | let lock_path = state_lock_path(&self.path); |
| 96 | let mut state = if path_entry_exists(&lock_path)? { |
| 97 | let lock_file = open_state_lock(&lock_path, false)?; |
| 98 | let lock = fd_lock::RwLock::new(lock_file); |
| 99 | let _guard = lock |
| 100 | .read() |
| 101 | .map_err(|e| format!("failed to read-lock marketplace state: {e}"))?; |
| 102 | self.load_unlocked()? |
| 103 | } else { |
| 104 | self.load_unlocked()? |
| 105 | }; |
| 106 | // Every surface consumes this same projection. Browsing is offline |
| 107 | // and read-only; installation and updates use the reviewed installer. |
| 108 | // A user's local catalog or removal always takes precedence. |
| 109 | if !state.first_party_removed && !state.catalogs.contains_key("codewhale") { |
| 110 | state |
| 111 | .catalogs |
| 112 | .insert("codewhale".into(), first_party_catalog()?); |
| 113 | } |
| 114 | Ok(state) |
| 115 | } |
| 116 | |
| 117 | /// Maximum bytes read from the marketplace state file. |
| 118 | const MAX_STATE_BYTES: u64 = 1024 * 1024; |
| 119 | |
| 120 | fn load_unlocked(&self) -> Result<MarketplaceState, String> { |
| 121 | let Some(file) = open_existing_regular_file(&self.path, false)? else { |
| 122 | return Ok(MarketplaceState::default()); |
| 123 | }; |
| 124 | let mut raw = String::new(); |
| 125 | file.take(Self::MAX_STATE_BYTES + 1) |
| 126 | .read_to_string(&mut raw) |
| 127 | .map_err(|e| format!("failed to read {}: {e}", self.path.display()))?; |
| 128 | if raw.len() as u64 > Self::MAX_STATE_BYTES { |
| 129 | return Err(format!( |
| 130 | "marketplace state {} exceeds the 1 MiB limit", |
| 131 | self.path.display() |
| 132 | )); |
| 133 | } |
| 134 | let state: MarketplaceState = serde_json::from_str(&raw) |
| 135 | .map_err(|e| format!("failed to parse {}: {e}", self.path.display()))?; |
| 136 | if state.schema_version != MARKETPLACE_SCHEMA_VERSION { |
| 137 | return Err(format!( |
| 138 | "unsupported marketplace state schema {}; expected {MARKETPLACE_SCHEMA_VERSION} at {}", |
| 139 | state.schema_version, |
| 140 | self.path.display() |
| 141 | )); |
| 142 | } |
| 143 | Ok(collapse_same_source_catalogs(state)) |
| 144 | } |
| 145 | |
| 146 | /// Insert a catalog under `id`, keyed by its source document. |
| 147 | /// |
| 148 | /// A catalog *is* its source: re-adding the same document updates that |
| 149 | /// catalog in place — and renames it to the requested `id` when the name |
| 150 | /// differs — instead of colliding on whatever name the previous add chose. |
| 151 | /// That collision is what produced a second, stale snapshot of the |
| 152 | /// codewhale marketplace under a hand-made name ("cw2") sitting beside |
| 153 | /// `codewhale`, two copies of one source. An existing *different* source |
| 154 | /// under `id` still refuses, so a name never silently re-points. |
| 155 | pub fn add( |
| 156 | &self, |
| 157 | id: &MarketplaceCatalogId, |
| 158 | entry: StoredMarketplaceCatalog, |
| 159 | ) -> Result<(), String> { |
| 160 | self.mutate(|state| { |
| 161 | let source = canonical_source_path(&entry.source_path); |
| 162 | if let Some(existing) = state.catalogs.get(id.as_str()) |
| 163 | && canonical_source_path(&existing.source_path) != source |
| 164 | { |
| 165 | return Err(format!( |
| 166 | "a marketplace named `{}` already exists; /plugin marketplace remove {} first", |
| 167 | id.as_str(), |
| 168 | id.as_str() |
| 169 | )); |
| 170 | } |
| 171 | let duplicates: Vec<String> = state |
| 172 | .catalogs |
| 173 | .iter() |
| 174 | .filter(|(key, catalog)| { |
| 175 | key.as_str() != id.as_str() |
| 176 | && canonical_source_path(&catalog.source_path) == source |
| 177 | }) |
| 178 | .map(|(key, _)| key.clone()) |
| 179 | .collect(); |
| 180 | for key in duplicates { |
| 181 | state.catalogs.remove(&key); |
| 182 | } |
| 183 | state.catalogs.insert(id.as_str().to_string(), entry); |
| 184 | Ok(()) |
| 185 | }) |
| 186 | } |
| 187 | |
| 188 | /// Remove a catalog by name. `Ok(false)` means it was not stored. |
| 189 | pub fn remove(&self, name: &str) -> Result<bool, String> { |
| 190 | self.mutate(|state| { |
| 191 | let removed = state.catalogs.remove(name).is_some(); |
| 192 | let bundled = name == "codewhale" && !state.first_party_removed; |
| 193 | if name == "codewhale" { |
| 194 | state.first_party_removed = true; |
| 195 | } |
| 196 | Ok(removed || bundled) |
| 197 | }) |
| 198 | } |
| 199 | |
| 200 | fn mutate<R>( |
| 201 | &self, |
| 202 | mutate: impl FnOnce(&mut MarketplaceState) -> Result<R, String>, |
| 203 | ) -> Result<R, String> { |
| 204 | let lock_path = state_lock_path(&self.path); |
| 205 | if let Some(parent) = lock_path.parent() { |
| 206 | ensure_private_plugin_state_directory(parent)?; |
| 207 | } |
| 208 | let lock_file = open_state_lock(&lock_path, true)?; |
| 209 | let mut lock = fd_lock::RwLock::new(lock_file); |
| 210 | let _guard = lock |
| 211 | .write() |
| 212 | .map_err(|e| format!("failed to lock marketplace state for update: {e}"))?; |
| 213 | let mut next = self.load_unlocked()?; |
| 214 | let result = mutate(&mut next)?; |
| 215 | save_state_with_hardener(&self.path, &next, harden_plugin_state_file)?; |
| 216 | Ok(result) |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | /// Canonical form of a catalog's source document, for source-identity |
| 221 | /// comparison. This is the identity the store keys updates and duplicate |
| 222 | /// detection on (the marketplace sibling of grokbuild's |
| 223 | /// `MarketplaceSource::identity`). Falls back to the stored string when the |
| 224 | /// path is not readable as a file — a GitHub URL, or a document that has |
| 225 | /// since moved away. |
| 226 | /// |
| 227 | /// The `std::fs` site below carries a budget entry in |
| 228 | /// `scripts/check-blocking-calls-budget.json`: the ratchet counts a |
| 229 | /// synchronous helper's site regardless of its callers, and this helper |
| 230 | /// cannot move to the blocking pool without restructuring the store's |
| 231 | /// synchronous `load`/`add`/`remove` API (#6149). |
| 232 | /// |
| 233 | /// Known limitation, recorded because the ratchet cannot see it: the scanner |
| 234 | /// is lexical and per-file, so it does not notice that |
| 235 | /// `runtime_api::plugins` calls [`MarketplaceStore::load`]/`add`/`remove` |
| 236 | /// inline from async route handlers. The budget entry is therefore debt |
| 237 | /// acknowledged, not debt paid. |
| 238 | fn canonical_source_path(path: &str) -> PathBuf { |
| 239 | std::fs::canonicalize(path).unwrap_or_else(|_| PathBuf::from(path)) |
| 240 | } |
| 241 | |
| 242 | /// Collapse catalogs that point at the same source document. |
| 243 | /// |
| 244 | /// Older states can hold one marketplace twice under two names (that is what |
| 245 | /// `"cw2"` was: a second snapshot of the same document, taken only because the |
| 246 | /// first name was already in use). Every surface should see one catalog per |
| 247 | /// source. The survivor is the entry whose key is the document's own declared |
| 248 | /// name when one exists, otherwise the most recently added; the projection is |
| 249 | /// in-memory, and the next write persists it. |
| 250 | fn collapse_same_source_catalogs(mut state: MarketplaceState) -> MarketplaceState { |
| 251 | let mut groups: BTreeMap<PathBuf, Vec<String>> = BTreeMap::new(); |
| 252 | for (key, catalog) in &state.catalogs { |
| 253 | groups |
| 254 | .entry(canonical_source_path(&catalog.source_path)) |
| 255 | .or_default() |
| 256 | .push(key.clone()); |
| 257 | } |
| 258 | for keys in groups.values() { |
| 259 | if keys.len() < 2 { |
| 260 | continue; |
| 261 | } |
| 262 | let named_after_document = keys.iter().find(|key| { |
| 263 | state |
| 264 | .catalogs |
| 265 | .get(key.as_str()) |
| 266 | .is_some_and(|catalog| catalog.catalog.name == key.as_str()) |
| 267 | }); |
| 268 | let keeper = named_after_document.cloned().or_else(|| { |
| 269 | keys.iter() |
| 270 | .max_by(|left, right| { |
| 271 | let added = |key: &String| { |
| 272 | state |
| 273 | .catalogs |
| 274 | .get(key.as_str()) |
| 275 | .map(|catalog| catalog.added_at.as_str()) |
| 276 | .unwrap_or_default() |
| 277 | }; |
| 278 | added(left).cmp(added(right)) |
| 279 | }) |
| 280 | .cloned() |
| 281 | }); |
| 282 | let Some(keeper) = keeper else { continue }; |
| 283 | for key in keys { |
| 284 | if *key != keeper { |
| 285 | state.catalogs.remove(key); |
| 286 | } |
| 287 | } |
| 288 | } |
| 289 | state |
| 290 | } |
| 291 | |
| 292 | fn first_party_catalog() -> Result<StoredMarketplaceCatalog, String> { |
| 293 | #[derive(Deserialize)] |
| 294 | struct Snapshot { |
| 295 | repository: String, |
| 296 | revision: String, |
| 297 | catalog: serde_json::Value, |
| 298 | } |
| 299 | let snapshot: Snapshot = |
| 300 | serde_json::from_str(include_str!("../../../assets/first-party-marketplace.json")) |
| 301 | .map_err(|error| format!("invalid bundled marketplace: {error}"))?; |
| 302 | let source = format!("{}/tree/{}", snapshot.repository, snapshot.revision); |
| 303 | let mut catalog = super::parsers::parse_catalog(super::parsers::MarketplaceDocument { |
| 304 | catalog_id: MarketplaceCatalogId::new("codewhale"), |
| 305 | format: super::types::MarketplaceFormat::Codewhale, |
| 306 | root: snapshot.catalog, |
| 307 | base: Some(source.clone()), |
| 308 | }); |
| 309 | if catalog.error_count() > 0 { |
| 310 | return Err("bundled marketplace contains invalid entries".into()); |
| 311 | } |
| 312 | catalog.provenance.tier = super::types::CatalogTier::Official; |
| 313 | catalog.provenance.source_url = Some(source.clone()); |
| 314 | for candidate in &mut catalog.candidates { |
| 315 | candidate.provenance = catalog.provenance.clone(); |
| 316 | } |
| 317 | Ok(StoredMarketplaceCatalog { |
| 318 | added_at: String::new(), |
| 319 | source_path: source, |
| 320 | catalog, |
| 321 | }) |
| 322 | } |
| 323 | |
| 324 | #[cfg(test)] |
| 325 | mod tests { |
| 326 | use super::*; |
| 327 | use std::fs; |
| 328 | |
| 329 | #[test] |
| 330 | fn bundled_catalog_is_offline_installable_and_removable_without_rewriting_plugin_state() { |
| 331 | let root = tempfile::tempdir().unwrap(); |
| 332 | let state_path = root.path().join("plugins/state.json"); |
| 333 | let store = MarketplaceStore::open(Some(&state_path)).unwrap(); |
| 334 | let initial = store.load().unwrap(); |
| 335 | let catalog = initial.get("codewhale").expect("first-party catalog"); |
| 336 | assert_eq!(catalog.catalog.total_candidates(), 5); |
| 337 | assert_eq!(catalog.catalog.error_count(), 0); |
| 338 | assert_eq!(catalog.catalog.warning_count(), 0); |
| 339 | assert!(!catalog.catalog.provenance.grants_trust()); |
| 340 | let registry = crate::plugins::PluginRegistry::empty(root.path()); |
| 341 | for candidate in &catalog.catalog.candidates { |
| 342 | assert!(candidate.install_plan.is_supported()); |
| 343 | assert!(!candidate.provenance.grants_trust()); |
| 344 | let super::super::document::CatalogInstallResolution::Supported { spec, .. } = |
| 345 | super::super::document::resolve_candidate_install(catalog, candidate, ®istry) |
| 346 | else { |
| 347 | panic!("uninstallable candidate") |
| 348 | }; |
| 349 | assert!( |
| 350 | spec.starts_with( |
| 351 | "https://codeload.github.com/Hmbown/codewhale-plugin-marketplace/" |
| 352 | ) |
| 353 | ); |
| 354 | assert!(spec.contains("#path=")); |
| 355 | } |
| 356 | assert!(!store.path().exists(), "browsing must not write or fetch"); |
| 357 | assert!(store.remove("codewhale").unwrap()); |
| 358 | assert!(!store.remove("codewhale").unwrap()); |
| 359 | assert!(store.load().unwrap().catalogs().is_empty()); |
| 360 | assert!( |
| 361 | !state_path.exists(), |
| 362 | "catalog removal must not change plugin trust state" |
| 363 | ); |
| 364 | let mut local = first_party_catalog().unwrap(); |
| 365 | local.source_path = "/reviewed/local/marketplace.json".into(); |
| 366 | store |
| 367 | .add(&MarketplaceCatalogId::new("codewhale"), local) |
| 368 | .unwrap(); |
| 369 | assert_eq!( |
| 370 | store.load().unwrap().get("codewhale").unwrap().source_path, |
| 371 | "/reviewed/local/marketplace.json" |
| 372 | ); |
| 373 | } |
| 374 | |
| 375 | #[test] |
| 376 | fn malformed_store_is_not_hidden_by_bundled_catalog() { |
| 377 | let root = tempfile::tempdir().unwrap(); |
| 378 | let store = MarketplaceStore::open(Some(&root.path().join("state.json"))).unwrap(); |
| 379 | fs::write(store.path(), "broken").unwrap(); |
| 380 | assert!(store.load().is_err()); |
| 381 | assert_eq!(fs::read_to_string(store.path()).unwrap(), "broken"); |
| 382 | } |
| 383 | |
| 384 | fn catalog_from_source(source: &Path) -> StoredMarketplaceCatalog { |
| 385 | let mut catalog = first_party_catalog().unwrap(); |
| 386 | catalog.source_path = source.display().to_string(); |
| 387 | catalog |
| 388 | } |
| 389 | |
| 390 | /// `add` keys on the source document, not the name the previous add used: |
| 391 | /// re-adding the same marketplace under its real name replaces the |
| 392 | /// hand-made duplicate instead of colliding or leaving both behind. |
| 393 | #[test] |
| 394 | fn re_adding_the_same_source_renames_in_place() { |
| 395 | let root = tempfile::tempdir().unwrap(); |
| 396 | let store = MarketplaceStore::open(Some(&root.path().join("plugins/state.json"))).unwrap(); |
| 397 | let document = root.path().join("marketplace.json"); |
| 398 | fs::write(&document, "{}").unwrap(); |
| 399 | |
| 400 | store |
| 401 | .add( |
| 402 | &MarketplaceCatalogId::new("cw2"), |
| 403 | catalog_from_source(&document), |
| 404 | ) |
| 405 | .unwrap(); |
| 406 | store |
| 407 | .add( |
| 408 | &MarketplaceCatalogId::new("codewhale"), |
| 409 | catalog_from_source(&document), |
| 410 | ) |
| 411 | .unwrap(); |
| 412 | |
| 413 | let state = store.load().unwrap(); |
| 414 | assert!(state.get("cw2").is_none(), "the duplicate name is gone"); |
| 415 | assert_eq!( |
| 416 | state.get("codewhale").unwrap().source_path, |
| 417 | document.display().to_string() |
| 418 | ); |
| 419 | } |
| 420 | |
| 421 | /// A name is never silently re-pointed at a different source. |
| 422 | #[test] |
| 423 | fn a_name_never_re_points_at_a_different_source() { |
| 424 | let root = tempfile::tempdir().unwrap(); |
| 425 | let store = MarketplaceStore::open(Some(&root.path().join("plugins/state.json"))).unwrap(); |
| 426 | let first = root.path().join("first.json"); |
| 427 | let second = root.path().join("second.json"); |
| 428 | fs::write(&first, "{}").unwrap(); |
| 429 | fs::write(&second, "{}").unwrap(); |
| 430 | |
| 431 | store |
| 432 | .add( |
| 433 | &MarketplaceCatalogId::new("codewhale"), |
| 434 | catalog_from_source(&first), |
| 435 | ) |
| 436 | .unwrap(); |
| 437 | let refused = store |
| 438 | .add( |
| 439 | &MarketplaceCatalogId::new("codewhale"), |
| 440 | catalog_from_source(&second), |
| 441 | ) |
| 442 | .expect_err("a different source under a taken name must refuse"); |
| 443 | assert!(refused.contains("already exists"), "{refused}"); |
| 444 | assert_eq!( |
| 445 | store.load().unwrap().get("codewhale").unwrap().source_path, |
| 446 | first.display().to_string() |
| 447 | ); |
| 448 | } |
| 449 | |
| 450 | /// A state written before source identity was keyed can hold one source |
| 451 | /// twice under two names; the projection collapses it to the entry named |
| 452 | /// after the document itself. |
| 453 | #[test] |
| 454 | fn load_collapses_one_source_stored_under_two_names() { |
| 455 | let root = tempfile::tempdir().unwrap(); |
| 456 | let store = MarketplaceStore::open(Some(&root.path().join("plugins/state.json"))).unwrap(); |
| 457 | let document = root.path().join("marketplace.json"); |
| 458 | fs::write(&document, "{}").unwrap(); |
| 459 | |
| 460 | let mut state = MarketplaceState::default(); |
| 461 | for (key, added_at) in [ |
| 462 | ("codewhale", "2026-09-02T00:00:00Z"), |
| 463 | ("cw2", "2026-09-16T00:00:00Z"), |
| 464 | ] { |
| 465 | let mut catalog = catalog_from_source(&document); |
| 466 | catalog.added_at = added_at.to_string(); |
| 467 | state.catalogs.insert(key.to_string(), catalog); |
| 468 | } |
| 469 | ensure_private_plugin_state_directory(store.path().parent().unwrap()).unwrap(); |
| 470 | fs::write(store.path(), serde_json::to_string(&state).unwrap()).unwrap(); |
| 471 | |
| 472 | let loaded = store.load().unwrap(); |
| 473 | assert!(loaded.get("cw2").is_none()); |
| 474 | assert!(loaded.get("codewhale").is_some()); |
| 475 | assert_eq!( |
| 476 | loaded.get("codewhale").unwrap().source_path, |
| 477 | document.display().to_string() |
| 478 | ); |
| 479 | } |
| 480 | } |
| 481 |