| 1 | //! Content-addressed store for pre-compression image originals. |
| 2 | //! |
| 3 | //! When `read_media` downsamples or lossily re-encodes an image to fit the |
| 4 | //! delivery budget, the model only ever sees the degraded copy. Persisting |
| 5 | //! the pre-compression bytes — named by content hash — lets a later |
| 6 | //! `read_media` call with a crop region read the full-resolution source back |
| 7 | //! (the tool's delivery note points at the stored path), even if the |
| 8 | //! workspace file has moved or changed since the first read. |
| 9 | //! |
| 10 | //! Design notes: |
| 11 | //! - Content-addressed (sha256): repeated reads of the same image reuse one |
| 12 | //! file and repeated writes are idempotent. |
| 13 | //! - Best effort: any filesystem failure returns `None`; media delivery never |
| 14 | //! blocks on persistence. |
| 15 | //! - Size-capped: after each write the store is swept oldest-first (mtime) |
| 16 | //! until it fits [`MAX_TOTAL_BYTES`], so long sessions cannot fill the disk. |
| 17 | //! - Placement: the production engine wires the store dir through |
| 18 | //! `RuntimeToolServices::media_originals_dir` |
| 19 | //! ([`default_store_dir`]); test and one-off contexts leave it `None`, which |
| 20 | //! disables persistence so unit tests never touch the real state dir. |
| 21 | |
| 22 | use std::path::{Path, PathBuf}; |
| 23 | |
| 24 | use sha2::{Digest as _, Sha256}; |
| 25 | |
| 26 | /// Total size ceiling for the store (1 GiB). Each admitted source is already |
| 27 | /// capped at `read_media`'s 20 MiB source limit, so the store holds dozens of |
| 28 | /// full-resolution originals before the sweep engages. |
| 29 | pub const MAX_TOTAL_BYTES: u64 = 1024 * 1024 * 1024; |
| 30 | |
| 31 | /// Store subdirectory under the CodeWhale home directory. |
| 32 | pub const MEDIA_ORIGINALS_SUBDIR: &str = "media-originals"; |
| 33 | |
| 34 | /// The production store location: `<codewhale home>/media-originals`. |
| 35 | /// |
| 36 | /// The directory is created lazily on first persist, not here, so resolving |
| 37 | /// the path never has a filesystem side effect. |
| 38 | #[must_use] |
| 39 | pub fn default_store_dir() -> Option<PathBuf> { |
| 40 | codewhale_config::codewhale_home() |
| 41 | .ok() |
| 42 | .map(|home| home.join(MEDIA_ORIGINALS_SUBDIR)) |
| 43 | } |
| 44 | |
| 45 | fn extension_for_mime(mime_type: &str) -> &'static str { |
| 46 | match mime_type.trim().to_ascii_lowercase().as_str() { |
| 47 | "image/png" => "png", |
| 48 | "image/jpeg" | "image/jpg" => "jpg", |
| 49 | "image/gif" => "gif", |
| 50 | "image/webp" => "webp", |
| 51 | "image/bmp" => "bmp", |
| 52 | "image/tiff" => "tif", |
| 53 | _ => "img", |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | /// Persist `bytes` under `dir`, returning the content-addressed path. |
| 58 | /// |
| 59 | /// Best effort: returns `None` on any filesystem failure. A same-named entry |
| 60 | /// with the same length is reused as-is (content addressing makes a length |
| 61 | /// match a content match for practical purposes). |
| 62 | pub fn persist_original_image(bytes: &[u8], mime_type: &str, dir: &Path) -> Option<PathBuf> { |
| 63 | if bytes.is_empty() { |
| 64 | return None; |
| 65 | } |
| 66 | let digest = Sha256::digest(bytes); |
| 67 | let mut hash = String::with_capacity(32); |
| 68 | for byte in digest.iter().take(16) { |
| 69 | use std::fmt::Write as _; |
| 70 | let _ = write!(hash, "{byte:02x}"); |
| 71 | } |
| 72 | let path = dir.join(format!("{hash}.{}", extension_for_mime(mime_type))); |
| 73 | std::fs::create_dir_all(dir).ok()?; |
| 74 | let write_needed = match std::fs::metadata(&path) { |
| 75 | Ok(meta) => meta.len() != bytes.len() as u64, |
| 76 | Err(_) => true, |
| 77 | }; |
| 78 | if write_needed { |
| 79 | std::fs::write(&path, bytes).ok()?; |
| 80 | } |
| 81 | sweep_store(dir, MAX_TOTAL_BYTES); |
| 82 | std::fs::metadata(&path).ok()?; |
| 83 | Some(path) |
| 84 | } |
| 85 | |
| 86 | /// Resolve `raw` as a read-back path inside the store. |
| 87 | /// |
| 88 | /// Returns the canonical path only when the file exists and lives under |
| 89 | /// `store_dir`. `read_media` uses this as a fallback after the workspace |
| 90 | /// resolver rejects a path: the store only contains image bytes that already |
| 91 | /// passed the tool's read guards, so read-back widens nothing. |
| 92 | #[must_use] |
| 93 | pub fn resolve_stored_original(raw: &str, workspace: &Path, store_dir: &Path) -> Option<PathBuf> { |
| 94 | let candidate = if Path::new(raw).is_absolute() { |
| 95 | PathBuf::from(raw) |
| 96 | } else { |
| 97 | workspace.join(raw) |
| 98 | }; |
| 99 | let candidate = candidate.canonicalize().ok()?; |
| 100 | let store_dir = store_dir.canonicalize().ok()?; |
| 101 | candidate.starts_with(store_dir).then_some(candidate) |
| 102 | } |
| 103 | |
| 104 | /// Oldest-first (mtime) sweep until the store fits `max_total_bytes`. |
| 105 | /// Best effort: individual stat/unlink failures are skipped. |
| 106 | fn sweep_store(dir: &Path, max_total_bytes: u64) { |
| 107 | let Ok(entries) = std::fs::read_dir(dir) else { |
| 108 | return; |
| 109 | }; |
| 110 | let mut files: Vec<(PathBuf, u64, std::time::SystemTime)> = Vec::new(); |
| 111 | for entry in entries.flatten() { |
| 112 | let Ok(meta) = entry.metadata() else { |
| 113 | continue; |
| 114 | }; |
| 115 | if !meta.is_file() { |
| 116 | continue; |
| 117 | } |
| 118 | files.push(( |
| 119 | entry.path(), |
| 120 | meta.len(), |
| 121 | meta.modified().unwrap_or(std::time::SystemTime::UNIX_EPOCH), |
| 122 | )); |
| 123 | } |
| 124 | let mut total: u64 = files.iter().map(|(_, len, _)| *len).sum(); |
| 125 | if total <= max_total_bytes { |
| 126 | return; |
| 127 | } |
| 128 | files.sort_by_key(|(_, _, mtime)| *mtime); |
| 129 | for (path, len, _) in files { |
| 130 | if total <= max_total_bytes { |
| 131 | break; |
| 132 | } |
| 133 | if std::fs::remove_file(&path).is_ok() { |
| 134 | total = total.saturating_sub(len); |
| 135 | } |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | #[cfg(test)] |
| 140 | mod tests { |
| 141 | use super::*; |
| 142 | |
| 143 | #[test] |
| 144 | fn persist_is_content_addressed_and_idempotent() { |
| 145 | let dir = tempfile::tempdir().expect("tempdir"); |
| 146 | let store = dir.path().join("media-originals"); |
| 147 | let bytes = b"\x89PNG fake-but-deterministic payload"; |
| 148 | |
| 149 | let first = persist_original_image(bytes, "image/png", &store).expect("persist"); |
| 150 | assert!(first.exists()); |
| 151 | assert_eq!(first.parent(), Some(store.as_path())); |
| 152 | assert_eq!( |
| 153 | first.extension().and_then(|e| e.to_str()), |
| 154 | Some("png"), |
| 155 | "mime drives the extension" |
| 156 | ); |
| 157 | let stem = first.file_stem().unwrap().to_str().unwrap().to_string(); |
| 158 | assert_eq!(stem.len(), 32, "sha256 prefix names the file: {stem}"); |
| 159 | |
| 160 | let second = persist_original_image(bytes, "image/png", &store).expect("re-persist"); |
| 161 | assert_eq!(first, second, "same bytes reuse one file"); |
| 162 | assert_eq!(std::fs::read_dir(&store).unwrap().count(), 1); |
| 163 | } |
| 164 | |
| 165 | #[test] |
| 166 | fn sweep_removes_oldest_until_under_cap() { |
| 167 | let dir = tempfile::tempdir().expect("tempdir"); |
| 168 | let store = dir.path().to_path_buf(); |
| 169 | let mut paths = Vec::new(); |
| 170 | for i in 0..4u8 { |
| 171 | let path = store.join(format!("img{i}.png")); |
| 172 | std::fs::write(&path, vec![i; 100]).expect("write"); |
| 173 | paths.push(path); |
| 174 | } |
| 175 | // 400 bytes total; cap at 250 -> at least the two oldest go. |
| 176 | sweep_store(&store, 250); |
| 177 | let remaining = std::fs::read_dir(&store).unwrap().count(); |
| 178 | assert!( |
| 179 | remaining <= 2, |
| 180 | "sweep must evict oldest-first until under cap, {remaining} left" |
| 181 | ); |
| 182 | let total: u64 = std::fs::read_dir(&store) |
| 183 | .unwrap() |
| 184 | .flatten() |
| 185 | .filter_map(|e| e.metadata().ok()) |
| 186 | .map(|m| m.len()) |
| 187 | .sum(); |
| 188 | assert!(total <= 250); |
| 189 | } |
| 190 | |
| 191 | #[test] |
| 192 | fn resolve_stored_original_admits_store_paths_only() { |
| 193 | let dir = tempfile::tempdir().expect("tempdir"); |
| 194 | let store = dir.path().join("media-originals"); |
| 195 | std::fs::create_dir_all(&store).unwrap(); |
| 196 | let inside = store.join("abc123.png"); |
| 197 | std::fs::write(&inside, b"png").unwrap(); |
| 198 | let outside = dir.path().join("other.png"); |
| 199 | std::fs::write(&outside, b"png").unwrap(); |
| 200 | |
| 201 | let resolved = |
| 202 | resolve_stored_original(inside.to_str().unwrap(), dir.path(), &store).expect("inside"); |
| 203 | assert!(resolved.ends_with("abc123.png")); |
| 204 | assert!( |
| 205 | resolve_stored_original(outside.to_str().unwrap(), dir.path(), &store).is_none(), |
| 206 | "paths outside the store must not resolve through the fallback" |
| 207 | ); |
| 208 | assert!( |
| 209 | resolve_stored_original("missing.png", dir.path(), &store).is_none(), |
| 210 | "non-existent files do not resolve" |
| 211 | ); |
| 212 | } |
| 213 | } |
| 214 |