| 1 | //! `read_media` tool — safe, first-class image reading and preprocessing. |
| 2 | //! |
| 3 | //! Provides bounded decoding, decompression-bomb protection, active route |
| 4 | //! vision-capability checks, crop region extraction, resolution detail modes, |
| 5 | //! and typed receipt metadata without leaking credentials. |
| 6 | //! |
| 7 | //! Delivery is byte-budget-first: after an optional crop and the detail-mode |
| 8 | //! edge cap, the image descends an encoding ladder (JPEG quality steps for |
| 9 | //! photo-like content, PNG for alpha-bearing or flat/line-art content, then |
| 10 | //! longest-edge halving) until the payload fits the budget. Results carry a |
| 11 | //! delivery note stating exactly how the image was delivered (untouched / |
| 12 | //! downsampled / crop / full) with zoom guidance, and the pre-compression |
| 13 | //! original is persisted in the content-addressed [`crate::media_originals`] |
| 14 | //! store so a later crop read can pull the full-resolution source. When no |
| 15 | //! ladder result fits the budget the tool fails closed — nothing is sent — |
| 16 | //! with the exact conversion command to retry with. |
| 17 | |
| 18 | #[cfg(test)] |
| 19 | use image::ImageReader; |
| 20 | #[cfg(test)] |
| 21 | use std::io::Cursor; |
| 22 | |
| 23 | use async_trait::async_trait; |
| 24 | use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64}; |
| 25 | use codewhale_config::route::CapabilityState; |
| 26 | use image::codecs::jpeg::JpegEncoder; |
| 27 | use image::codecs::png::{CompressionType, FilterType as PngFilter, PngEncoder}; |
| 28 | use image::imageops::FilterType; |
| 29 | use image::{DynamicImage, ExtendedColorType, GenericImageView, ImageEncoder}; |
| 30 | use serde::{Deserialize, Serialize}; |
| 31 | use serde_json::{Value, json}; |
| 32 | |
| 33 | use super::spec::{ |
| 34 | ApprovalRequirement, RichToolResult, ToolCapability, ToolContext, ToolError, ToolResult, |
| 35 | ToolResultContentBlock, ToolSpec, optional_str, required_str, type_mismatch, |
| 36 | }; |
| 37 | |
| 38 | /// Maximum source image size before decoding (20 MiB). |
| 39 | pub const MAX_SOURCE_IMAGE_BYTES: usize = 20 * 1024 * 1024; |
| 40 | |
| 41 | use crate::image_attach::decode_and_guard_image; |
| 42 | |
| 43 | /// Maximum inline image payload admitted on the wire (5 MiB). |
| 44 | pub const MAX_WIRE_IMAGE_BYTES: usize = crate::image_attach::MAX_IMAGE_BYTES; |
| 45 | |
| 46 | /// Delivery byte budget for a default read (`detail` `low`/`auto`), 256 KiB. |
| 47 | /// |
| 48 | /// Matches the reference budget in kimi-code's `READ_IMAGE_BYTE_BUDGET`: after |
| 49 | /// base64 inflation this is ~350 KB on the wire — far under the 5 MiB provider |
| 50 | /// ceiling — and it keeps routine reads from spending megabytes of context on |
| 51 | /// PNG screenshots when a JPEG ladder step fits. `high`/`original` detail and |
| 52 | /// crop (deliberate zoom) reads keep the full [`MAX_WIRE_IMAGE_BYTES`] budget. |
| 53 | pub const READ_IMAGE_BYTE_BUDGET: usize = 256 * 1024; |
| 54 | |
| 55 | /// JPEG quality steps tried highest-first at each ladder rung (mirrors |
| 56 | /// kimi-code's `JPEG_QUALITY_STEPS`). |
| 57 | const JPEG_QUALITY_LADDER: [u8; 4] = [80, 60, 40, 20]; |
| 58 | |
| 59 | /// Longest-edge floor for ladder halving (px); below this the ladder fails |
| 60 | /// closed rather than delivering an unreadable thumbnail. |
| 61 | const MIN_DELIVERY_EDGE_PX: u32 = 256; |
| 62 | |
| 63 | /// Flat/line-art images keep PNG rungs down to this longest edge before the |
| 64 | /// ladder falls back to JPEG (mirrors kimi-code's `PNG_RESCALE_FLOOR_PX`). |
| 65 | const PNG_EDGE_FLOOR_PX: u32 = 1000; |
| 66 | |
| 67 | /// Opaque images whose colorfulness sigma (spread of the `r-g` and |
| 68 | /// `(r+g)/2-b` opponent channels, measured on a ≤128 px thumbnail) falls below |
| 69 | /// this threshold are treated as flat/line-art and stay PNG; above it the |
| 70 | /// image is photo-like and takes the JPEG ladder. Solid fills and line art |
| 71 | /// score ~0; ordinary photos score well above. Heuristic, deliberately simple. |
| 72 | const FLAT_COLORFULNESS_THRESHOLD: f64 = 12.0; |
| 73 | |
| 74 | /// Resolution/detail preference for image processing. |
| 75 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] |
| 76 | #[serde(rename_all = "snake_case")] |
| 77 | pub enum DetailMode { |
| 78 | #[default] |
| 79 | Auto, |
| 80 | Low, |
| 81 | High, |
| 82 | Original, |
| 83 | } |
| 84 | |
| 85 | impl DetailMode { |
| 86 | fn from_str_opt(s: Option<&str>) -> Result<Self, ToolError> { |
| 87 | match s { |
| 88 | None | Some("auto") => Ok(Self::Auto), |
| 89 | Some("low") => Ok(Self::Low), |
| 90 | Some("high") => Ok(Self::High), |
| 91 | Some("original") | Some("full") => Ok(Self::Original), |
| 92 | Some(other) => Err(ToolError::invalid_input(format!( |
| 93 | "invalid detail mode '{other}'; expected 'auto', 'low', 'high', or 'original'" |
| 94 | ))), |
| 95 | } |
| 96 | } |
| 97 | |
| 98 | fn max_dimension(self) -> u32 { |
| 99 | match self { |
| 100 | Self::Low => 1024, |
| 101 | Self::Auto => 2048, |
| 102 | Self::High | Self::Original => 4096, |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | /// Delivery byte budget for a full (non-crop) read at this detail level. |
| 107 | /// Crop reads are the deliberate zoom action and always get the full wire |
| 108 | /// budget, matching kimi-code's `cropImageForModel` default. |
| 109 | fn byte_budget(self) -> usize { |
| 110 | match self { |
| 111 | Self::Low | Self::Auto => READ_IMAGE_BYTE_BUDGET, |
| 112 | Self::High | Self::Original => MAX_WIRE_IMAGE_BYTES, |
| 113 | } |
| 114 | } |
| 115 | |
| 116 | fn as_str(self) -> &'static str { |
| 117 | match self { |
| 118 | Self::Auto => "auto", |
| 119 | Self::Low => "low", |
| 120 | Self::High => "high", |
| 121 | Self::Original => "original", |
| 122 | } |
| 123 | } |
| 124 | } |
| 125 | |
| 126 | /// Optional bounding box for cropping an image. |
| 127 | #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] |
| 128 | pub struct CropRegion { |
| 129 | pub x: u32, |
| 130 | pub y: u32, |
| 131 | pub width: u32, |
| 132 | pub height: u32, |
| 133 | } |
| 134 | |
| 135 | impl CropRegion { |
| 136 | fn parse_from_value(value: Option<&Value>) -> Result<Option<Self>, ToolError> { |
| 137 | let Some(val) = value else { |
| 138 | return Ok(None); |
| 139 | }; |
| 140 | if val.is_null() { |
| 141 | return Ok(None); |
| 142 | } |
| 143 | let obj = val |
| 144 | .as_object() |
| 145 | .ok_or_else(|| type_mismatch("crop", val, "an object with x, y, width, and height"))?; |
| 146 | |
| 147 | let extract_u32 = |field: &str| -> Result<u32, ToolError> { |
| 148 | let num_val = obj.get(field).ok_or_else(|| { |
| 149 | ToolError::invalid_input(format!("crop missing required field '{field}'")) |
| 150 | })?; |
| 151 | if let Some(n) = num_val.as_u64() { |
| 152 | u32::try_from(n).map_err(|_| { |
| 153 | ToolError::invalid_input(format!( |
| 154 | "crop field '{field}' is out of range for u32" |
| 155 | )) |
| 156 | }) |
| 157 | } else if let Some(n) = num_val.as_i64() { |
| 158 | if n < 0 { |
| 159 | return Err(ToolError::invalid_input(format!( |
| 160 | "crop field '{field}' must be non-negative; got {n}" |
| 161 | ))); |
| 162 | } |
| 163 | u32::try_from(n).map_err(|_| { |
| 164 | ToolError::invalid_input(format!( |
| 165 | "crop field '{field}' is out of range for u32" |
| 166 | )) |
| 167 | }) |
| 168 | } else { |
| 169 | Err(type_mismatch( |
| 170 | &format!("crop.{field}"), |
| 171 | num_val, |
| 172 | "an integer", |
| 173 | )) |
| 174 | } |
| 175 | }; |
| 176 | |
| 177 | let x = extract_u32("x")?; |
| 178 | let y = extract_u32("y")?; |
| 179 | let width = extract_u32("width")?; |
| 180 | let height = extract_u32("height")?; |
| 181 | |
| 182 | if width == 0 || height == 0 { |
| 183 | return Err(ToolError::invalid_input( |
| 184 | "crop width and height must be greater than 0", |
| 185 | )); |
| 186 | } |
| 187 | |
| 188 | Ok(Some(Self { |
| 189 | x, |
| 190 | y, |
| 191 | width, |
| 192 | height, |
| 193 | })) |
| 194 | } |
| 195 | } |
| 196 | |
| 197 | /// The first-class `read_media` tool. |
| 198 | pub struct ReadMediaTool; |
| 199 | |
| 200 | impl Default for ReadMediaTool { |
| 201 | fn default() -> Self { |
| 202 | Self |
| 203 | } |
| 204 | } |
| 205 | |
| 206 | #[async_trait] |
| 207 | impl ToolSpec for ReadMediaTool { |
| 208 | fn name(&self) -> &'static str { |
| 209 | "read_media" |
| 210 | } |
| 211 | |
| 212 | fn description(&self) -> &'static str { |
| 213 | "Read an image file (PNG, JPEG, GIF, WebP) into context for multimodal/vision inspection, with optional crop region and detail level. Safe, bounded decode with pixel and byte guards." |
| 214 | } |
| 215 | |
| 216 | fn input_schema(&self) -> Value { |
| 217 | json!({ |
| 218 | "type": "object", |
| 219 | "properties": { |
| 220 | "path": { |
| 221 | "type": "string", |
| 222 | "description": "Path to the image file (relative to workspace or absolute). PNG, JPEG, GIF, and WebP are supported." |
| 223 | }, |
| 224 | "crop": { |
| 225 | "type": "object", |
| 226 | "description": "Optional bounding box to crop [x, y, width, height] in pixel coordinates.", |
| 227 | "properties": { |
| 228 | "x": { |
| 229 | "type": "integer", |
| 230 | "description": "Left coordinate (X) of the crop region in pixels (0-based)." |
| 231 | }, |
| 232 | "y": { |
| 233 | "type": "integer", |
| 234 | "description": "Top coordinate (Y) of the crop region in pixels (0-based)." |
| 235 | }, |
| 236 | "width": { |
| 237 | "type": "integer", |
| 238 | "description": "Width of the crop region in pixels (must be > 0)." |
| 239 | }, |
| 240 | "height": { |
| 241 | "type": "integer", |
| 242 | "description": "Height of the crop region in pixels (must be > 0)." |
| 243 | } |
| 244 | }, |
| 245 | "required": ["x", "y", "width", "height"] |
| 246 | }, |
| 247 | "detail": { |
| 248 | "type": "string", |
| 249 | "enum": ["auto", "low", "high", "original"], |
| 250 | "description": "Resolution/detail preference. 'auto' (default) downscales large images to max 2048px; 'low' to max 1024px; 'high' / 'original' preserves resolution up to max 4096px within the 5 MiB payload limit." |
| 251 | } |
| 252 | }, |
| 253 | "required": ["path"] |
| 254 | }) |
| 255 | } |
| 256 | |
| 257 | fn capabilities(&self) -> Vec<ToolCapability> { |
| 258 | vec![ToolCapability::ReadOnly, ToolCapability::Sandboxable] |
| 259 | } |
| 260 | |
| 261 | fn approval_requirement(&self) -> ApprovalRequirement { |
| 262 | ApprovalRequirement::Auto |
| 263 | } |
| 264 | |
| 265 | fn supports_parallel(&self) -> bool { |
| 266 | true |
| 267 | } |
| 268 | |
| 269 | fn defer_loading(&self) -> bool { |
| 270 | true |
| 271 | } |
| 272 | |
| 273 | async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> { |
| 274 | self.execute_rich(input, context) |
| 275 | .await |
| 276 | .map(RichToolResult::into_result) |
| 277 | } |
| 278 | |
| 279 | async fn execute_rich( |
| 280 | &self, |
| 281 | input: Value, |
| 282 | context: &ToolContext, |
| 283 | ) -> Result<RichToolResult, ToolError> { |
| 284 | execute_read_media(input, context).await |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | /// Execute the `read_media` tool logic. |
| 289 | pub(crate) async fn execute_read_media( |
| 290 | input: Value, |
| 291 | context: &ToolContext, |
| 292 | ) -> Result<RichToolResult, ToolError> { |
| 293 | // 0. Check cancellation early (before any path resolution, capability checks, or I/O) |
| 294 | if context |
| 295 | .cancel_token |
| 296 | .as_ref() |
| 297 | .is_some_and(tokio_util::sync::CancellationToken::is_cancelled) |
| 298 | { |
| 299 | return Err(ToolError::cancelled("Operation aborted")); |
| 300 | } |
| 301 | |
| 302 | let path_str = required_str(&input, "path")?; |
| 303 | let detail_str = optional_str(&input, "detail")?; |
| 304 | let detail_mode = DetailMode::from_str_opt(detail_str)?; |
| 305 | let crop_region = CropRegion::parse_from_value(input.get("crop"))?; |
| 306 | |
| 307 | // 1. Check active route vision capability |
| 308 | if context.route_capabilities.image_input == CapabilityState::Unsupported { |
| 309 | return Err(ToolError::execution_failed( |
| 310 | "read_media: the active model route does not support image input. Switch to a route marked vision-capable with /model, or configure the route's image_input capability, then try again.", |
| 311 | )); |
| 312 | } |
| 313 | |
| 314 | // 2. Resolve path and protect credentials (including symlink/canonicalization escapes) |
| 315 | // S1/F2: the deny-list check on the caller's raw spelling runs BEFORE |
| 316 | // `resolve_path` — resolution canonicalizes a workspace symlink to the |
| 317 | // secret's real location, and an error raised only afterwards would name |
| 318 | // that location. `read_guard::check` canonicalizes internally, so the raw |
| 319 | // spelling still matches by target; the resolved check below remains as |
| 320 | // defense in depth. |
| 321 | crate::tools::file::enforce_read_denylist(std::path::Path::new(path_str), "read_media")?; |
| 322 | let file_path = match context.resolve_path(path_str) { |
| 323 | Ok(path) => path, |
| 324 | Err(primary_err) => { |
| 325 | // Read-back fallback for tool-owned stored originals: the store |
| 326 | // only holds image bytes that already passed the guards above, |
| 327 | // named by content hash, so admitting it widens nothing. |
| 328 | match context |
| 329 | .runtime |
| 330 | .media_originals_dir |
| 331 | .as_deref() |
| 332 | .and_then(|dir| { |
| 333 | crate::media_originals::resolve_stored_original( |
| 334 | path_str, |
| 335 | &context.workspace, |
| 336 | dir, |
| 337 | ) |
| 338 | }) { |
| 339 | Some(path) => path, |
| 340 | None => return Err(primary_err), |
| 341 | } |
| 342 | } |
| 343 | }; |
| 344 | if crate::tools::file::is_codewhale_credential_path(&file_path) { |
| 345 | return Err(ToolError::permission_denied( |
| 346 | "read_media cannot read Codewhale configuration or credential-store files; use `codewhale config list` or `codewhale auth status` for safe inspection", |
| 347 | )); |
| 348 | } |
| 349 | crate::tools::file::enforce_read_denylist(&file_path, "read_media")?; |
| 350 | |
| 351 | // Check cancellation immediately before dispatching blocking I/O and decode work |
| 352 | if context |
| 353 | .cancel_token |
| 354 | .as_ref() |
| 355 | .is_some_and(tokio_util::sync::CancellationToken::is_cancelled) |
| 356 | { |
| 357 | return Err(ToolError::cancelled("Operation aborted")); |
| 358 | } |
| 359 | |
| 360 | let file_path_clone = file_path.clone(); |
| 361 | let originals_dir = context.runtime.media_originals_dir.clone(); |
| 362 | let processed = tokio::task::spawn_blocking(move || { |
| 363 | process_media_file(&file_path_clone, crop_region, detail_mode, originals_dir) |
| 364 | }) |
| 365 | .await |
| 366 | .map_err(|join_err| { |
| 367 | ToolError::execution_failed(format!("read_media task failed: {join_err}")) |
| 368 | })??; |
| 369 | |
| 370 | // Check cancellation after await |
| 371 | if context |
| 372 | .cancel_token |
| 373 | .as_ref() |
| 374 | .is_some_and(tokio_util::sync::CancellationToken::is_cancelled) |
| 375 | { |
| 376 | return Err(ToolError::cancelled("Operation aborted")); |
| 377 | } |
| 378 | |
| 379 | // 10. Construct receipt and typed metadata without credentials |
| 380 | context.note_file_read(&file_path); |
| 381 | |
| 382 | let crop_summary = if let Some(crop) = crop_region { |
| 383 | format!( |
| 384 | "region [x: {}, y: {}, w: {}, h: {}]", |
| 385 | crop.x, crop.y, crop.width, crop.height |
| 386 | ) |
| 387 | } else { |
| 388 | "none".to_string() |
| 389 | }; |
| 390 | |
| 391 | let delivery_note = build_delivery_note(&processed, crop_region); |
| 392 | |
| 393 | let mut content_text = format!( |
| 394 | "Read media file: {path_str} [{delivered_mime}]\n\ |
| 395 | Original dimensions: {orig_width}x{orig_height} ({mime_type})\n\ |
| 396 | Processed dimensions: {final_width}x{final_height}\n\ |
| 397 | Crop: {crop_summary}\n\ |
| 398 | Detail: {}\n\ |
| 399 | Delivery: {delivery_note}\n\ |
| 400 | Size: {} (source) -> {} (wire payload)", |
| 401 | detail_mode.as_str(), |
| 402 | human_bytes(processed.source_bytes), |
| 403 | human_bytes(processed.encoded_bytes), |
| 404 | delivered_mime = processed.delivered_mime, |
| 405 | orig_width = processed.orig_width, |
| 406 | orig_height = processed.orig_height, |
| 407 | mime_type = processed.source_mime, |
| 408 | final_width = processed.final_width, |
| 409 | final_height = processed.final_height, |
| 410 | ); |
| 411 | if context.route_capabilities.image_input == CapabilityState::Unknown { |
| 412 | content_text.push_str( |
| 413 | "\nRoute image support is unverified; if the model cannot see this image, use image_ocr.", |
| 414 | ); |
| 415 | } |
| 416 | |
| 417 | let metadata_json = json!({ |
| 418 | "path": path_str, |
| 419 | "mime_type": processed.delivered_mime, |
| 420 | "original_width": processed.orig_width, |
| 421 | "original_height": processed.orig_height, |
| 422 | "original_mime_type": processed.source_mime, |
| 423 | "width": processed.final_width, |
| 424 | "height": processed.final_height, |
| 425 | "cropped": processed.crop_applied, |
| 426 | "crop": crop_region.map(|c| json!({ |
| 427 | "x": c.x, |
| 428 | "y": c.y, |
| 429 | "width": c.width, |
| 430 | "height": c.height |
| 431 | })), |
| 432 | "detail": detail_mode.as_str(), |
| 433 | "delivery": processed.delivery.as_str(), |
| 434 | "original_path": processed |
| 435 | .original_path |
| 436 | .as_ref() |
| 437 | .map(|p| p.display().to_string()), |
| 438 | "source_bytes": processed.source_bytes, |
| 439 | "encoded_bytes": processed.encoded_bytes |
| 440 | }); |
| 441 | |
| 442 | Ok(RichToolResult::with_content_blocks( |
| 443 | ToolResult::success(content_text).with_metadata(metadata_json), |
| 444 | vec![ToolResultContentBlock::Image { |
| 445 | mime_type: processed.delivered_mime.to_string(), |
| 446 | data: processed.base64_payload, |
| 447 | }], |
| 448 | )) |
| 449 | } |
| 450 | |
| 451 | /// The delivery-mode sentence of the result note: exactly how the image was |
| 452 | /// delivered, plus zoom/offset guidance. Compression is never silent. |
| 453 | fn build_delivery_note(processed: &ProcessedMedia, crop_region: Option<CropRegion>) -> String { |
| 454 | let delivered_size = human_bytes(processed.encoded_bytes); |
| 455 | match processed.delivery { |
| 456 | DeliveryMode::Untouched => format!( |
| 457 | "untouched — delivered at native resolution {}x{} ({}, {}); no downsampling applied.", |
| 458 | processed.final_width, processed.final_height, processed.delivered_mime, delivered_size |
| 459 | ), |
| 460 | DeliveryMode::Full => format!( |
| 461 | "full — shown at native resolution {}x{} ({}, {}); no downscaling applied.", |
| 462 | processed.final_width, processed.final_height, processed.delivered_mime, delivered_size |
| 463 | ), |
| 464 | DeliveryMode::Downsampled { resolution_lost } => { |
| 465 | let how = if resolution_lost { |
| 466 | format!( |
| 467 | "downsampled to {}x{}", |
| 468 | processed.final_width, processed.final_height |
| 469 | ) |
| 470 | } else { |
| 471 | format!( |
| 472 | "re-encoded at native resolution {}x{}", |
| 473 | processed.final_width, processed.final_height |
| 474 | ) |
| 475 | }; |
| 476 | let mut note = format!( |
| 477 | "{how} ({}, {delivered_size}) to fit model limits; fine detail may be lost. \ |
| 478 | To inspect fine detail, call read_media again with the crop parameter \ |
| 479 | (original-image pixel coordinates) to view a region at full fidelity.", |
| 480 | processed.delivered_mime |
| 481 | ); |
| 482 | if let Some(original_path) = &processed.original_path { |
| 483 | note.push_str(&format!( |
| 484 | " The uncompressed original is preserved at \"{}\".", |
| 485 | original_path.display() |
| 486 | )); |
| 487 | } |
| 488 | note |
| 489 | } |
| 490 | DeliveryMode::Crop { resized } => { |
| 491 | let region = crop_region.expect("crop delivery implies a crop region"); |
| 492 | format!( |
| 493 | "crop (x={}, y={}, width={}, height={}) of the original image{} ({}, {}). \ |
| 494 | To output coordinates in original-image pixels, locate them within this crop \ |
| 495 | and add the region offset (x={}, y={}).", |
| 496 | region.x, |
| 497 | region.y, |
| 498 | region.width, |
| 499 | region.height, |
| 500 | if resized { |
| 501 | format!( |
| 502 | ", downsampled to {}x{}", |
| 503 | processed.final_width, processed.final_height |
| 504 | ) |
| 505 | } else { |
| 506 | " at native resolution".to_string() |
| 507 | }, |
| 508 | processed.delivered_mime, |
| 509 | delivered_size, |
| 510 | region.x, |
| 511 | region.y |
| 512 | ) |
| 513 | } |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | /// How the delivered payload relates to the source image. |
| 518 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 519 | enum DeliveryMode { |
| 520 | /// Native resolution, first ladder step, `detail` `low`/`auto`. |
| 521 | Untouched, |
| 522 | /// Native resolution, first ladder step, `detail` `high`/`original`. |
| 523 | Full, |
| 524 | /// Resolution and/or encoding quality was reduced to fit the budget. |
| 525 | Downsampled { resolution_lost: bool }, |
| 526 | /// A crop region was delivered; `resized` when the crop itself had to |
| 527 | /// shrink to fit the budget. |
| 528 | Crop { resized: bool }, |
| 529 | } |
| 530 | |
| 531 | impl DeliveryMode { |
| 532 | fn as_str(self) -> &'static str { |
| 533 | match self { |
| 534 | Self::Untouched => "untouched", |
| 535 | Self::Full => "full", |
| 536 | Self::Downsampled { .. } => "downsampled", |
| 537 | Self::Crop { .. } => "crop", |
| 538 | } |
| 539 | } |
| 540 | } |
| 541 | |
| 542 | struct ProcessedMedia { |
| 543 | base64_payload: String, |
| 544 | delivered_mime: &'static str, |
| 545 | orig_width: u32, |
| 546 | orig_height: u32, |
| 547 | source_mime: &'static str, |
| 548 | final_width: u32, |
| 549 | final_height: u32, |
| 550 | crop_applied: bool, |
| 551 | delivery: DeliveryMode, |
| 552 | source_bytes: usize, |
| 553 | encoded_bytes: usize, |
| 554 | original_path: Option<std::path::PathBuf>, |
| 555 | } |
| 556 | |
| 557 | fn process_media_file( |
| 558 | file_path: &std::path::Path, |
| 559 | crop_region: Option<CropRegion>, |
| 560 | detail_mode: DetailMode, |
| 561 | originals_dir: Option<std::path::PathBuf>, |
| 562 | ) -> Result<ProcessedMedia, ToolError> { |
| 563 | // 3. Inspect metadata and check file bounds |
| 564 | if !file_path.exists() { |
| 565 | return Err(ToolError::execution_failed(format!( |
| 566 | "read_media: image file does not exist: {}", |
| 567 | file_path.display() |
| 568 | ))); |
| 569 | } |
| 570 | |
| 571 | let meta = std::fs::metadata(file_path).map_err(|e| { |
| 572 | ToolError::execution_failed(format!( |
| 573 | "read_media: failed to inspect {}: {e}", |
| 574 | file_path.display() |
| 575 | )) |
| 576 | })?; |
| 577 | |
| 578 | if meta.is_dir() { |
| 579 | return Err(ToolError::execution_failed(format!( |
| 580 | "read_media: path is a directory, not an image file: {}", |
| 581 | file_path.display() |
| 582 | ))); |
| 583 | } |
| 584 | |
| 585 | let file_len = meta.len(); |
| 586 | if file_len == 0 { |
| 587 | return Err(ToolError::execution_failed(format!( |
| 588 | "read_media: image file is empty (0 bytes): {}", |
| 589 | file_path.display() |
| 590 | ))); |
| 591 | } |
| 592 | |
| 593 | if file_len > MAX_SOURCE_IMAGE_BYTES as u64 { |
| 594 | return Err(ToolError::execution_failed(format!( |
| 595 | "read_media: image file size ({}) exceeds the maximum source limit of {}. Downscale or crop the file first.", |
| 596 | human_bytes(file_len as usize), |
| 597 | human_bytes(MAX_SOURCE_IMAGE_BYTES) |
| 598 | ))); |
| 599 | } |
| 600 | |
| 601 | // 4. Read source bytes |
| 602 | let raw_bytes = std::fs::read(file_path).map_err(|e| { |
| 603 | ToolError::execution_failed(format!( |
| 604 | "read_media: failed to read {}: {e}", |
| 605 | file_path.display() |
| 606 | )) |
| 607 | })?; |
| 608 | |
| 609 | if raw_bytes.is_empty() { |
| 610 | return Err(ToolError::execution_failed(format!( |
| 611 | "read_media: image file is empty (0 bytes): {}", |
| 612 | file_path.display() |
| 613 | ))); |
| 614 | } |
| 615 | |
| 616 | if raw_bytes.len() > MAX_SOURCE_IMAGE_BYTES { |
| 617 | return Err(ToolError::execution_failed(format!( |
| 618 | "read_media: image file size ({}) exceeds the maximum source limit of {}. Downscale or crop the file first.", |
| 619 | human_bytes(raw_bytes.len()), |
| 620 | human_bytes(MAX_SOURCE_IMAGE_BYTES) |
| 621 | ))); |
| 622 | } |
| 623 | |
| 624 | // 5. Sniff format and guard against non-images or rejected formats |
| 625 | let sniffed_mime = crate::image_attach::sniff_media_type(&raw_bytes); |
| 626 | let mime_type = match sniffed_mime { |
| 627 | Some(m) => m, |
| 628 | None => { |
| 629 | if let Some(rejected) = crate::image_attach::detect_rejected_format(&raw_bytes) { |
| 630 | return Err(ToolError::execution_failed(format!( |
| 631 | "read_media: {rejected} format is not directly supported by vision models. Convert it to PNG, JPEG, GIF, or WebP first." |
| 632 | ))); |
| 633 | } |
| 634 | return Err(ToolError::execution_failed(format!( |
| 635 | "read_media: file is not a recognized or supported image format (expected PNG, JPEG, GIF, or WebP): {}", |
| 636 | file_path.display() |
| 637 | ))); |
| 638 | } |
| 639 | }; |
| 640 | |
| 641 | // 6. Bounded decoding with decompression-bomb guards |
| 642 | let (processed_image, orig_width, orig_height) = decode_and_guard_image(&raw_bytes) |
| 643 | .map_err(|error| ToolError::execution_failed(format!("read_media: {error}")))?; |
| 644 | |
| 645 | // 7. Apply crop if requested |
| 646 | let (cropped_image, crop_applied) = if let Some(crop) = crop_region { |
| 647 | let crop_right = crop.x.checked_add(crop.width); |
| 648 | let crop_bottom = crop.y.checked_add(crop.height); |
| 649 | if crop_right.is_none_or(|right| right > orig_width) |
| 650 | || crop_bottom.is_none_or(|bottom| bottom > orig_height) |
| 651 | { |
| 652 | return Err(ToolError::invalid_input(format!( |
| 653 | "read_media: crop region [x: {}, y: {}, width: {}, height: {}] is out of bounds for image dimensions {}x{}", |
| 654 | crop.x, crop.y, crop.width, crop.height, orig_width, orig_height |
| 655 | ))); |
| 656 | } |
| 657 | let cropped = |
| 658 | image::imageops::crop_imm(&processed_image, crop.x, crop.y, crop.width, crop.height) |
| 659 | .to_image(); |
| 660 | (DynamicImage::ImageRgba8(cropped), true) |
| 661 | } else { |
| 662 | (processed_image, false) |
| 663 | }; |
| 664 | |
| 665 | // 8. Apply detail resolution resizing |
| 666 | let (current_w, current_h) = cropped_image.dimensions(); |
| 667 | let max_target = detail_mode.max_dimension(); |
| 668 | let detail_resized = current_w > max_target || current_h > max_target; |
| 669 | let fitted_image = if detail_resized { |
| 670 | cropped_image.resize(max_target, max_target, FilterType::Lanczos3) |
| 671 | } else { |
| 672 | cropped_image |
| 673 | }; |
| 674 | |
| 675 | // 9. Byte-budget-first encoding ladder: classify the content, then descend |
| 676 | // quality/edge rungs until the payload fits the delivery budget. Fails |
| 677 | // closed (nothing sent) with a conversion recipe when nothing fits. |
| 678 | let policy = classify_image(&fitted_image); |
| 679 | let budget = if crop_applied { |
| 680 | MAX_WIRE_IMAGE_BYTES |
| 681 | } else { |
| 682 | detail_mode.byte_budget() |
| 683 | }; |
| 684 | let outcome = encode_within_budget(&fitted_image, policy, budget, file_path)?; |
| 685 | |
| 686 | // 10. Persist the pre-compression original whenever the delivered copy is |
| 687 | // degraded, so a later crop read can pull the full-resolution source. |
| 688 | // Best effort: persistence never blocks delivery. |
| 689 | let degraded = detail_resized || outcome.resized || outcome.quality_reduced; |
| 690 | let original_path = if degraded && !crop_applied { |
| 691 | originals_dir.and_then(|dir| { |
| 692 | crate::media_originals::persist_original_image(&raw_bytes, mime_type, &dir) |
| 693 | }) |
| 694 | } else { |
| 695 | None |
| 696 | }; |
| 697 | |
| 698 | let delivery = if crop_applied { |
| 699 | DeliveryMode::Crop { resized: degraded } |
| 700 | } else if degraded { |
| 701 | DeliveryMode::Downsampled { |
| 702 | resolution_lost: detail_resized || outcome.resized, |
| 703 | } |
| 704 | } else { |
| 705 | match detail_mode { |
| 706 | DetailMode::High | DetailMode::Original => DeliveryMode::Full, |
| 707 | DetailMode::Low | DetailMode::Auto => DeliveryMode::Untouched, |
| 708 | } |
| 709 | }; |
| 710 | |
| 711 | let source_bytes = raw_bytes.len(); |
| 712 | let encoded_bytes = outcome.bytes.len(); |
| 713 | let base64_payload = BASE64.encode(&outcome.bytes); |
| 714 | |
| 715 | Ok(ProcessedMedia { |
| 716 | base64_payload, |
| 717 | delivered_mime: outcome.mime, |
| 718 | orig_width, |
| 719 | orig_height, |
| 720 | source_mime: mime_type, |
| 721 | final_width: outcome.width, |
| 722 | final_height: outcome.height, |
| 723 | crop_applied, |
| 724 | delivery, |
| 725 | source_bytes, |
| 726 | encoded_bytes, |
| 727 | original_path, |
| 728 | }) |
| 729 | } |
| 730 | |
| 731 | /// How the encoding ladder should treat an image's content. |
| 732 | #[derive(Debug, Clone, Copy, PartialEq, Eq)] |
| 733 | enum EncodePolicy { |
| 734 | /// Meaningful alpha present: PNG only. JPEG would silently destroy the |
| 735 | /// alpha channel, so the ladder fails closed rather than falling back. |
| 736 | RequireAlphaPng, |
| 737 | /// Opaque and flat (solid fills, line art, UI screenshots): PNG rungs |
| 738 | /// first — flat content compresses well losslessly and JPEG rings on hard |
| 739 | /// edges — then the JPEG ladder below the PNG edge floor. |
| 740 | PreferPng, |
| 741 | /// Opaque and photo-like: the JPEG quality ladder immediately. Photos as |
| 742 | /// PNG waste an order of magnitude of context bytes. |
| 743 | PreferJpeg, |
| 744 | } |
| 745 | |
| 746 | impl EncodePolicy { |
| 747 | fn prefers_png(self) -> bool { |
| 748 | !matches!(self, Self::PreferJpeg) |
| 749 | } |
| 750 | } |
| 751 | |
| 752 | /// Content classification for the ladder: alpha first (correctness), then a |
| 753 | /// colorfulness heuristic on a small thumbnail (cost). |
| 754 | fn classify_image(image: &DynamicImage) -> EncodePolicy { |
| 755 | if has_meaningful_alpha(image) { |
| 756 | return EncodePolicy::RequireAlphaPng; |
| 757 | } |
| 758 | if colorfulness_sigma(image) < FLAT_COLORFULNESS_THRESHOLD { |
| 759 | EncodePolicy::PreferPng |
| 760 | } else { |
| 761 | EncodePolicy::PreferJpeg |
| 762 | } |
| 763 | } |
| 764 | |
| 765 | /// Whether any pixel is actually transparent — not merely whether the decoded |
| 766 | /// layout has an alpha channel (an opaque RGBA PNG decodes to `Rgba8` too). |
| 767 | fn has_meaningful_alpha(image: &DynamicImage) -> bool { |
| 768 | match image { |
| 769 | DynamicImage::ImageRgba8(buf) => buf.pixels().any(|p| p.0[3] != u8::MAX), |
| 770 | DynamicImage::ImageLumaA8(buf) => buf.pixels().any(|p| p.0[1] != u8::MAX), |
| 771 | DynamicImage::ImageRgba16(buf) => buf.pixels().any(|p| p.0[3] != u16::MAX), |
| 772 | DynamicImage::ImageLumaA16(buf) => buf.pixels().any(|p| p.0[1] != u16::MAX), |
| 773 | _ => false, |
| 774 | } |
| 775 | } |
| 776 | |
| 777 | /// Spread of the opponent color channels (Hasler–Süsstrunk sigma terms only — |
| 778 | /// the mean terms would inflate saturated flat fills), measured on a ≤128 px |
| 779 | /// thumbnail for speed. Solid fills and line art score ~0. |
| 780 | fn colorfulness_sigma(image: &DynamicImage) -> f64 { |
| 781 | let thumb = image.thumbnail(128, 128); |
| 782 | let rgb = thumb.to_rgb8(); |
| 783 | let n = (rgb.len() / 3) as f64; |
| 784 | if n == 0.0 { |
| 785 | return 0.0; |
| 786 | } |
| 787 | let (mut sum_rg, mut sum_yb, mut sum_rg2, mut sum_yb2) = (0.0f64, 0.0f64, 0.0f64, 0.0f64); |
| 788 | // An `Rgb<u8>` buffer is exactly 3 bytes per pixel, so the remainder is |
| 789 | // always empty; `as_chunks` makes the pixel stride a compile-time fact. |
| 790 | let (pixels, remainder) = rgb.as_raw().as_chunks::<3>(); |
| 791 | debug_assert!( |
| 792 | remainder.is_empty(), |
| 793 | "RGB8 buffer length must be a multiple of 3" |
| 794 | ); |
| 795 | for &[r, g, b] in pixels { |
| 796 | let (r, g, b) = (f64::from(r), f64::from(g), f64::from(b)); |
| 797 | let rg = r - g; |
| 798 | let yb = (r + g) / 2.0 - b; |
| 799 | sum_rg += rg; |
| 800 | sum_yb += yb; |
| 801 | sum_rg2 += rg * rg; |
| 802 | sum_yb2 += yb * yb; |
| 803 | } |
| 804 | let var_rg = (sum_rg2 / n - (sum_rg / n).powi(2)).max(0.0); |
| 805 | let var_yb = (sum_yb2 / n - (sum_yb / n).powi(2)).max(0.0); |
| 806 | (var_rg + var_yb).sqrt() |
| 807 | } |
| 808 | |
| 809 | /// One encoded ladder candidate. |
| 810 | struct EncodedImage { |
| 811 | bytes: Vec<u8>, |
| 812 | mime: &'static str, |
| 813 | width: u32, |
| 814 | height: u32, |
| 815 | } |
| 816 | |
| 817 | /// The ladder's winning candidate plus how it got there. |
| 818 | #[derive(Debug)] |
| 819 | struct LadderOutcome { |
| 820 | bytes: Vec<u8>, |
| 821 | mime: &'static str, |
| 822 | width: u32, |
| 823 | height: u32, |
| 824 | /// The candidate's dimensions differ from the ladder's input image. |
| 825 | resized: bool, |
| 826 | /// A JPEG quality step below the ladder's first (80) was needed. |
| 827 | quality_reduced: bool, |
| 828 | } |
| 829 | |
| 830 | fn encode_png_best(image: &DynamicImage) -> Result<EncodedImage, ToolError> { |
| 831 | let (width, height) = image.dimensions(); |
| 832 | let mut bytes: Vec<u8> = Vec::new(); |
| 833 | let encoder = |
| 834 | PngEncoder::new_with_quality(&mut bytes, CompressionType::Best, PngFilter::Adaptive); |
| 835 | if image.color().has_alpha() { |
| 836 | let rgba = image.to_rgba8(); |
| 837 | encoder |
| 838 | .write_image(rgba.as_raw(), width, height, ExtendedColorType::Rgba8) |
| 839 | .map_err(|e| { |
| 840 | ToolError::execution_failed(format!("read_media: failed to encode PNG: {e}")) |
| 841 | })?; |
| 842 | } else { |
| 843 | let rgb = image.to_rgb8(); |
| 844 | encoder |
| 845 | .write_image(rgb.as_raw(), width, height, ExtendedColorType::Rgb8) |
| 846 | .map_err(|e| { |
| 847 | ToolError::execution_failed(format!("read_media: failed to encode PNG: {e}")) |
| 848 | })?; |
| 849 | } |
| 850 | Ok(EncodedImage { |
| 851 | bytes, |
| 852 | mime: "image/png", |
| 853 | width, |
| 854 | height, |
| 855 | }) |
| 856 | } |
| 857 | |
| 858 | fn encode_jpeg(image: &DynamicImage, quality: u8) -> Result<EncodedImage, ToolError> { |
| 859 | let (width, height) = image.dimensions(); |
| 860 | // Only reached for images without meaningful alpha, so dropping the |
| 861 | // channel changes nothing visible. |
| 862 | let rgb = image.to_rgb8(); |
| 863 | let mut bytes: Vec<u8> = Vec::new(); |
| 864 | JpegEncoder::new_with_quality(&mut bytes, quality) |
| 865 | .write_image(rgb.as_raw(), width, height, ExtendedColorType::Rgb8) |
| 866 | .map_err(|e| { |
| 867 | ToolError::execution_failed(format!("read_media: failed to encode JPEG: {e}")) |
| 868 | })?; |
| 869 | Ok(EncodedImage { |
| 870 | bytes, |
| 871 | mime: "image/jpeg", |
| 872 | width, |
| 873 | height, |
| 874 | }) |
| 875 | } |
| 876 | |
| 877 | /// Half-size Lanczos copy, or `None` once the longest edge is at the floor. |
| 878 | fn halved(image: &DynamicImage) -> Option<DynamicImage> { |
| 879 | let (w, h) = image.dimensions(); |
| 880 | if w.max(h) <= MIN_DELIVERY_EDGE_PX { |
| 881 | return None; |
| 882 | } |
| 883 | Some(image.resize((w / 2).max(1), (h / 2).max(1), FilterType::Lanczos3)) |
| 884 | } |
| 885 | |
| 886 | fn track_smallest(smallest: &mut Option<EncodedImage>, candidate: EncodedImage) { |
| 887 | if smallest |
| 888 | .as_ref() |
| 889 | .is_none_or(|s| candidate.bytes.len() < s.bytes.len()) |
| 890 | { |
| 891 | *smallest = Some(candidate); |
| 892 | } |
| 893 | } |
| 894 | |
| 895 | /// Descend the encoding ladder until a rung fits `budget`. |
| 896 | /// |
| 897 | /// Rung order (mirrors kimi-code's `encodeWithinBudget`): PNG-preferring |
| 898 | /// images try best-compression PNG at the current size, then halved PNG rungs |
| 899 | /// down to [`PNG_EDGE_FLOOR_PX`], then the JPEG quality ladder; photo-like |
| 900 | /// images start at the JPEG ladder; alpha-bearing images only ever see PNG |
| 901 | /// rungs. JPEG rungs repeat the quality ladder at each halved edge down to |
| 902 | /// [`MIN_DELIVERY_EDGE_PX`]. The first fitting rung wins (highest fidelity |
| 903 | /// within budget). When nothing fits, fails closed — nothing is sent — with |
| 904 | /// the exact conversion command to retry with. |
| 905 | fn encode_within_budget( |
| 906 | image: &DynamicImage, |
| 907 | policy: EncodePolicy, |
| 908 | budget: usize, |
| 909 | path: &std::path::Path, |
| 910 | ) -> Result<LadderOutcome, ToolError> { |
| 911 | let (start_w, start_h) = image.dimensions(); |
| 912 | let mut smallest: Option<EncodedImage> = None; |
| 913 | let mut candidate = image.clone(); |
| 914 | |
| 915 | if policy.prefers_png() { |
| 916 | let png_floor = match policy { |
| 917 | EncodePolicy::RequireAlphaPng => MIN_DELIVERY_EDGE_PX, |
| 918 | _ => PNG_EDGE_FLOOR_PX, |
| 919 | }; |
| 920 | loop { |
| 921 | let encoded = encode_png_best(&candidate)?; |
| 922 | if encoded.bytes.len() <= budget { |
| 923 | let resized = encoded.width != start_w || encoded.height != start_h; |
| 924 | return Ok(LadderOutcome { |
| 925 | bytes: encoded.bytes, |
| 926 | mime: encoded.mime, |
| 927 | width: encoded.width, |
| 928 | height: encoded.height, |
| 929 | resized, |
| 930 | quality_reduced: false, |
| 931 | }); |
| 932 | } |
| 933 | let longest = encoded.width.max(encoded.height); |
| 934 | track_smallest(&mut smallest, encoded); |
| 935 | if longest <= png_floor { |
| 936 | break; |
| 937 | } |
| 938 | let Some(next) = halved(&candidate) else { |
| 939 | break; |
| 940 | }; |
| 941 | candidate = next; |
| 942 | } |
| 943 | if policy == EncodePolicy::RequireAlphaPng { |
| 944 | return Err(over_budget_error(path, budget, smallest)); |
| 945 | } |
| 946 | } |
| 947 | |
| 948 | loop { |
| 949 | for (step, quality) in JPEG_QUALITY_LADDER.iter().enumerate() { |
| 950 | let encoded = encode_jpeg(&candidate, *quality)?; |
| 951 | if encoded.bytes.len() <= budget { |
| 952 | let resized = encoded.width != start_w || encoded.height != start_h; |
| 953 | return Ok(LadderOutcome { |
| 954 | bytes: encoded.bytes, |
| 955 | mime: encoded.mime, |
| 956 | width: encoded.width, |
| 957 | height: encoded.height, |
| 958 | resized, |
| 959 | quality_reduced: step > 0, |
| 960 | }); |
| 961 | } |
| 962 | track_smallest(&mut smallest, encoded); |
| 963 | } |
| 964 | let Some(next) = halved(&candidate) else { |
| 965 | break; |
| 966 | }; |
| 967 | candidate = next; |
| 968 | } |
| 969 | Err(over_budget_error(path, budget, smallest)) |
| 970 | } |
| 971 | |
| 972 | /// Fail-closed error naming the exact conversion recipe to retry with. |
| 973 | fn over_budget_error( |
| 974 | path: &std::path::Path, |
| 975 | budget: usize, |
| 976 | smallest: Option<EncodedImage>, |
| 977 | ) -> ToolError { |
| 978 | let smallest_desc = match &smallest { |
| 979 | Some(s) => format!( |
| 980 | "smallest deliverable: {} at {}x{}", |
| 981 | human_bytes(s.bytes.len()), |
| 982 | s.width, |
| 983 | s.height |
| 984 | ), |
| 985 | None => "no encodable result".to_string(), |
| 986 | }; |
| 987 | ToolError::execution_failed(format!( |
| 988 | "read_media: {path} could not be compressed under the {budget} delivery budget ({smallest_desc}). \ |
| 989 | Nothing was sent to the model; do not retry the same file unchanged. \ |
| 990 | Create a smaller copy and read that instead: \ |
| 991 | `sips -Z 1024 \"{path}\" --out /tmp/codewhale-smaller.png` (macOS) or \ |
| 992 | `magick \"{path}\" -resize 1024x1024 /tmp/codewhale-smaller.png` (ImageMagick); \ |
| 993 | for a photo-like image, JPEG output (`--out /tmp/codewhale-smaller.jpg` / `-quality 70`) \ |
| 994 | compresses further. Then call read_media on the smaller copy.", |
| 995 | path = path.display(), |
| 996 | budget = human_bytes(budget), |
| 997 | )) |
| 998 | } |
| 999 | |
| 1000 | fn human_bytes(bytes: usize) -> String { |
| 1001 | if bytes >= 1024 * 1024 { |
| 1002 | format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) |
| 1003 | } else if bytes >= 1024 { |
| 1004 | format!("{:.1} KB", bytes as f64 / 1024.0) |
| 1005 | } else { |
| 1006 | format!("{bytes} bytes") |
| 1007 | } |
| 1008 | } |
| 1009 | |
| 1010 | #[cfg(test)] |
| 1011 | mod tests { |
| 1012 | use super::*; |
| 1013 | use codewhale_models::Role; |
| 1014 | use image::ImageFormat; |
| 1015 | use tempfile::tempdir; |
| 1016 | |
| 1017 | fn create_test_png(width: u32, height: u32, color: [u8; 4]) -> Vec<u8> { |
| 1018 | let img = image::RgbaImage::from_pixel(width, height, image::Rgba(color)); |
| 1019 | let mut cursor = Cursor::new(Vec::new()); |
| 1020 | img.write_to(&mut cursor, ImageFormat::Png).unwrap(); |
| 1021 | cursor.into_inner() |
| 1022 | } |
| 1023 | |
| 1024 | fn create_test_jpeg(width: u32, height: u32) -> Vec<u8> { |
| 1025 | let img = image::RgbImage::from_pixel(width, height, image::Rgb([120, 200, 50])); |
| 1026 | let mut cursor = Cursor::new(Vec::new()); |
| 1027 | img.write_to(&mut cursor, ImageFormat::Jpeg).unwrap(); |
| 1028 | cursor.into_inner() |
| 1029 | } |
| 1030 | |
| 1031 | #[tokio::test] |
| 1032 | async fn read_media_spec_metadata_and_capabilities() { |
| 1033 | let tool = ReadMediaTool; |
| 1034 | assert_eq!(tool.name(), "read_media"); |
| 1035 | assert!(tool.capabilities().contains(&ToolCapability::ReadOnly)); |
| 1036 | assert!(tool.capabilities().contains(&ToolCapability::Sandboxable)); |
| 1037 | assert!(tool.supports_parallel()); |
| 1038 | assert!(tool.defer_loading()); |
| 1039 | assert_eq!(tool.approval_requirement(), ApprovalRequirement::Auto); |
| 1040 | } |
| 1041 | |
| 1042 | #[tokio::test] |
| 1043 | async fn read_media_success_png_roundtrip() { |
| 1044 | let dir = tempdir().unwrap(); |
| 1045 | let img_path = dir.path().join("test.png"); |
| 1046 | let png_data = create_test_png(100, 50, [255, 0, 0, 255]); |
| 1047 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1048 | |
| 1049 | let mut ctx = ToolContext::new(dir.path()); |
| 1050 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1051 | |
| 1052 | let tool = ReadMediaTool; |
| 1053 | let input = json!({ |
| 1054 | "path": "test.png", |
| 1055 | "detail": "auto" |
| 1056 | }); |
| 1057 | |
| 1058 | let rich = tool.execute_rich(input, &ctx).await.unwrap(); |
| 1059 | assert!(rich.success); |
| 1060 | assert_eq!(rich.content_blocks.len(), 1); |
| 1061 | let ToolResultContentBlock::Image { mime_type, data } = &rich.content_blocks[0]; |
| 1062 | assert_eq!(mime_type, "image/png"); |
| 1063 | assert!(!data.is_empty()); |
| 1064 | |
| 1065 | let meta = rich.metadata.as_ref().unwrap(); |
| 1066 | assert_eq!(meta["path"], "test.png"); |
| 1067 | assert_eq!(meta["original_width"], 100); |
| 1068 | assert_eq!(meta["original_height"], 50); |
| 1069 | assert_eq!(meta["width"], 100); |
| 1070 | assert_eq!(meta["height"], 50); |
| 1071 | assert_eq!(meta["cropped"], false); |
| 1072 | assert_eq!(meta["detail"], "auto"); |
| 1073 | } |
| 1074 | |
| 1075 | #[tokio::test] |
| 1076 | async fn read_media_success_jpeg_decoded_and_reencoded_to_png() { |
| 1077 | let dir = tempdir().unwrap(); |
| 1078 | let img_path = dir.path().join("photo.jpg"); |
| 1079 | let jpeg_data = create_test_jpeg(80, 60); |
| 1080 | std::fs::write(&img_path, &jpeg_data).unwrap(); |
| 1081 | |
| 1082 | let mut ctx = ToolContext::new(dir.path()); |
| 1083 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1084 | |
| 1085 | let tool = ReadMediaTool; |
| 1086 | let input = json!({ |
| 1087 | "path": "photo.jpg", |
| 1088 | "detail": "low" |
| 1089 | }); |
| 1090 | |
| 1091 | let rich = tool.execute_rich(input, &ctx).await.unwrap(); |
| 1092 | assert!(rich.success); |
| 1093 | let ToolResultContentBlock::Image { mime_type, data } = &rich.content_blocks[0]; |
| 1094 | assert_eq!(mime_type, "image/png"); |
| 1095 | assert!(!data.is_empty()); |
| 1096 | |
| 1097 | let meta = rich.metadata.as_ref().unwrap(); |
| 1098 | assert_eq!(meta["original_width"], 80); |
| 1099 | assert_eq!(meta["original_height"], 60); |
| 1100 | assert_eq!(meta["original_mime_type"], "image/jpeg"); |
| 1101 | } |
| 1102 | |
| 1103 | #[tokio::test] |
| 1104 | async fn read_media_crop_region_bounds_and_execution() { |
| 1105 | let dir = tempdir().unwrap(); |
| 1106 | let img_path = dir.path().join("grid.png"); |
| 1107 | let png_data = create_test_png(200, 100, [0, 255, 0, 255]); |
| 1108 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1109 | |
| 1110 | let mut ctx = ToolContext::new(dir.path()); |
| 1111 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1112 | |
| 1113 | let tool = ReadMediaTool; |
| 1114 | |
| 1115 | // 1. Valid crop |
| 1116 | let valid_input = json!({ |
| 1117 | "path": "grid.png", |
| 1118 | "crop": { |
| 1119 | "x": 10, |
| 1120 | "y": 20, |
| 1121 | "width": 50, |
| 1122 | "height": 40 |
| 1123 | } |
| 1124 | }); |
| 1125 | let rich = tool.execute_rich(valid_input, &ctx).await.unwrap(); |
| 1126 | assert!(rich.success); |
| 1127 | let meta = rich.metadata.as_ref().unwrap(); |
| 1128 | assert_eq!(meta["cropped"], true); |
| 1129 | assert_eq!(meta["width"], 50); |
| 1130 | assert_eq!(meta["height"], 40); |
| 1131 | |
| 1132 | // 2. Out-of-bounds crop |
| 1133 | let oob_input = json!({ |
| 1134 | "path": "grid.png", |
| 1135 | "crop": { |
| 1136 | "x": 180, |
| 1137 | "y": 20, |
| 1138 | "width": 50, |
| 1139 | "height": 40 |
| 1140 | } |
| 1141 | }); |
| 1142 | let err = tool.execute_rich(oob_input, &ctx).await.unwrap_err(); |
| 1143 | assert!(err.to_string().contains("out of bounds")); |
| 1144 | |
| 1145 | // 3. Zero-dimension crop |
| 1146 | let zero_input = json!({ |
| 1147 | "path": "grid.png", |
| 1148 | "crop": { |
| 1149 | "x": 10, |
| 1150 | "y": 10, |
| 1151 | "width": 0, |
| 1152 | "height": 10 |
| 1153 | } |
| 1154 | }); |
| 1155 | let err = tool.execute_rich(zero_input, &ctx).await.unwrap_err(); |
| 1156 | assert!(err.to_string().contains("greater than 0")); |
| 1157 | |
| 1158 | // Adversarial coordinates must be rejected rather than wrapping in |
| 1159 | // release builds or panicking in debug builds. |
| 1160 | for overflow_input in [ |
| 1161 | json!({ |
| 1162 | "path": "grid.png", |
| 1163 | "crop": { |
| 1164 | "x": u32::MAX, |
| 1165 | "y": 0, |
| 1166 | "width": 2, |
| 1167 | "height": 1 |
| 1168 | } |
| 1169 | }), |
| 1170 | json!({ |
| 1171 | "path": "grid.png", |
| 1172 | "crop": { |
| 1173 | "x": 0, |
| 1174 | "y": u32::MAX, |
| 1175 | "width": 1, |
| 1176 | "height": 2 |
| 1177 | } |
| 1178 | }), |
| 1179 | ] { |
| 1180 | let err = tool.execute_rich(overflow_input, &ctx).await.unwrap_err(); |
| 1181 | assert!(err.to_string().contains("out of bounds"), "{err}"); |
| 1182 | } |
| 1183 | } |
| 1184 | |
| 1185 | #[tokio::test] |
| 1186 | async fn read_media_detail_modes_and_resizing() { |
| 1187 | let dir = tempdir().unwrap(); |
| 1188 | let img_path = dir.path().join("large.png"); |
| 1189 | let png_data = create_test_png(3000, 1500, [0, 0, 255, 255]); |
| 1190 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1191 | |
| 1192 | let mut ctx = ToolContext::new(dir.path()); |
| 1193 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1194 | let tool = ReadMediaTool; |
| 1195 | |
| 1196 | // Low detail -> max 1024 |
| 1197 | let low_input = json!({ "path": "large.png", "detail": "low" }); |
| 1198 | let rich_low = tool.execute_rich(low_input, &ctx).await.unwrap(); |
| 1199 | let meta_low = rich_low.metadata.as_ref().unwrap(); |
| 1200 | assert_eq!(meta_low["width"], 1024); |
| 1201 | assert_eq!(meta_low["height"], 512); |
| 1202 | |
| 1203 | // Auto detail -> max 2048 |
| 1204 | let auto_input = json!({ "path": "large.png", "detail": "auto" }); |
| 1205 | let rich_auto = tool.execute_rich(auto_input, &ctx).await.unwrap(); |
| 1206 | let meta_auto = rich_auto.metadata.as_ref().unwrap(); |
| 1207 | assert_eq!(meta_auto["width"], 2048); |
| 1208 | assert_eq!(meta_auto["height"], 1024); |
| 1209 | |
| 1210 | // Original detail -> within 4096, keeps 3000x1500 |
| 1211 | let orig_input = json!({ "path": "large.png", "detail": "original" }); |
| 1212 | let rich_orig = tool.execute_rich(orig_input, &ctx).await.unwrap(); |
| 1213 | let meta_orig = rich_orig.metadata.as_ref().unwrap(); |
| 1214 | assert_eq!(meta_orig["width"], 3000); |
| 1215 | assert_eq!(meta_orig["height"], 1500); |
| 1216 | } |
| 1217 | |
| 1218 | #[tokio::test] |
| 1219 | async fn read_media_route_capability_unsupported_fails_actionable() { |
| 1220 | let dir = tempdir().unwrap(); |
| 1221 | let img_path = dir.path().join("img.png"); |
| 1222 | let png_data = create_test_png(10, 10, [1, 2, 3, 255]); |
| 1223 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1224 | |
| 1225 | let mut ctx = ToolContext::new(dir.path()); |
| 1226 | ctx.route_capabilities.image_input = CapabilityState::Unsupported; |
| 1227 | |
| 1228 | let tool = ReadMediaTool; |
| 1229 | let input = json!({ "path": "img.png" }); |
| 1230 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1231 | let err_msg = err.to_string(); |
| 1232 | assert!(err_msg.contains("active model route does not support image input")); |
| 1233 | assert!(err_msg.contains("/model")); |
| 1234 | assert!(err_msg.contains("image_input")); |
| 1235 | assert!(!err_msg.contains("deepseek-v4-pro")); |
| 1236 | } |
| 1237 | |
| 1238 | #[tokio::test] |
| 1239 | async fn read_media_missing_file_fails_actionable() { |
| 1240 | let dir = tempdir().unwrap(); |
| 1241 | let ctx = ToolContext::new(dir.path()); |
| 1242 | let tool = ReadMediaTool; |
| 1243 | let input = json!({ "path": "nonexistent.png" }); |
| 1244 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1245 | assert!(err.to_string().contains("image file does not exist")); |
| 1246 | } |
| 1247 | |
| 1248 | #[tokio::test] |
| 1249 | async fn read_media_empty_file_fails_actionable() { |
| 1250 | let dir = tempdir().unwrap(); |
| 1251 | let img_path = dir.path().join("empty.png"); |
| 1252 | std::fs::write(&img_path, b"").unwrap(); |
| 1253 | |
| 1254 | let ctx = ToolContext::new(dir.path()); |
| 1255 | let tool = ReadMediaTool; |
| 1256 | let input = json!({ "path": "empty.png" }); |
| 1257 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1258 | assert!(err.to_string().contains("image file is empty")); |
| 1259 | } |
| 1260 | |
| 1261 | #[tokio::test] |
| 1262 | async fn read_media_corrupted_or_non_image_fails_actionable() { |
| 1263 | let dir = tempdir().unwrap(); |
| 1264 | let bad_path = dir.path().join("corrupted.png"); |
| 1265 | std::fs::write(&bad_path, b"not a real image payload at all").unwrap(); |
| 1266 | |
| 1267 | let ctx = ToolContext::new(dir.path()); |
| 1268 | let tool = ReadMediaTool; |
| 1269 | let input = json!({ "path": "corrupted.png" }); |
| 1270 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1271 | assert!( |
| 1272 | err.to_string() |
| 1273 | .contains("not a recognized or supported image format") |
| 1274 | ); |
| 1275 | } |
| 1276 | |
| 1277 | #[tokio::test] |
| 1278 | async fn read_media_rejected_format_svg_fails_with_conversion_hint() { |
| 1279 | let dir = tempdir().unwrap(); |
| 1280 | let svg_path = dir.path().join("vector.svg"); |
| 1281 | std::fs::write(&svg_path, b"<svg><circle r='10'/></svg>").unwrap(); |
| 1282 | |
| 1283 | let ctx = ToolContext::new(dir.path()); |
| 1284 | let tool = ReadMediaTool; |
| 1285 | let input = json!({ "path": "vector.svg" }); |
| 1286 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1287 | let msg = err.to_string(); |
| 1288 | assert!(msg.contains("SVG format is not directly supported by vision models")); |
| 1289 | assert!(msg.contains("Convert it to PNG, JPEG, GIF, or WebP")); |
| 1290 | } |
| 1291 | |
| 1292 | #[allow(clippy::await_holding_lock)] |
| 1293 | #[tokio::test] |
| 1294 | async fn read_media_credential_path_is_denied() { |
| 1295 | let _env_lock = crate::test_support::lock_test_env(); |
| 1296 | let tmp = tempdir().expect("tempdir"); |
| 1297 | let _codewhale_home = crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", tmp.path()); |
| 1298 | let _config_path = crate::test_support::EnvVarGuard::remove("CODEWHALE_CONFIG_PATH"); |
| 1299 | let _legacy_config_path = crate::test_support::EnvVarGuard::remove("DEEPSEEK_CONFIG_PATH"); |
| 1300 | |
| 1301 | std::fs::write(tmp.path().join("config.toml"), "api_key = \"secret\"\n") |
| 1302 | .expect("write config"); |
| 1303 | |
| 1304 | let ctx = ToolContext::new(tmp.path().to_path_buf()); |
| 1305 | let tool = ReadMediaTool; |
| 1306 | let input = json!({ "path": "config.toml" }); |
| 1307 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1308 | assert!( |
| 1309 | err.to_string() |
| 1310 | .contains("cannot read Codewhale configuration or credential-store"), |
| 1311 | "{}", |
| 1312 | err |
| 1313 | ); |
| 1314 | } |
| 1315 | |
| 1316 | #[allow(clippy::await_holding_lock)] |
| 1317 | #[cfg(unix)] |
| 1318 | #[tokio::test] |
| 1319 | async fn read_media_credential_symlink_escape_is_denied() { |
| 1320 | let _env_lock = crate::test_support::lock_test_env(); |
| 1321 | let home_tmp = tempdir().expect("home tempdir"); |
| 1322 | let ws_tmp = tempdir().expect("workspace tempdir"); |
| 1323 | let _codewhale_home = |
| 1324 | crate::test_support::EnvVarGuard::set("CODEWHALE_HOME", home_tmp.path()); |
| 1325 | let _config_path = crate::test_support::EnvVarGuard::remove("CODEWHALE_CONFIG_PATH"); |
| 1326 | let _legacy_config_path = crate::test_support::EnvVarGuard::remove("DEEPSEEK_CONFIG_PATH"); |
| 1327 | |
| 1328 | let real_config = home_tmp.path().join("config.toml"); |
| 1329 | std::fs::write(&real_config, "api_key = \"secret_in_home\"\n").expect("write config"); |
| 1330 | |
| 1331 | // Create a symlink in the workspace pointing to the credentials |
| 1332 | let symlink_path = ws_tmp.path().join("fake_image.png"); |
| 1333 | std::os::unix::fs::symlink(&real_config, &symlink_path).expect("create symlink"); |
| 1334 | |
| 1335 | { |
| 1336 | // Case 1: Default context (no symlink follow outside workspace) -> path escape error |
| 1337 | let ctx_default = ToolContext::new(ws_tmp.path().to_path_buf()); |
| 1338 | let tool = ReadMediaTool; |
| 1339 | let input = json!({ "path": "fake_image.png" }); |
| 1340 | let err_default = tool |
| 1341 | .execute_rich(input.clone(), &ctx_default) |
| 1342 | .await |
| 1343 | .unwrap_err(); |
| 1344 | let msg_default = err_default.to_string(); |
| 1345 | assert!( |
| 1346 | msg_default.contains("escapes workspace") || msg_default.contains("credential"), |
| 1347 | "default policy must reject symlink outside workspace: {msg_default}" |
| 1348 | ); |
| 1349 | |
| 1350 | // Case 2: follow_symlinks enabled -> credential guard must still deny read |
| 1351 | let ctx_follow = |
| 1352 | ToolContext::new(ws_tmp.path().to_path_buf()).with_follow_symlinks(true); |
| 1353 | let err_follow = tool.execute_rich(input, &ctx_follow).await.unwrap_err(); |
| 1354 | assert!( |
| 1355 | err_follow |
| 1356 | .to_string() |
| 1357 | .contains("cannot read Codewhale configuration or credential-store"), |
| 1358 | "follow_symlinks policy must catch canonical credential path: {}", |
| 1359 | err_follow |
| 1360 | ); |
| 1361 | } |
| 1362 | } |
| 1363 | |
| 1364 | #[tokio::test] |
| 1365 | async fn read_media_path_escape_is_rejected() { |
| 1366 | let dir = tempdir().unwrap(); |
| 1367 | let ctx = ToolContext::new(dir.path()); |
| 1368 | let tool = ReadMediaTool; |
| 1369 | let input = json!({ "path": "../../etc/shadow" }); |
| 1370 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1371 | assert!( |
| 1372 | err.to_string().contains("escapes workspace") || err.to_string().contains("permission"), |
| 1373 | "{}", |
| 1374 | err |
| 1375 | ); |
| 1376 | } |
| 1377 | |
| 1378 | #[tokio::test] |
| 1379 | async fn read_media_respects_cancel_token() { |
| 1380 | let dir = tempdir().unwrap(); |
| 1381 | let img_path = dir.path().join("cancel.png"); |
| 1382 | let png_data = create_test_png(10, 10, [1, 2, 3, 255]); |
| 1383 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1384 | |
| 1385 | let cancel_token = tokio_util::sync::CancellationToken::new(); |
| 1386 | cancel_token.cancel(); |
| 1387 | let mut ctx = ToolContext::new(dir.path()).with_cancel_token(cancel_token); |
| 1388 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1389 | |
| 1390 | let tool = ReadMediaTool; |
| 1391 | let err = tool |
| 1392 | .execute_rich(json!({ "path": "cancel.png" }), &ctx) |
| 1393 | .await |
| 1394 | .unwrap_err(); |
| 1395 | let msg = err.to_string(); |
| 1396 | assert!(msg.contains("aborted") || msg.contains("cancel"), "{msg}"); |
| 1397 | } |
| 1398 | |
| 1399 | #[tokio::test] |
| 1400 | async fn read_media_cancellation_before_dispatch_fails_without_read() { |
| 1401 | let dir = tempdir().unwrap(); |
| 1402 | let cancel_token = tokio_util::sync::CancellationToken::new(); |
| 1403 | cancel_token.cancel(); |
| 1404 | let mut ctx = ToolContext::new(dir.path()).with_cancel_token(cancel_token); |
| 1405 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1406 | |
| 1407 | let tool = ReadMediaTool; |
| 1408 | // A non-existent file path would normally fail with "image file does not exist", |
| 1409 | // but when cancelled before dispatch, it must abort with Cancelled without touching disk. |
| 1410 | let input = json!({ "path": "nonexistent_before_dispatch.png" }); |
| 1411 | let err = tool.execute_rich(input, &ctx).await.unwrap_err(); |
| 1412 | let msg = err.to_string(); |
| 1413 | assert!( |
| 1414 | msg.contains("aborted") || msg.contains("cancel"), |
| 1415 | "expected cancellation error before dispatch, got: {msg}" |
| 1416 | ); |
| 1417 | assert!( |
| 1418 | !msg.contains("does not exist"), |
| 1419 | "should not reach filesystem checks when cancelled before dispatch" |
| 1420 | ); |
| 1421 | } |
| 1422 | |
| 1423 | #[tokio::test] |
| 1424 | async fn read_media_provider_request_wiring_integration() { |
| 1425 | let dir = tempdir().unwrap(); |
| 1426 | let img_path = dir.path().join("wire.png"); |
| 1427 | let png_data = create_test_png(40, 40, [10, 20, 30, 255]); |
| 1428 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1429 | |
| 1430 | let ctx = ToolContext::new(dir.path()); |
| 1431 | let tool = ReadMediaTool; |
| 1432 | let rich = tool |
| 1433 | .execute_rich(json!({ "path": "wire.png" }), &ctx) |
| 1434 | .await |
| 1435 | .unwrap(); |
| 1436 | |
| 1437 | let ToolResultContentBlock::Image { mime_type, data } = &rich.content_blocks[0]; |
| 1438 | |
| 1439 | // 1. Check Anthropic tool result wiring |
| 1440 | let anthropic_content = crate::client::anthropic_tool_result_content_for_test( |
| 1441 | &rich.content, |
| 1442 | Some(&[json!({ |
| 1443 | "type": "image", |
| 1444 | "mime_type": mime_type, |
| 1445 | "data": data |
| 1446 | })]), |
| 1447 | ); |
| 1448 | let blocks = anthropic_content |
| 1449 | .as_array() |
| 1450 | .expect("anthropic content array"); |
| 1451 | assert!( |
| 1452 | blocks |
| 1453 | .iter() |
| 1454 | .any(|b| b["type"] == "image" && b["source"]["media_type"] == "image/png") |
| 1455 | ); |
| 1456 | |
| 1457 | // 2. Check OpenAI Responses tool output wiring |
| 1458 | let responses_content = crate::client::responses_tool_output_for_test( |
| 1459 | &rich.content, |
| 1460 | Some(&[json!({ |
| 1461 | "type": "image", |
| 1462 | "mime_type": mime_type, |
| 1463 | "data": data |
| 1464 | })]), |
| 1465 | ); |
| 1466 | let resp_blocks = responses_content |
| 1467 | .as_array() |
| 1468 | .expect("responses content array"); |
| 1469 | assert!(resp_blocks.iter().any(|b| b["type"] == "input_image")); |
| 1470 | |
| 1471 | // 3. Check Chat Completions provider request body wiring |
| 1472 | let messages = vec![ |
| 1473 | codewhale_models::Message { |
| 1474 | role: Role::Assistant, |
| 1475 | content: vec![codewhale_models::ContentBlock::ToolUse { |
| 1476 | id: "call_read_media".to_string(), |
| 1477 | name: "read_media".to_string(), |
| 1478 | input: json!({ "path": "wire.png" }), |
| 1479 | caller: None, |
| 1480 | thought_signature: None, |
| 1481 | }], |
| 1482 | }, |
| 1483 | codewhale_models::Message { |
| 1484 | role: Role::User, |
| 1485 | content: vec![codewhale_models::ContentBlock::ToolResult { |
| 1486 | tool_use_id: "call_read_media".to_string(), |
| 1487 | content: rich.content.clone(), |
| 1488 | is_error: None, |
| 1489 | content_blocks: Some(vec![json!({ |
| 1490 | "type": "image", |
| 1491 | "mime_type": mime_type, |
| 1492 | "data": data |
| 1493 | })]), |
| 1494 | }], |
| 1495 | }, |
| 1496 | ]; |
| 1497 | let chat_msgs = crate::client::chat_messages_for_test(&messages); |
| 1498 | let tool_msg = chat_msgs |
| 1499 | .iter() |
| 1500 | .find(|m| m["role"] == "tool" && m["tool_call_id"] == "call_read_media") |
| 1501 | .expect("tool response message"); |
| 1502 | assert!( |
| 1503 | tool_msg["content"] |
| 1504 | .as_str() |
| 1505 | .unwrap() |
| 1506 | .contains("Read media file") |
| 1507 | ); |
| 1508 | let follow_up_user = chat_msgs |
| 1509 | .iter() |
| 1510 | .find(|m| m["role"] == "user" && m["content"].is_array()) |
| 1511 | .expect("follow-up user message carrying tool image"); |
| 1512 | let parts = follow_up_user["content"].as_array().unwrap(); |
| 1513 | assert!( |
| 1514 | parts.iter().any(|part| { |
| 1515 | part["type"] == "image_url" |
| 1516 | && part["image_url"]["url"] |
| 1517 | .as_str() |
| 1518 | .is_some_and(|u| u.starts_with("data:image/png;base64,")) |
| 1519 | }), |
| 1520 | "expected image_url part in chat completions follow-up message: {parts:?}" |
| 1521 | ); |
| 1522 | |
| 1523 | // 4. Check Chat Completions provider tool result refs helper |
| 1524 | let blocks = [json!({ |
| 1525 | "type": "image", |
| 1526 | "mime_type": mime_type, |
| 1527 | "data": data |
| 1528 | })]; |
| 1529 | let (image_ref, omitted) = |
| 1530 | crate::image_attach::provider_tool_result_image_refs(Some(&blocks)); |
| 1531 | assert_eq!(omitted, 0); |
| 1532 | assert!(image_ref.is_some()); |
| 1533 | let (sniffed_mime, payload) = image_ref.unwrap(); |
| 1534 | assert_eq!(sniffed_mime, "image/png"); |
| 1535 | assert_eq!(payload, data); |
| 1536 | } |
| 1537 | |
| 1538 | #[tokio::test] |
| 1539 | async fn read_media_decompression_bomb_rejected() { |
| 1540 | let dir = tempdir().unwrap(); |
| 1541 | let bomb_path = dir.path().join("bomb.png"); |
| 1542 | |
| 1543 | // Construct a synthetic PNG header with 10,000 x 10,000 dimensions (100 megapixels > 33.5 megapixel guard) |
| 1544 | // PNG signature + IHDR chunk (length 13, type IHDR, width, height, bit depth 8, color type 6, etc.) |
| 1545 | let mut fake_png = Vec::new(); |
| 1546 | fake_png.extend_from_slice(b"\x89PNG\r\n\x1a\n"); |
| 1547 | fake_png.extend_from_slice(&13_u32.to_be_bytes()); // IHDR length |
| 1548 | fake_png.extend_from_slice(b"IHDR"); |
| 1549 | fake_png.extend_from_slice(&10_000_u32.to_be_bytes()); // width: 10,000 |
| 1550 | fake_png.extend_from_slice(&10_000_u32.to_be_bytes()); // height: 10,000 |
| 1551 | fake_png.extend_from_slice(&[8, 6, 0, 0, 0]); // 8-bit RGBA |
| 1552 | fake_png.extend_from_slice(&[0, 0, 0, 0]); // CRC (dummy) |
| 1553 | std::fs::write(&bomb_path, &fake_png).unwrap(); |
| 1554 | |
| 1555 | let ctx = ToolContext::new(dir.path()); |
| 1556 | let tool = ReadMediaTool; |
| 1557 | let err = tool |
| 1558 | .execute_rich(json!({ "path": "bomb.png" }), &ctx) |
| 1559 | .await |
| 1560 | .unwrap_err(); |
| 1561 | let msg = err.to_string(); |
| 1562 | assert!( |
| 1563 | msg.contains("decompression bomb") |
| 1564 | || msg.contains("exceed safe limits") |
| 1565 | || msg.contains("guard triggered"), |
| 1566 | "{}", |
| 1567 | msg |
| 1568 | ); |
| 1569 | } |
| 1570 | |
| 1571 | #[tokio::test] |
| 1572 | async fn read_media_gif_format_supported() { |
| 1573 | let dir = tempdir().unwrap(); |
| 1574 | let gif_path = dir.path().join("anim.gif"); |
| 1575 | let img = image::RgbaImage::from_pixel(30, 30, image::Rgba([100, 150, 200, 255])); |
| 1576 | let mut cursor = Cursor::new(Vec::new()); |
| 1577 | img.write_to(&mut cursor, ImageFormat::Gif).unwrap(); |
| 1578 | std::fs::write(&gif_path, cursor.into_inner()).unwrap(); |
| 1579 | |
| 1580 | let ctx = ToolContext::new(dir.path()); |
| 1581 | let tool = ReadMediaTool; |
| 1582 | let rich = tool |
| 1583 | .execute_rich(json!({ "path": "anim.gif" }), &ctx) |
| 1584 | .await |
| 1585 | .unwrap(); |
| 1586 | assert!(rich.success); |
| 1587 | let meta = rich.metadata.as_ref().unwrap(); |
| 1588 | assert_eq!(meta["original_mime_type"], "image/gif"); |
| 1589 | assert_eq!(meta["mime_type"], "image/png"); |
| 1590 | } |
| 1591 | |
| 1592 | #[tokio::test] |
| 1593 | async fn read_media_webp_format_supported() { |
| 1594 | let dir = tempdir().unwrap(); |
| 1595 | let webp_path = dir.path().join("image.webp"); |
| 1596 | let webp_bytes = BASE64 |
| 1597 | .decode("UklGRkoAAABXRUJQVlA4WAoAAAAQAAAAAAAAAAAAQUxQSAwAAAARBxAR/Q9ERP8DAABWUDggGAAAADABAJ0BKgEAAQABABwlpAADcAD+/gbQAA==") |
| 1598 | .expect("valid webp"); |
| 1599 | std::fs::write(&webp_path, webp_bytes).unwrap(); |
| 1600 | |
| 1601 | let ctx = ToolContext::new(dir.path()); |
| 1602 | let tool = ReadMediaTool; |
| 1603 | let rich = tool |
| 1604 | .execute_rich(json!({ "path": "image.webp" }), &ctx) |
| 1605 | .await |
| 1606 | .unwrap(); |
| 1607 | assert!(rich.success); |
| 1608 | let meta = rich.metadata.as_ref().unwrap(); |
| 1609 | assert_eq!(meta["original_mime_type"], "image/webp"); |
| 1610 | assert_eq!(meta["mime_type"], "image/png"); |
| 1611 | } |
| 1612 | |
| 1613 | #[tokio::test] |
| 1614 | async fn read_media_supported_and_unknown_routes_admitted() { |
| 1615 | let dir = tempdir().unwrap(); |
| 1616 | let img_path = dir.path().join("check.png"); |
| 1617 | let png_data = create_test_png(20, 20, [50, 50, 50, 255]); |
| 1618 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1619 | |
| 1620 | let tool = ReadMediaTool; |
| 1621 | |
| 1622 | // 1. Supported route |
| 1623 | let mut ctx_sup = ToolContext::new(dir.path()); |
| 1624 | ctx_sup.route_capabilities.image_input = CapabilityState::Supported; |
| 1625 | let res_sup = tool |
| 1626 | .execute_rich(json!({ "path": "check.png" }), &ctx_sup) |
| 1627 | .await; |
| 1628 | assert!(res_sup.is_ok()); |
| 1629 | |
| 1630 | // Unknown deliberately matches the established attachment contract: |
| 1631 | // custom/self-hosted routes frequently lack modality metadata, so |
| 1632 | // only a known Unsupported verdict blocks this explicit user action |
| 1633 | // and the provider remains authoritative. |
| 1634 | let mut ctx_unk = ToolContext::new(dir.path()); |
| 1635 | ctx_unk.route_capabilities.image_input = CapabilityState::Unknown; |
| 1636 | let res_unk = tool |
| 1637 | .execute_rich(json!({ "path": "check.png" }), &ctx_unk) |
| 1638 | .await; |
| 1639 | let rich_unk = res_unk.unwrap(); |
| 1640 | assert!(rich_unk.content.contains( |
| 1641 | "Route image support is unverified; if the model cannot see this image, use image_ocr." |
| 1642 | )); |
| 1643 | } |
| 1644 | |
| 1645 | #[tokio::test] |
| 1646 | async fn read_media_receipt_contains_no_credentials() { |
| 1647 | let dir = tempdir().unwrap(); |
| 1648 | let img_path = dir.path().join("safe.png"); |
| 1649 | let png_data = create_test_png(15, 15, [255, 255, 255, 255]); |
| 1650 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1651 | |
| 1652 | let ctx = ToolContext::new(dir.path()); |
| 1653 | let tool = ReadMediaTool; |
| 1654 | let rich = tool |
| 1655 | .execute_rich(json!({ "path": "safe.png" }), &ctx) |
| 1656 | .await |
| 1657 | .unwrap(); |
| 1658 | |
| 1659 | let meta = rich.metadata.as_ref().unwrap(); |
| 1660 | let meta_str = meta.to_string(); |
| 1661 | assert!(!meta_str.contains("key")); |
| 1662 | assert!(!meta_str.contains("secret")); |
| 1663 | assert!(!meta_str.contains("token")); |
| 1664 | assert!(!meta_str.contains("password")); |
| 1665 | assert!(!meta_str.contains("auth")); |
| 1666 | } |
| 1667 | |
| 1668 | /// Deterministic photo-like noise image (high colorfulness, hard to |
| 1669 | /// compress), so the ladder takes the JPEG path without any network or |
| 1670 | /// fixture dependency. |
| 1671 | fn create_noise_png(width: u32, height: u32) -> Vec<u8> { |
| 1672 | let mut img = image::RgbImage::new(width, height); |
| 1673 | let mut state: u32 = 0x1234_5678; |
| 1674 | let mut next = move || { |
| 1675 | state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); |
| 1676 | (state >> 24) as u8 |
| 1677 | }; |
| 1678 | for px in img.pixels_mut() { |
| 1679 | *px = image::Rgb([next(), next(), next()]); |
| 1680 | } |
| 1681 | let mut cursor = Cursor::new(Vec::new()); |
| 1682 | img.write_to(&mut cursor, ImageFormat::Png).unwrap(); |
| 1683 | cursor.into_inner() |
| 1684 | } |
| 1685 | |
| 1686 | fn decode_payload(data: &str) -> (u32, u32, Vec<u8>) { |
| 1687 | let bytes = BASE64.decode(data).expect("valid base64 payload"); |
| 1688 | let img = ImageReader::new(Cursor::new(&bytes)) |
| 1689 | .with_guessed_format() |
| 1690 | .unwrap() |
| 1691 | .decode() |
| 1692 | .expect("delivered payload decodes"); |
| 1693 | (img.width(), img.height(), bytes) |
| 1694 | } |
| 1695 | |
| 1696 | #[tokio::test] |
| 1697 | async fn read_media_photo_over_budget_arrives_under_budget_as_jpeg() { |
| 1698 | let dir = tempdir().unwrap(); |
| 1699 | let img_path = dir.path().join("photo.png"); |
| 1700 | let png_data = create_noise_png(1600, 1200); |
| 1701 | assert!( |
| 1702 | png_data.len() > READ_IMAGE_BYTE_BUDGET, |
| 1703 | "test image must start over the default read budget" |
| 1704 | ); |
| 1705 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1706 | |
| 1707 | let mut ctx = ToolContext::new(dir.path()); |
| 1708 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1709 | |
| 1710 | let tool = ReadMediaTool; |
| 1711 | let rich = tool |
| 1712 | .execute_rich(json!({ "path": "photo.png", "detail": "auto" }), &ctx) |
| 1713 | .await |
| 1714 | .unwrap(); |
| 1715 | assert!(rich.success); |
| 1716 | |
| 1717 | let ToolResultContentBlock::Image { mime_type, data } = &rich.content_blocks[0]; |
| 1718 | assert_eq!( |
| 1719 | mime_type, "image/jpeg", |
| 1720 | "photo-like content must go out as JPEG, not PNG" |
| 1721 | ); |
| 1722 | let payload_len = BASE64.decode(data).unwrap().len(); |
| 1723 | assert!( |
| 1724 | payload_len <= READ_IMAGE_BYTE_BUDGET, |
| 1725 | "delivered payload {payload_len} must fit the {}-byte read budget", |
| 1726 | READ_IMAGE_BYTE_BUDGET |
| 1727 | ); |
| 1728 | |
| 1729 | let meta = rich.metadata.as_ref().unwrap(); |
| 1730 | assert_eq!(meta["mime_type"], "image/jpeg"); |
| 1731 | assert_eq!(meta["original_width"], 1600); |
| 1732 | assert_eq!(meta["original_height"], 1200); |
| 1733 | assert_eq!( |
| 1734 | meta["delivery"], "downsampled", |
| 1735 | "a quality- or resolution-reduced delivery is never silent" |
| 1736 | ); |
| 1737 | assert!(rich.content.contains("fine detail may be lost")); |
| 1738 | assert!( |
| 1739 | rich.content |
| 1740 | .contains("crop parameter (original-image pixel coordinates)") |
| 1741 | ); |
| 1742 | } |
| 1743 | |
| 1744 | #[tokio::test] |
| 1745 | async fn read_media_small_image_passes_untouched() { |
| 1746 | let dir = tempdir().unwrap(); |
| 1747 | let img_path = dir.path().join("small.png"); |
| 1748 | let png_data = create_test_png(120, 80, [10, 120, 200, 255]); |
| 1749 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1750 | |
| 1751 | let mut ctx = ToolContext::new(dir.path()); |
| 1752 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1753 | |
| 1754 | let tool = ReadMediaTool; |
| 1755 | let rich = tool |
| 1756 | .execute_rich(json!({ "path": "small.png" }), &ctx) |
| 1757 | .await |
| 1758 | .unwrap(); |
| 1759 | assert!(rich.success); |
| 1760 | |
| 1761 | let meta = rich.metadata.as_ref().unwrap(); |
| 1762 | assert_eq!(meta["delivery"], "untouched"); |
| 1763 | assert_eq!(meta["mime_type"], "image/png"); |
| 1764 | assert_eq!(meta["width"], 120); |
| 1765 | assert_eq!(meta["height"], 80); |
| 1766 | assert!( |
| 1767 | meta["original_path"].is_null(), |
| 1768 | "untouched reads keep no stored copy" |
| 1769 | ); |
| 1770 | assert!(rich.content.contains("untouched")); |
| 1771 | assert!(rich.content.contains("no downsampling applied")); |
| 1772 | } |
| 1773 | |
| 1774 | #[tokio::test] |
| 1775 | async fn read_media_alpha_image_stays_png() { |
| 1776 | let dir = tempdir().unwrap(); |
| 1777 | let img_path = dir.path().join("alpha.png"); |
| 1778 | // Colorful RGB with a real alpha gradient: alpha must survive. |
| 1779 | let mut img = image::RgbaImage::new(640, 480); |
| 1780 | for (x, _y, px) in img.enumerate_pixels_mut() { |
| 1781 | *px = image::Rgba([(x % 256) as u8, 200, 90, if x < 320 { 255 } else { 100 }]); |
| 1782 | } |
| 1783 | let mut cursor = Cursor::new(Vec::new()); |
| 1784 | img.write_to(&mut cursor, ImageFormat::Png).unwrap(); |
| 1785 | std::fs::write(&img_path, cursor.into_inner()).unwrap(); |
| 1786 | |
| 1787 | let mut ctx = ToolContext::new(dir.path()); |
| 1788 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1789 | |
| 1790 | let tool = ReadMediaTool; |
| 1791 | let rich = tool |
| 1792 | .execute_rich(json!({ "path": "alpha.png" }), &ctx) |
| 1793 | .await |
| 1794 | .unwrap(); |
| 1795 | assert!(rich.success); |
| 1796 | |
| 1797 | let ToolResultContentBlock::Image { mime_type, data } = &rich.content_blocks[0]; |
| 1798 | assert_eq!( |
| 1799 | mime_type, "image/png", |
| 1800 | "meaningful alpha must never be flattened to JPEG" |
| 1801 | ); |
| 1802 | let (w, h, bytes) = decode_payload(data); |
| 1803 | assert_eq!((w, h), (640, 480)); |
| 1804 | let decoded = ImageReader::new(Cursor::new(&bytes)) |
| 1805 | .with_guessed_format() |
| 1806 | .unwrap() |
| 1807 | .decode() |
| 1808 | .unwrap() |
| 1809 | .to_rgba8(); |
| 1810 | assert!( |
| 1811 | decoded.pixels().any(|p| p.0[3] == 100), |
| 1812 | "alpha channel must survive delivery" |
| 1813 | ); |
| 1814 | } |
| 1815 | |
| 1816 | #[tokio::test] |
| 1817 | async fn read_media_delivery_note_reports_mode_dims_and_zoom_guidance() { |
| 1818 | let dir = tempdir().unwrap(); |
| 1819 | let img_path = dir.path().join("big.png"); |
| 1820 | let png_data = create_test_png(3000, 1500, [0, 0, 255, 255]); |
| 1821 | std::fs::write(&img_path, &png_data).unwrap(); |
| 1822 | |
| 1823 | let mut ctx = ToolContext::new(dir.path()); |
| 1824 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1825 | |
| 1826 | let tool = ReadMediaTool; |
| 1827 | let rich = tool |
| 1828 | .execute_rich(json!({ "path": "big.png", "detail": "auto" }), &ctx) |
| 1829 | .await |
| 1830 | .unwrap(); |
| 1831 | assert!(rich.success); |
| 1832 | |
| 1833 | let meta = rich.metadata.as_ref().unwrap(); |
| 1834 | assert_eq!(meta["delivery"], "downsampled"); |
| 1835 | assert_eq!(meta["width"], 2048); |
| 1836 | assert_eq!(meta["height"], 1024); |
| 1837 | |
| 1838 | let note = &rich.content; |
| 1839 | assert!(note.contains("Original dimensions: 3000x1500"), "{note}"); |
| 1840 | assert!(note.contains("downsampled to 2048x1024"), "{note}"); |
| 1841 | assert!(note.contains("fine detail may be lost"), "{note}"); |
| 1842 | assert!( |
| 1843 | note.contains( |
| 1844 | "call read_media again with the crop parameter (original-image pixel coordinates)" |
| 1845 | ), |
| 1846 | "downsampled note must guide zoom via crop: {note}" |
| 1847 | ); |
| 1848 | } |
| 1849 | |
| 1850 | #[tokio::test] |
| 1851 | async fn read_media_crop_after_downsample_pulls_full_res_stored_original() { |
| 1852 | let dir = tempdir().unwrap(); |
| 1853 | let store_dir = dir.path().join("media-originals"); |
| 1854 | let img_path = dir.path().join("scene.png"); |
| 1855 | // 2600x1300 photo-like source: auto detail downsamples to 2048x1024. |
| 1856 | std::fs::write(&img_path, create_noise_png(2600, 1300)).unwrap(); |
| 1857 | |
| 1858 | let mut ctx = ToolContext::new(dir.path()).with_runtime_services( |
| 1859 | crate::tools::spec::RuntimeToolServices { |
| 1860 | media_originals_dir: Some(store_dir.clone()), |
| 1861 | ..crate::tools::spec::RuntimeToolServices::default() |
| 1862 | }, |
| 1863 | ); |
| 1864 | ctx.route_capabilities.image_input = CapabilityState::Supported; |
| 1865 | |
| 1866 | let tool = ReadMediaTool; |
| 1867 | let rich = tool |
| 1868 | .execute_rich(json!({ "path": "scene.png", "detail": "auto" }), &ctx) |
| 1869 | .await |
| 1870 | .unwrap(); |
| 1871 | let meta = rich.metadata.as_ref().unwrap(); |
| 1872 | assert_eq!(meta["delivery"], "downsampled"); |
| 1873 | let delivered_width = meta["width"].as_u64().unwrap(); |
| 1874 | assert!( |
| 1875 | delivered_width <= 2048, |
| 1876 | "auto detail caps at 2048px; noise may ladder further: {delivered_width}" |
| 1877 | ); |
| 1878 | assert_eq!(meta["original_width"], 2600); |
| 1879 | assert_eq!(meta["original_height"], 1300); |
| 1880 | |
| 1881 | // The pre-compression original is persisted, content-addressed. |
| 1882 | let original_path = meta["original_path"] |
| 1883 | .as_str() |
| 1884 | .expect("downsampled reads persist the original") |
| 1885 | .to_string(); |
| 1886 | assert!(original_path.starts_with(store_dir.to_str().unwrap())); |
| 1887 | let stored = std::fs::read(&original_path).expect("stored original bytes"); |
| 1888 | let stored_img = ImageReader::new(Cursor::new(&stored)) |
| 1889 | .with_guessed_format() |
| 1890 | .unwrap() |
| 1891 | .decode() |
| 1892 | .unwrap(); |
| 1893 | assert_eq!( |
| 1894 | (stored_img.width(), stored_img.height()), |
| 1895 | (2600, 1300), |
| 1896 | "the store holds the full-resolution source, not the downsampled copy" |
| 1897 | ); |
| 1898 | |
| 1899 | // A crop that is out of bounds for the downsampled 2048-wide copy but |
| 1900 | // valid for the 2600-wide original must succeed at the requested |
| 1901 | // pixel size — proof the crop pulls the full-res stored source. |
| 1902 | let crop = tool |
| 1903 | .execute_rich( |
| 1904 | json!({ |
| 1905 | "path": original_path, |
| 1906 | "crop": { "x": 2200, "y": 900, "width": 300, "height": 300 } |
| 1907 | }), |
| 1908 | &ctx, |
| 1909 | ) |
| 1910 | .await |
| 1911 | .unwrap(); |
| 1912 | let crop_meta = crop.metadata.as_ref().unwrap(); |
| 1913 | assert_eq!(crop_meta["delivery"], "crop"); |
| 1914 | assert_eq!(crop_meta["width"], 300); |
| 1915 | assert_eq!(crop_meta["height"], 300); |
| 1916 | assert_eq!(crop_meta["original_width"], 2600); |
| 1917 | assert!( |
| 1918 | crop.content.contains("region offset (x=2200, y=900)"), |
| 1919 | "crop note must carry the offset guidance: {}", |
| 1920 | crop.content |
| 1921 | ); |
| 1922 | } |
| 1923 | |
| 1924 | #[tokio::test] |
| 1925 | async fn read_media_over_budget_failure_names_conversion_recipe() { |
| 1926 | let dir = tempdir().unwrap(); |
| 1927 | let img_path = dir.path().join("noise.png"); |
| 1928 | std::fs::write(&img_path, create_noise_png(512, 512)).unwrap(); |
| 1929 | let raw = std::fs::read(&img_path).unwrap(); |
| 1930 | let (image, _, _) = decode_and_guard_image(&raw).unwrap(); |
| 1931 | |
| 1932 | // A 1024-byte budget is unreachable even at the 256px/q20 floor. |
| 1933 | let err = |
| 1934 | encode_within_budget(&image, EncodePolicy::PreferJpeg, 1024, &img_path).unwrap_err(); |
| 1935 | let msg = err.to_string(); |
| 1936 | assert!(msg.contains("Nothing was sent"), "{msg}"); |
| 1937 | assert!(msg.contains("sips -Z 1024"), "recipe names sips: {msg}"); |
| 1938 | assert!(msg.contains("magick"), "recipe names ImageMagick: {msg}"); |
| 1939 | assert!(msg.contains("smaller copy"), "{msg}"); |
| 1940 | |
| 1941 | // Alpha-bearing images fail closed the same way rather than |
| 1942 | // falling back to JPEG. |
| 1943 | let err = encode_within_budget(&image, EncodePolicy::RequireAlphaPng, 1024, &img_path) |
| 1944 | .unwrap_err(); |
| 1945 | assert!(err.to_string().contains("Nothing was sent"), "{err}"); |
| 1946 | } |
| 1947 | } |
| 1948 |