| 1 | //! Turning a local image file into a wire-ready image content block. |
| 2 | //! |
| 3 | //! CodeWhale's message model has been multimodal for a long time — |
| 4 | //! [`ContentBlock::ImageUrl`] round-trips |
| 5 | //! through session persistence, compaction and all three wire builders. What it |
| 6 | //! never had was a *faucet*: nothing outside `#[cfg(test)]` ever constructed |
| 7 | //! one, so a user who attached a screenshot got the literal text |
| 8 | //! `[Attached image: /path/to/shot.png]` and a model that correctly concluded it |
| 9 | //! could not see the picture. |
| 10 | //! |
| 11 | //! This module is that faucet. It reads a file, proves it is an image the |
| 12 | //! providers actually accept, holds it to a size budget, and emits a |
| 13 | //! `data:` URL. |
| 14 | //! |
| 15 | //! # Two stages, because the two failure kinds differ |
| 16 | //! |
| 17 | //! [`expand_attachment_blocks`] runs when the message is built and decides the |
| 18 | //! *permanent* questions: does this file exist, is it really an image, is it |
| 19 | //! small enough. Those answers cannot change, so they are baked into history. |
| 20 | //! |
| 21 | //! [`strip_images_when_unsupported`] runs per outbound request and decides the |
| 22 | //! one *contingent* question: can the model this request is going to actually |
| 23 | //! see images. Routes change mid-session, so answering that at build time |
| 24 | //! would mean attaching a screenshot under a text-only model and losing it |
| 25 | //! permanently, even after switching to a vision model. History keeps the |
| 26 | //! image; each request is normalized against its own route. |
| 27 | //! |
| 28 | //! # Provider neutrality |
| 29 | //! |
| 30 | //! There is exactly one internal representation — a `data:<media-type>;base64,…` |
| 31 | //! URL on `ContentBlock::ImageUrl` — and each wire builder projects it: |
| 32 | //! |
| 33 | //! | wire format | shape | |
| 34 | //! |---|---| |
| 35 | //! | Chat Completions | `{"type":"image_url","image_url":{"url":"data:…"}}` | |
| 36 | //! | Responses | `{"type":"input_image","image_url":"data:…"}` | |
| 37 | //! | Anthropic Messages | `{"type":"image","source":{"type":"base64","media_type":…,"data":…}}` | |
| 38 | //! |
| 39 | //! The Anthropic split lives in [`parse_data_url`], which |
| 40 | //! `client::anthropic` calls. Anthropic is the reason the accepted-format list |
| 41 | //! below is not simply "whatever the `image` crate can decode": it accepts only |
| 42 | //! PNG/JPEG/GIF/WebP, and so, therefore, do we. Refusing a BMP here with a |
| 43 | //! readable message beats letting one through to a provider-side 400. |
| 44 | |
| 45 | use anyhow::{Result, bail}; |
| 46 | use codewhale_protocol::runtime::{ |
| 47 | MAX_RUNTIME_IMAGE_BYTES, MAX_RUNTIME_IMAGE_TOTAL_BYTES, MAX_RUNTIME_IMAGES, RuntimeImageInput, |
| 48 | }; |
| 49 | use image::{DynamicImage, ImageReader, Limits}; |
| 50 | use std::io::Cursor; |
| 51 | use std::path::Path; |
| 52 | |
| 53 | use base64::{Engine as _, engine::general_purpose::STANDARD}; |
| 54 | |
| 55 | use crate::model_profile::SupportState; |
| 56 | use codewhale_models::{ContentBlock, ImageUrlContent}; |
| 57 | |
| 58 | /// Largest source image accepted, in bytes, before base64 expansion. |
| 59 | /// |
| 60 | /// Base64 inflates by 4/3, so this admits roughly 6.7 MB of request body per |
| 61 | /// image. The number is Anthropic's documented per-image ceiling; keeping the |
| 62 | /// tightest provider limit as the shared limit is what makes a "CodeWhale |
| 63 | /// accepted it" verdict portable across routes. |
| 64 | pub const MAX_IMAGE_BYTES: usize = 5 * 1024 * 1024; |
| 65 | |
| 66 | /// Maximum width or height admitted for an input image (8192 px). |
| 67 | pub const MAX_IMAGE_DIMENSION: u32 = 8192; |
| 68 | |
| 69 | /// Maximum total pixels admitted before decoding is aborted (~33.5 megapixels). |
| 70 | pub const MAX_IMAGE_PIXELS: u64 = 33_554_432; |
| 71 | |
| 72 | /// Memory allocation limit for image decoding (64 MiB). |
| 73 | pub const MAX_DECODE_ALLOC_BYTES: u64 = 64 * 1024 * 1024; |
| 74 | |
| 75 | pub(crate) fn decode_and_guard_image(bytes: &[u8]) -> Result<(DynamicImage, u32, u32)> { |
| 76 | let limits = || { |
| 77 | let mut limits = Limits::default(); |
| 78 | limits.max_alloc = Some(MAX_DECODE_ALLOC_BYTES); |
| 79 | limits.max_image_width = Some(MAX_IMAGE_DIMENSION); |
| 80 | limits.max_image_height = Some(MAX_IMAGE_DIMENSION); |
| 81 | limits |
| 82 | }; |
| 83 | let mut reader = ImageReader::new(Cursor::new(bytes)).with_guessed_format()?; |
| 84 | reader.limits(limits()); |
| 85 | let (width, height) = reader |
| 86 | .into_dimensions() |
| 87 | .map_err(|_| anyhow::anyhow!("invalid image header or decompression bomb guard"))?; |
| 88 | if u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS |
| 89 | || width > MAX_IMAGE_DIMENSION |
| 90 | || height > MAX_IMAGE_DIMENSION |
| 91 | { |
| 92 | bail!("image dimensions exceed the decompression bomb guard; downscale or crop first"); |
| 93 | } |
| 94 | let mut reader = ImageReader::new(Cursor::new(bytes)).with_guessed_format()?; |
| 95 | reader.limits(limits()); |
| 96 | let decoded = reader |
| 97 | .decode() |
| 98 | .map_err(|_| anyhow::anyhow!("invalid image content or decode allocation limit"))?; |
| 99 | Ok((decoded, width, height)) |
| 100 | } |
| 101 | |
| 102 | /// Validate untrusted inline input before route selection or durable admission. |
| 103 | /// Return the existing provider-neutral history representation; no file is opened. |
| 104 | pub(crate) fn prepare_runtime_images(images: &[RuntimeImageInput]) -> Result<Vec<ContentBlock>> { |
| 105 | if images.len() > MAX_RUNTIME_IMAGES { |
| 106 | bail!("images exceed the {MAX_RUNTIME_IMAGES} attachment limit"); |
| 107 | } |
| 108 | prepare_images_with_limit( |
| 109 | images, |
| 110 | MAX_RUNTIME_IMAGE_BYTES, |
| 111 | Some(MAX_RUNTIME_IMAGE_TOTAL_BYTES), |
| 112 | ) |
| 113 | } |
| 114 | |
| 115 | /// Internal Engine/history input retains the established local 5 MiB ceiling. |
| 116 | /// Network callers must first pass `prepare_runtime_images` (4 MiB per image, |
| 117 | /// 10 images and 5 MiB total). Local history never had those aggregate/count |
| 118 | /// limits; impose only its existing per-image bound and bounded full decode. |
| 119 | pub(crate) fn prepare_stored_images(images: &[RuntimeImageInput]) -> Result<Vec<ContentBlock>> { |
| 120 | prepare_images_with_limit(images, MAX_IMAGE_BYTES, None) |
| 121 | } |
| 122 | |
| 123 | fn prepare_images_with_limit( |
| 124 | images: &[RuntimeImageInput], |
| 125 | per_image_limit: usize, |
| 126 | total_limit: Option<usize>, |
| 127 | ) -> Result<Vec<ContentBlock>> { |
| 128 | let mut total = 0usize; |
| 129 | images |
| 130 | .iter() |
| 131 | .enumerate() |
| 132 | .map(|(index, image)| { |
| 133 | if image.data_base64.len() > per_image_limit.div_ceil(3) * 4 { |
| 134 | bail!( |
| 135 | "image {} exceeds the {} MiB limit", |
| 136 | index + 1, |
| 137 | per_image_limit / (1024 * 1024) |
| 138 | ); |
| 139 | } |
| 140 | let bytes = STANDARD |
| 141 | .decode(&image.data_base64) |
| 142 | .map_err(|_| anyhow::anyhow!("image {} has invalid base64", index + 1))?; |
| 143 | if bytes.len() > per_image_limit { |
| 144 | bail!( |
| 145 | "image {} exceeds the {} MiB limit", |
| 146 | index + 1, |
| 147 | per_image_limit / (1024 * 1024) |
| 148 | ); |
| 149 | } |
| 150 | total = total.saturating_add(bytes.len()); |
| 151 | if total_limit.is_some_and(|limit| total > limit) { |
| 152 | bail!("images exceed the 5 MiB total limit"); |
| 153 | } |
| 154 | let attached = encode_image_bytes(&bytes, &format!("image {}", index + 1))?; |
| 155 | if image.mime != attached.media_type { |
| 156 | bail!("image {} MIME does not match its content", index + 1); |
| 157 | } |
| 158 | decode_and_guard_image(&bytes)?; |
| 159 | // Standard padded base64 is the one replay representation. |
| 160 | if STANDARD.encode(&bytes) != image.data_base64 { |
| 161 | bail!("image {} base64 is not canonical", index + 1); |
| 162 | } |
| 163 | Ok(attached.content_block()) |
| 164 | }) |
| 165 | .collect() |
| 166 | } |
| 167 | |
| 168 | /// Reuse durable canonical bytes for retry, never reread a path or URL. |
| 169 | pub(crate) fn runtime_images_from_blocks( |
| 170 | blocks: &[ContentBlock], |
| 171 | ) -> Result<Vec<RuntimeImageInput>> { |
| 172 | let mut images = Vec::new(); |
| 173 | for block in blocks { |
| 174 | if let ContentBlock::ImageUrl { image_url } = block { |
| 175 | if image_url.url.len() > MAX_IMAGE_BYTES.div_ceil(3) * 4 + 32 { |
| 176 | bail!("stored image exceeds the attachment limit"); |
| 177 | } |
| 178 | let (mime, data) = parse_data_url(&image_url.url) |
| 179 | .ok_or_else(|| anyhow::anyhow!("stored image requires canonical inline content"))?; |
| 180 | images.push(RuntimeImageInput { |
| 181 | mime: mime.to_string(), |
| 182 | data_base64: data.to_string(), |
| 183 | }); |
| 184 | } |
| 185 | } |
| 186 | prepare_stored_images(&images)?; |
| 187 | Ok(images) |
| 188 | } |
| 189 | |
| 190 | /// Validate new image-bearing durable records without rewriting their block order. |
| 191 | /// Legacy schema 2 history continues to use its original interpretation. |
| 192 | pub(crate) fn validate_stored_image_content(blocks: &[ContentBlock]) -> Result<()> { |
| 193 | if blocks.iter().any(|block| { |
| 194 | !matches!( |
| 195 | block, |
| 196 | ContentBlock::Text { .. } | ContentBlock::ImageUrl { .. } |
| 197 | ) |
| 198 | }) { |
| 199 | bail!("invalid persisted user image content kind"); |
| 200 | } |
| 201 | if runtime_images_from_blocks(blocks)?.is_empty() { |
| 202 | bail!("persisted image input must contain an image"); |
| 203 | } |
| 204 | Ok(()) |
| 205 | } |
| 206 | |
| 207 | /// Why a file could not be attached as an image. |
| 208 | /// |
| 209 | /// Every variant renders to a sentence naming the file and the reason. These |
| 210 | /// strings reach both the user (as a command error) and the model (as an |
| 211 | /// in-band notice), so they say what to do next rather than only what failed. |
| 212 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 213 | pub enum ImageAttachError { |
| 214 | /// The file could not be read at all. |
| 215 | Unreadable { path: String, reason: String }, |
| 216 | /// The file is zero bytes. |
| 217 | Empty { path: String }, |
| 218 | /// Over [`MAX_IMAGE_BYTES`]. |
| 219 | TooLarge { path: String, bytes: usize }, |
| 220 | /// Magic bytes identify a format no provider in the set accepts. |
| 221 | UnsupportedFormat { path: String, detected: String }, |
| 222 | /// Magic bytes match nothing we recognize as an image. |
| 223 | NotAnImage { path: String }, |
| 224 | } |
| 225 | |
| 226 | impl std::fmt::Display for ImageAttachError { |
| 227 | fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { |
| 228 | match self { |
| 229 | Self::Unreadable { path, reason } => { |
| 230 | write!(f, "Cannot attach {path}: {reason}") |
| 231 | } |
| 232 | Self::Empty { path } => { |
| 233 | write!(f, "Cannot attach {path}: the file is empty") |
| 234 | } |
| 235 | Self::TooLarge { path, bytes } => write!( |
| 236 | f, |
| 237 | "Cannot attach {path}: {} exceeds the {} per-image limit. \ |
| 238 | Downscale or crop it first.", |
| 239 | human_bytes(*bytes), |
| 240 | human_bytes(MAX_IMAGE_BYTES), |
| 241 | ), |
| 242 | Self::UnsupportedFormat { path, detected } => write!( |
| 243 | f, |
| 244 | "Cannot attach {path}: {detected} is not accepted by vision \ |
| 245 | models. Convert it to PNG, JPEG, GIF or WebP.", |
| 246 | ), |
| 247 | Self::NotAnImage { path } => write!( |
| 248 | f, |
| 249 | "Cannot attach {path}: the file is not a PNG, JPEG, GIF or \ |
| 250 | WebP image (its contents do not match any of those formats).", |
| 251 | ), |
| 252 | } |
| 253 | } |
| 254 | } |
| 255 | |
| 256 | impl std::error::Error for ImageAttachError {} |
| 257 | |
| 258 | fn human_bytes(bytes: usize) -> String { |
| 259 | if bytes >= 1024 * 1024 { |
| 260 | format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0)) |
| 261 | } else if bytes >= 1024 { |
| 262 | format!("{:.1} KB", bytes as f64 / 1024.0) |
| 263 | } else { |
| 264 | format!("{bytes} bytes") |
| 265 | } |
| 266 | } |
| 267 | |
| 268 | /// An image that is ready to go on the wire. |
| 269 | #[derive(Debug, Clone, PartialEq, Eq)] |
| 270 | pub struct AttachedImage { |
| 271 | /// e.g. `"image/png"`. |
| 272 | pub media_type: &'static str, |
| 273 | /// `data:<media_type>;base64,<payload>`. |
| 274 | pub data_url: String, |
| 275 | /// Size of the source file, before base64. |
| 276 | pub source_bytes: usize, |
| 277 | } |
| 278 | |
| 279 | /// Validated image content returned by lowercase `read`. |
| 280 | pub struct PreparedToolImage { |
| 281 | pub block: Option<codewhale_tools::ToolResultContentBlock>, |
| 282 | pub note: String, |
| 283 | } |
| 284 | |
| 285 | /// Prepare one provider-neutral tool-result image using Codewhale's existing |
| 286 | /// cross-provider format and size policy. Failure is a visible text receipt, |
| 287 | /// not a failed file read. |
| 288 | #[must_use] |
| 289 | pub fn prepare_tool_image_bytes(bytes: &[u8], mime_type: &str) -> PreparedToolImage { |
| 290 | let mime_type = mime_type.split(';').next().unwrap_or(mime_type).trim(); |
| 291 | let valid = bytes.len() <= MAX_IMAGE_BYTES |
| 292 | && sniff_media_type(bytes) == Some(mime_type) |
| 293 | && decode_and_guard_image(bytes).is_ok(); |
| 294 | if !valid { |
| 295 | return PreparedToolImage { |
| 296 | block: None, |
| 297 | note: format!( |
| 298 | "Read image file [{mime_type}]\n[Image omitted: unsupported, invalid, or above the 5 MiB inline limit.]" |
| 299 | ), |
| 300 | }; |
| 301 | } |
| 302 | PreparedToolImage { |
| 303 | block: Some(codewhale_tools::ToolResultContentBlock::Image { |
| 304 | mime_type: mime_type.to_string(), |
| 305 | data: STANDARD.encode(bytes), |
| 306 | }), |
| 307 | note: format!("Read image file [{mime_type}]"), |
| 308 | } |
| 309 | } |
| 310 | |
| 311 | fn valid_tool_image(mime_type: &str, data: &str) -> bool { |
| 312 | matches!( |
| 313 | mime_type, |
| 314 | "image/png" | "image/jpeg" | "image/gif" | "image/webp" |
| 315 | ) && data.len() <= MAX_IMAGE_BYTES.div_ceil(3) * 4 |
| 316 | && STANDARD.decode(data).is_ok_and(|bytes| { |
| 317 | bytes.len() <= MAX_IMAGE_BYTES |
| 318 | && sniff_media_type(&bytes) == Some(mime_type) |
| 319 | && decode_and_guard_image(&bytes).is_ok() |
| 320 | }) |
| 321 | } |
| 322 | |
| 323 | /// Enforce the same one-image limit at the tool execution boundary so plugin |
| 324 | /// or future rich tools cannot create unbounded history. |
| 325 | #[must_use] |
| 326 | pub(crate) fn bound_rich_tool_result( |
| 327 | mut rich: crate::tools::spec::RichToolResult, |
| 328 | ) -> crate::tools::spec::RichToolResult { |
| 329 | let mut kept = Vec::with_capacity(1); |
| 330 | let mut omitted = 0usize; |
| 331 | for block in rich.content_blocks.drain(..) { |
| 332 | let codewhale_tools::ToolResultContentBlock::Image { mime_type, data } = █ |
| 333 | if kept.is_empty() && valid_tool_image(mime_type, data) { |
| 334 | kept.push(block); |
| 335 | } else { |
| 336 | omitted += 1; |
| 337 | } |
| 338 | } |
| 339 | rich.result.content = tool_result_text_with_omission(&rich.result.content, omitted); |
| 340 | rich.content_blocks = kept; |
| 341 | rich |
| 342 | } |
| 343 | |
| 344 | /// Borrow the first valid inline image and count everything omitted. |
| 345 | #[must_use] |
| 346 | pub(crate) fn provider_tool_result_image_refs( |
| 347 | blocks: Option<&[serde_json::Value]>, |
| 348 | ) -> (Option<(&str, &str)>, usize) { |
| 349 | let mut image = None; |
| 350 | let mut omitted = 0usize; |
| 351 | for block in blocks.unwrap_or_default() { |
| 352 | let fields = block |
| 353 | .get("type") |
| 354 | .and_then(serde_json::Value::as_str) |
| 355 | .filter(|kind| *kind == "image") |
| 356 | .and_then(|_| { |
| 357 | block |
| 358 | .get("mime_type") |
| 359 | .and_then(serde_json::Value::as_str) |
| 360 | .zip(block.get("data").and_then(serde_json::Value::as_str)) |
| 361 | }); |
| 362 | if image.is_none() |
| 363 | && let Some((mime_type, data)) = fields |
| 364 | && valid_tool_image(mime_type, data) |
| 365 | { |
| 366 | image = Some((mime_type, data)); |
| 367 | } else { |
| 368 | omitted += 1; |
| 369 | } |
| 370 | } |
| 371 | (image, omitted) |
| 372 | } |
| 373 | |
| 374 | #[must_use] |
| 375 | pub(crate) fn tool_result_text_with_omission(content: &str, omitted: usize) -> String { |
| 376 | if omitted == 0 { |
| 377 | return content.to_string(); |
| 378 | } |
| 379 | format!( |
| 380 | "{content}\n[{omitted} tool-result image block(s) omitted: invalid, unsupported, oversized, or additional.]" |
| 381 | ) |
| 382 | } |
| 383 | |
| 384 | /// Copy/export projection that retains metadata but never inline base64. |
| 385 | #[must_use] |
| 386 | pub(crate) fn safe_tool_result_content_blocks( |
| 387 | blocks: Option<&[serde_json::Value]>, |
| 388 | ) -> Option<Vec<serde_json::Value>> { |
| 389 | blocks.map(|blocks| { |
| 390 | blocks |
| 391 | .iter() |
| 392 | .map(|block| { |
| 393 | if block.get("type").and_then(serde_json::Value::as_str) == Some("image") { |
| 394 | serde_json::json!({ |
| 395 | "type": "image", |
| 396 | "mime_type": block.get("mime_type").and_then(serde_json::Value::as_str).unwrap_or("application/octet-stream"), |
| 397 | "omission_code": "inline_or_local_image_payload", |
| 398 | "omitted_base64_bytes": block.get("data").and_then(serde_json::Value::as_str).map_or(0, str::len), |
| 399 | }) |
| 400 | } else { |
| 401 | block.clone() |
| 402 | } |
| 403 | }) |
| 404 | .collect() |
| 405 | }) |
| 406 | } |
| 407 | |
| 408 | #[must_use] |
| 409 | pub(crate) fn safe_tool_result_message_projection( |
| 410 | messages: &[codewhale_models::Message], |
| 411 | ) -> Vec<codewhale_models::Message> { |
| 412 | let mut projected = messages.to_vec(); |
| 413 | for message in &mut projected { |
| 414 | for block in &mut message.content { |
| 415 | if let ContentBlock::ToolResult { content_blocks, .. } = block { |
| 416 | *content_blocks = safe_tool_result_content_blocks(content_blocks.as_deref()); |
| 417 | } |
| 418 | } |
| 419 | } |
| 420 | projected |
| 421 | } |
| 422 | |
| 423 | impl AttachedImage { |
| 424 | /// The content block this image becomes in a message. |
| 425 | #[must_use] |
| 426 | pub fn content_block(&self) -> ContentBlock { |
| 427 | ContentBlock::ImageUrl { |
| 428 | image_url: ImageUrlContent { |
| 429 | url: self.data_url.clone(), |
| 430 | }, |
| 431 | } |
| 432 | } |
| 433 | } |
| 434 | |
| 435 | /// Identify an image format from its leading bytes. |
| 436 | /// |
| 437 | /// Extension sniffing is not enough here: the extension is attacker- and |
| 438 | /// typo-controlled, while what the provider validates is the payload. A |
| 439 | /// `.png` holding JPEG bytes must be declared `image/jpeg` or the request is |
| 440 | /// rejected with a media-type mismatch that reads like a CodeWhale bug. |
| 441 | /// |
| 442 | /// Returns `None` for anything that is not one of the four accepted formats; |
| 443 | /// [`detect_rejected_format`] names the near-misses so the error can be |
| 444 | /// specific. |
| 445 | #[must_use] |
| 446 | pub fn sniff_media_type(bytes: &[u8]) -> Option<&'static str> { |
| 447 | if bytes.starts_with(b"\x89PNG\r\n\x1a\n") { |
| 448 | return Some("image/png"); |
| 449 | } |
| 450 | if bytes.starts_with(b"\xff\xd8\xff") { |
| 451 | return Some("image/jpeg"); |
| 452 | } |
| 453 | if bytes.starts_with(b"GIF87a") || bytes.starts_with(b"GIF89a") { |
| 454 | return Some("image/gif"); |
| 455 | } |
| 456 | if bytes.len() >= 12 && bytes.starts_with(b"RIFF") && &bytes[8..12] == b"WEBP" { |
| 457 | return Some("image/webp"); |
| 458 | } |
| 459 | None |
| 460 | } |
| 461 | |
| 462 | /// Name a format we can recognize but deliberately refuse. |
| 463 | /// |
| 464 | /// These are real images, so `NotAnImage` would be a lie and would send the |
| 465 | /// user looking for a corrupt file. Naming the format points at the fix |
| 466 | /// (convert it) instead. |
| 467 | #[must_use] |
| 468 | pub fn detect_rejected_format(bytes: &[u8]) -> Option<&'static str> { |
| 469 | if bytes.starts_with(b"BM") { |
| 470 | return Some("BMP"); |
| 471 | } |
| 472 | if bytes.starts_with(b"II\x2a\x00") || bytes.starts_with(b"MM\x00\x2a") { |
| 473 | return Some("TIFF"); |
| 474 | } |
| 475 | if bytes.len() >= 12 && bytes.starts_with(b"\0\0\0") && &bytes[4..8] == b"ftyp" { |
| 476 | return Some("HEIC/AVIF"); |
| 477 | } |
| 478 | if bytes.starts_with(b"<svg") || bytes.starts_with(b"<?xml") { |
| 479 | return Some("SVG"); |
| 480 | } |
| 481 | if bytes.starts_with(b"%PDF") { |
| 482 | return Some("PDF"); |
| 483 | } |
| 484 | None |
| 485 | } |
| 486 | |
| 487 | /// Validate and encode raw bytes that were read from `path`. |
| 488 | /// |
| 489 | /// Split from [`attach_image_from_path`] so the whole policy — order of |
| 490 | /// checks, limits, format verdicts — is testable without touching a |
| 491 | /// filesystem. |
| 492 | pub fn encode_image_bytes(bytes: &[u8], path: &str) -> Result<AttachedImage, ImageAttachError> { |
| 493 | if bytes.is_empty() { |
| 494 | return Err(ImageAttachError::Empty { |
| 495 | path: path.to_string(), |
| 496 | }); |
| 497 | } |
| 498 | // Size is checked before format so a huge file is rejected on the cheap |
| 499 | // fact rather than after we have decided what it is. |
| 500 | if bytes.len() > MAX_IMAGE_BYTES { |
| 501 | return Err(ImageAttachError::TooLarge { |
| 502 | path: path.to_string(), |
| 503 | bytes: bytes.len(), |
| 504 | }); |
| 505 | } |
| 506 | let Some(media_type) = sniff_media_type(bytes) else { |
| 507 | return Err(match detect_rejected_format(bytes) { |
| 508 | Some(detected) => ImageAttachError::UnsupportedFormat { |
| 509 | path: path.to_string(), |
| 510 | detected: detected.to_string(), |
| 511 | }, |
| 512 | None => ImageAttachError::NotAnImage { |
| 513 | path: path.to_string(), |
| 514 | }, |
| 515 | }); |
| 516 | }; |
| 517 | let payload = STANDARD.encode(bytes); |
| 518 | Ok(AttachedImage { |
| 519 | media_type, |
| 520 | data_url: format!("data:{media_type};base64,{payload}"), |
| 521 | source_bytes: bytes.len(), |
| 522 | }) |
| 523 | } |
| 524 | |
| 525 | /// Read, validate and encode an image file. |
| 526 | pub fn attach_image_from_path(path: &Path) -> Result<AttachedImage, ImageAttachError> { |
| 527 | let display = path.display().to_string(); |
| 528 | // Check the size from metadata first so a multi-gigabyte file is refused |
| 529 | // without being read into memory. |
| 530 | if let Ok(meta) = std::fs::metadata(path) { |
| 531 | let len = meta.len(); |
| 532 | if len > MAX_IMAGE_BYTES as u64 { |
| 533 | return Err(ImageAttachError::TooLarge { |
| 534 | path: display, |
| 535 | bytes: usize::try_from(len).unwrap_or(usize::MAX), |
| 536 | }); |
| 537 | } |
| 538 | } |
| 539 | let bytes = std::fs::read(path).map_err(|error| ImageAttachError::Unreadable { |
| 540 | path: display.clone(), |
| 541 | reason: error.to_string(), |
| 542 | })?; |
| 543 | encode_image_bytes(&bytes, &display) |
| 544 | } |
| 545 | |
| 546 | /// Split a `data:<media-type>;base64,<payload>` URL. |
| 547 | /// |
| 548 | /// Anthropic's Messages API models an image as a tagged `source` rather than a |
| 549 | /// URL, so the native route has to take the data URL back apart. Returns |
| 550 | /// `None` for `http(s)` URLs and for anything malformed, which the caller |
| 551 | /// renders as a remote source or a visible degradation respectively. |
| 552 | #[must_use] |
| 553 | pub fn parse_data_url(url: &str) -> Option<(&str, &str)> { |
| 554 | let rest = url.strip_prefix("data:")?; |
| 555 | let (header, payload) = rest.split_once(',')?; |
| 556 | let media_type = header.strip_suffix(";base64")?; |
| 557 | if media_type.is_empty() || payload.is_empty() { |
| 558 | return None; |
| 559 | } |
| 560 | Some((media_type, payload)) |
| 561 | } |
| 562 | |
| 563 | /// Whether a URL is one a provider can fetch for itself. |
| 564 | #[must_use] |
| 565 | pub fn is_remote_image_url(url: &str) -> bool { |
| 566 | url.starts_with("https://") || url.starts_with("http://") |
| 567 | } |
| 568 | |
| 569 | /// The outcome of expanding a user turn's attachment placeholders. |
| 570 | #[derive(Debug, Clone, Default, PartialEq)] |
| 571 | pub struct ExpandedAttachments { |
| 572 | /// Image blocks to append to the user message, in placeholder order. |
| 573 | pub blocks: Vec<ContentBlock>, |
| 574 | /// One line per attachment that could not be sent. These are shown to the |
| 575 | /// user and also handed to the model, because a model that is not told an |
| 576 | /// image was dropped will confidently discuss it from the filename. |
| 577 | pub notices: Vec<String>, |
| 578 | } |
| 579 | |
| 580 | /// Build the image blocks for a user turn from its `[Attached image: …]` lines. |
| 581 | /// |
| 582 | /// This is the ingest half of the composer's placeholder design: the buffer |
| 583 | /// holds a path-bearing text line (which survives editing, history and session |
| 584 | /// reload for free), and the bytes are read here, once, as the message is |
| 585 | /// built. |
| 586 | /// |
| 587 | /// Only *permanent* failures are decided here — a file that is missing, |
| 588 | /// oversized, or not an image will still be all of those things next turn, so |
| 589 | /// baking the verdict into history costs nothing. Whether the *model* can see |
| 590 | /// images is deliberately not decided here; that is contingent on the active |
| 591 | /// route and is re-decided per request by |
| 592 | /// [`strip_images_when_unsupported`]. |
| 593 | /// |
| 594 | /// Each image is bracketed by text tags naming its path. Without them a turn |
| 595 | /// carrying three screenshots gives the model three anonymous images in a row |
| 596 | /// and no way to say which is which. |
| 597 | /// |
| 598 | /// Never returns an error: a turn with one bad attachment should still be |
| 599 | /// sent, with the failure stated in-band rather than swallowed. |
| 600 | #[must_use] |
| 601 | pub fn expand_attachment_blocks(text: &str) -> ExpandedAttachments { |
| 602 | let references = codewhale_core::media_attachment_references(text); |
| 603 | let mut out = ExpandedAttachments::default(); |
| 604 | for reference in references { |
| 605 | if reference.kind != "image" { |
| 606 | // Video and any future kind: left as the text reference it |
| 607 | // already was. Silently ignoring it here is not a drop — the |
| 608 | // path is still in the prompt, exactly as before this module |
| 609 | // existed. |
| 610 | continue; |
| 611 | } |
| 612 | match attach_image_from_path(Path::new(&reference.path)) { |
| 613 | Ok(image) => { |
| 614 | out.blocks |
| 615 | .push(tag_block(&format!("<image path=\"{}\">", reference.path))); |
| 616 | out.blocks.push(image.content_block()); |
| 617 | out.blocks.push(tag_block("</image>")); |
| 618 | } |
| 619 | Err(error) => out.notices.push(error.to_string()), |
| 620 | } |
| 621 | } |
| 622 | out |
| 623 | } |
| 624 | |
| 625 | fn tag_block(text: &str) -> ContentBlock { |
| 626 | ContentBlock::Text { |
| 627 | text: text.to_string(), |
| 628 | cache_control: None, |
| 629 | } |
| 630 | } |
| 631 | |
| 632 | /// Replace every image in a request with text when the route cannot see them. |
| 633 | /// |
| 634 | /// Capability is a property of the *route*, not of the attachment, and the |
| 635 | /// route changes freely mid-session. Deciding this when the message is built |
| 636 | /// would burn the answer into history: attach a screenshot while a text-only |
| 637 | /// model is selected, switch to a vision model, and the image would be gone |
| 638 | /// for good. So history always keeps the real image and each outbound request |
| 639 | /// is normalized against the model it is actually going to. |
| 640 | /// |
| 641 | /// Only a known `Unsupported` strips. Most routes report `Unknown` because |
| 642 | /// models.dev has no modality data for them, and treating unknown as "no" |
| 643 | /// would make the feature dead on arrival for exactly the self-hosted and |
| 644 | /// custom routes that most need it — so `Unknown` sends the image and lets the |
| 645 | /// provider be the authority. |
| 646 | /// |
| 647 | /// The image is replaced in place rather than removed, so the model is told |
| 648 | /// why it is looking at a gap instead of being left to invent one. |
| 649 | pub fn strip_images_when_unsupported( |
| 650 | messages: &mut [codewhale_models::Message], |
| 651 | vision: SupportState, |
| 652 | model: &str, |
| 653 | ) -> usize { |
| 654 | if vision != SupportState::Unsupported { |
| 655 | return 0; |
| 656 | } |
| 657 | let mut stripped = 0; |
| 658 | for message in messages.iter_mut() { |
| 659 | for block in &mut message.content { |
| 660 | match block { |
| 661 | ContentBlock::ImageUrl { .. } => { |
| 662 | *block = ContentBlock::Text { |
| 663 | text: format!( |
| 664 | "[image content omitted: the active model ({model}) does \ |
| 665 | not accept image input. Use the image_ocr tool to read \ |
| 666 | text from it, or switch to a vision-capable model with \ |
| 667 | /model.]" |
| 668 | ), |
| 669 | cache_control: None, |
| 670 | }; |
| 671 | stripped += 1; |
| 672 | } |
| 673 | ContentBlock::ToolResult { |
| 674 | content, |
| 675 | content_blocks, |
| 676 | .. |
| 677 | } => { |
| 678 | let count = content_blocks.as_ref().map_or(0, Vec::len); |
| 679 | if count > 0 { |
| 680 | *content_blocks = None; |
| 681 | *content = format!( |
| 682 | "{content}\n[{count} image block(s) omitted: the active model ({model}) does not accept image input. Use the image_ocr tool to read text from it, or switch to a vision-capable model with /model.]" |
| 683 | ); |
| 684 | stripped += count; |
| 685 | } |
| 686 | } |
| 687 | _ => {} |
| 688 | } |
| 689 | } |
| 690 | } |
| 691 | stripped |
| 692 | } |
| 693 | |
| 694 | /// Render dropped-attachment notices as a block the model will read. |
| 695 | /// |
| 696 | /// Wrapped in a tag rather than appended as bare prose so the model can tell |
| 697 | /// the difference between the user saying something and the harness reporting |
| 698 | /// on itself. |
| 699 | #[must_use] |
| 700 | pub fn notice_block(notices: &[String]) -> Option<ContentBlock> { |
| 701 | if notices.is_empty() { |
| 702 | return None; |
| 703 | } |
| 704 | let body = notices.join("\n"); |
| 705 | Some(ContentBlock::Text { |
| 706 | text: format!( |
| 707 | "<attachment_notice>\n{body}\nDo not describe these images from \ |
| 708 | memory or from their filenames; ask the user to re-share them.\n\ |
| 709 | </attachment_notice>" |
| 710 | ), |
| 711 | cache_control: None, |
| 712 | }) |
| 713 | } |
| 714 | |
| 715 | #[cfg(test)] |
| 716 | #[path = "image_attach/tests.rs"] |
| 717 | pub(crate) mod tests; |
| 718 |