| 1 | //! `image_ocr` tool — extract text from an image via local OCR. |
| 2 | //! |
| 3 | //! Tesseract is the cross-platform workhorse for "convert this image |
| 4 | //! to text". On macOS we also use the built-in Vision framework, so |
| 5 | //! screenshots keep working on a clean machine without making the |
| 6 | //! user install a separate OCR binary first. |
| 7 | //! |
| 8 | //! Surfacing OCR as a model-callable tool means the model can read an |
| 9 | //! asset the user drops into the workspace without bouncing through |
| 10 | //! `exec_shell`. |
| 11 | |
| 12 | use std::path::Path; |
| 13 | use std::process::{Command, Stdio}; |
| 14 | |
| 15 | use async_trait::async_trait; |
| 16 | use serde_json::{Value, json}; |
| 17 | |
| 18 | use super::spec::{ToolCapability, ToolContext, ToolError, ToolResult, ToolSpec, required_str}; |
| 19 | |
| 20 | /// Tool implementing `image_ocr`. Runs a local OCR backend and returns the |
| 21 | /// extracted text on success. |
| 22 | pub struct ImageOcrTool; |
| 23 | |
| 24 | #[async_trait] |
| 25 | impl ToolSpec for ImageOcrTool { |
| 26 | fn name(&self) -> &'static str { |
| 27 | "image_ocr" |
| 28 | } |
| 29 | |
| 30 | fn description(&self) -> &'static str { |
| 31 | "Extract text from an image (PNG, JPEG, or TIFF) via local OCR. On macOS this uses the built-in Vision framework; otherwise it uses local tesseract when available. Use this for screenshots, scanned receipts/whiteboards, image-only PDFs, or any visual that contains text the model needs to read. Returns the extracted text inline; no file is written." |
| 32 | } |
| 33 | |
| 34 | fn input_schema(&self) -> Value { |
| 35 | json!({ |
| 36 | "type": "object", |
| 37 | "properties": { |
| 38 | "path": { |
| 39 | "type": "string", |
| 40 | "description": "Path to the image file (relative to workspace or absolute). PNG / JPEG / TIFF supported." |
| 41 | } |
| 42 | }, |
| 43 | "required": ["path"] |
| 44 | }) |
| 45 | } |
| 46 | |
| 47 | fn capabilities(&self) -> Vec<ToolCapability> { |
| 48 | vec![ToolCapability::ReadOnly, ToolCapability::Sandboxable] |
| 49 | } |
| 50 | |
| 51 | fn supports_parallel(&self) -> bool { |
| 52 | true |
| 53 | } |
| 54 | |
| 55 | async fn execute(&self, input: Value, context: &ToolContext) -> Result<ToolResult, ToolError> { |
| 56 | let path_str = required_str(&input, "path")?; |
| 57 | let image_path = context.resolve_path(path_str)?; |
| 58 | if !image_path.exists() { |
| 59 | return Err(ToolError::execution_failed(format!( |
| 60 | "image_ocr: source path does not exist: {}", |
| 61 | image_path.display() |
| 62 | ))); |
| 63 | } |
| 64 | |
| 65 | let text = ocr_image_path(&image_path)?; |
| 66 | Ok(ToolResult::success(text)) |
| 67 | } |
| 68 | } |
| 69 | |
| 70 | pub(crate) fn ocr_available() -> bool { |
| 71 | std::env::var_os("CODEWHALE_LOCAL_OCR_UNAVAILABLE").is_none() |
| 72 | && (crate::dependencies::resolve_tesseract().is_some() || native_ocr_available()) |
| 73 | } |
| 74 | |
| 75 | pub(crate) fn ocr_image_path(image_path: &Path) -> Result<String, ToolError> { |
| 76 | // Prefer native OCR when the backend probe says it works. If native fails |
| 77 | // at runtime, fall through to tesseract rather than hard-failing — hosts |
| 78 | // can advertise Vision classes while still rejecting performRequests. |
| 79 | match try_native_ocr(image_path) { |
| 80 | Ok(Some(text)) => return Ok(text), |
| 81 | Ok(None) => {} |
| 82 | Err(err) => { |
| 83 | if crate::dependencies::resolve_tesseract().is_none() { |
| 84 | return Err(err); |
| 85 | } |
| 86 | // Native probe-or-run failed; tesseract remains as fallback. |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | if let Some(tesseract) = crate::dependencies::resolve_tesseract() { |
| 91 | return ocr_with_tesseract(&tesseract, image_path); |
| 92 | } |
| 93 | |
| 94 | Err(ToolError::execution_failed( |
| 95 | "image_ocr: no local OCR backend is available. On macOS, update to a version with the Vision framework; on Linux/Windows install tesseract and restart codewhale.", |
| 96 | )) |
| 97 | } |
| 98 | |
| 99 | fn ocr_with_tesseract(tesseract: &str, image_path: &Path) -> Result<String, ToolError> { |
| 100 | // `tesseract <image> -` writes the recognised text to stdout. The trailing |
| 101 | // `-` is documented and produces text mode by default (no `.txt` file). |
| 102 | let mut cmd = Command::new(tesseract); |
| 103 | cmd.arg(image_path); |
| 104 | cmd.arg("-"); |
| 105 | cmd.stdin(Stdio::null()) |
| 106 | .stdout(Stdio::piped()) |
| 107 | .stderr(Stdio::piped()); |
| 108 | |
| 109 | let output = cmd |
| 110 | .output() |
| 111 | .map_err(|e| ToolError::execution_failed(format!("failed to launch tesseract: {e}")))?; |
| 112 | |
| 113 | if !output.status.success() { |
| 114 | let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); |
| 115 | return Err(ToolError::execution_failed(format!( |
| 116 | "tesseract failed (exit {:?}): {stderr}", |
| 117 | output.status.code() |
| 118 | ))); |
| 119 | } |
| 120 | |
| 121 | // Tesseract appends a trailing form-feed on some platforms; trim trailing |
| 122 | // whitespace so the result reads cleanly inline. |
| 123 | Ok(String::from_utf8_lossy(&output.stdout) |
| 124 | .trim_end() |
| 125 | .to_string()) |
| 126 | } |
| 127 | |
| 128 | #[cfg(target_os = "macos")] |
| 129 | fn native_ocr_available() -> bool { |
| 130 | // Classes can exist at link time while runtime Vision is unusable |
| 131 | // (restricted CI hosts); probe the ObjC class table once to match real use. |
| 132 | macos_vision::vision_runtime_available() |
| 133 | } |
| 134 | |
| 135 | #[cfg(not(target_os = "macos"))] |
| 136 | fn native_ocr_available() -> bool { |
| 137 | false |
| 138 | } |
| 139 | |
| 140 | #[cfg(not(target_os = "macos"))] |
| 141 | fn try_native_ocr(_image_path: &Path) -> Result<Option<String>, ToolError> { |
| 142 | Ok(None) |
| 143 | } |
| 144 | |
| 145 | #[cfg(target_os = "macos")] |
| 146 | #[link(name = "Vision", kind = "framework")] |
| 147 | unsafe extern "C" {} |
| 148 | |
| 149 | #[cfg(target_os = "macos")] |
| 150 | fn try_native_ocr(image_path: &Path) -> Result<Option<String>, ToolError> { |
| 151 | if !native_ocr_available() { |
| 152 | return Ok(None); |
| 153 | } |
| 154 | macos_vision::recognize_text(image_path).map(Some) |
| 155 | } |
| 156 | |
| 157 | #[cfg(target_os = "macos")] |
| 158 | mod macos_vision { |
| 159 | use super::*; |
| 160 | use objc2::msg_send; |
| 161 | use objc2::rc::{Retained, autoreleasepool}; |
| 162 | use objc2::runtime::{AnyClass, AnyObject}; |
| 163 | use objc2_foundation::{NSArray, NSDictionary, NSError, NSString, NSURL}; |
| 164 | use std::ptr; |
| 165 | |
| 166 | pub(super) fn recognize_text(image_path: &Path) -> Result<String, ToolError> { |
| 167 | autoreleasepool(|_| recognize_text_inner(image_path)) |
| 168 | } |
| 169 | |
| 170 | /// True when the Vision text-recognition classes resolve at runtime. |
| 171 | /// Does not attempt a full OCR round-trip (that needs an image and can |
| 172 | /// fail for image-specific reasons); class resolution is the cheap probe |
| 173 | /// used by `ocr_available` / tool registration. |
| 174 | pub(super) fn vision_runtime_available() -> bool { |
| 175 | use std::sync::OnceLock; |
| 176 | static AVAILABLE: OnceLock<bool> = OnceLock::new(); |
| 177 | *AVAILABLE.get_or_init(|| { |
| 178 | AnyClass::get(c"VNRecognizeTextRequest").is_some() |
| 179 | && AnyClass::get(c"VNImageRequestHandler").is_some() |
| 180 | }) |
| 181 | } |
| 182 | |
| 183 | fn recognize_text_inner(image_path: &Path) -> Result<String, ToolError> { |
| 184 | let url = NSURL::from_file_path(image_path).ok_or_else(|| { |
| 185 | ToolError::execution_failed(format!( |
| 186 | "image_ocr: failed to build file URL for {}", |
| 187 | image_path.display() |
| 188 | )) |
| 189 | })?; |
| 190 | |
| 191 | let request_class = AnyClass::get(c"VNRecognizeTextRequest").ok_or_else(|| { |
| 192 | ToolError::execution_failed("image_ocr: macOS Vision text request is unavailable") |
| 193 | })?; |
| 194 | let handler_class = AnyClass::get(c"VNImageRequestHandler").ok_or_else(|| { |
| 195 | ToolError::execution_failed("image_ocr: macOS Vision image handler is unavailable") |
| 196 | })?; |
| 197 | |
| 198 | let request = new_object(request_class, "VNRecognizeTextRequest")?; |
| 199 | // VNRequestTextRecognitionLevelAccurate is 0. Use accurate mode for |
| 200 | // screenshots and receipts; the tool is user-facing, not latency-critical. |
| 201 | unsafe { |
| 202 | let _: () = msg_send![&*request, setRecognitionLevel: 0usize]; |
| 203 | let _: () = msg_send![&*request, setUsesLanguageCorrection: true]; |
| 204 | } |
| 205 | |
| 206 | let requests = NSArray::from_slice(&[&*request]); |
| 207 | let options: Retained<NSDictionary<NSString, AnyObject>> = NSDictionary::new(); |
| 208 | |
| 209 | let handler_alloc = alloc_object(handler_class, "VNImageRequestHandler")?; |
| 210 | let handler_raw: *mut AnyObject = |
| 211 | unsafe { msg_send![handler_alloc, initWithURL: &*url, options: &*options] }; |
| 212 | let handler = unsafe { Retained::from_raw(handler_raw) }.ok_or_else(|| { |
| 213 | ToolError::execution_failed("image_ocr: failed to initialize Vision image handler") |
| 214 | })?; |
| 215 | |
| 216 | let mut error: *mut NSError = ptr::null_mut(); |
| 217 | let ok: bool = |
| 218 | unsafe { msg_send![&*handler, performRequests: &*requests, error: &mut error] }; |
| 219 | if !ok { |
| 220 | return Err(ToolError::execution_failed(format!( |
| 221 | "image_ocr: macOS Vision failed{}", |
| 222 | vision_error_suffix(error) |
| 223 | ))); |
| 224 | } |
| 225 | |
| 226 | collect_recognized_text(&request) |
| 227 | } |
| 228 | |
| 229 | fn new_object(class: &AnyClass, label: &str) -> Result<Retained<AnyObject>, ToolError> { |
| 230 | let raw: *mut AnyObject = unsafe { msg_send![class, new] }; |
| 231 | unsafe { Retained::from_raw(raw) }.ok_or_else(|| { |
| 232 | ToolError::execution_failed(format!("image_ocr: failed to create {label}")) |
| 233 | }) |
| 234 | } |
| 235 | |
| 236 | fn alloc_object(class: &AnyClass, label: &str) -> Result<*mut AnyObject, ToolError> { |
| 237 | let raw: *mut AnyObject = unsafe { msg_send![class, alloc] }; |
| 238 | if raw.is_null() { |
| 239 | Err(ToolError::execution_failed(format!( |
| 240 | "image_ocr: failed to allocate {label}" |
| 241 | ))) |
| 242 | } else { |
| 243 | Ok(raw) |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | fn collect_recognized_text(request: &AnyObject) -> Result<String, ToolError> { |
| 248 | let results: *mut AnyObject = unsafe { msg_send![request, results] }; |
| 249 | if results.is_null() { |
| 250 | return Ok(String::new()); |
| 251 | } |
| 252 | |
| 253 | let count: usize = unsafe { msg_send![results, count] }; |
| 254 | let mut lines = Vec::new(); |
| 255 | for idx in 0..count { |
| 256 | let observation: *mut AnyObject = unsafe { msg_send![results, objectAtIndex: idx] }; |
| 257 | if observation.is_null() { |
| 258 | continue; |
| 259 | } |
| 260 | let candidates: *mut AnyObject = |
| 261 | unsafe { msg_send![observation, topCandidates: 1usize] }; |
| 262 | if candidates.is_null() { |
| 263 | continue; |
| 264 | } |
| 265 | let candidate_count: usize = unsafe { msg_send![candidates, count] }; |
| 266 | if candidate_count == 0 { |
| 267 | continue; |
| 268 | } |
| 269 | let candidate: *mut AnyObject = unsafe { msg_send![candidates, objectAtIndex: 0usize] }; |
| 270 | if candidate.is_null() { |
| 271 | continue; |
| 272 | } |
| 273 | let text: *mut NSString = unsafe { msg_send![candidate, string] }; |
| 274 | if text.is_null() { |
| 275 | continue; |
| 276 | } |
| 277 | let line = unsafe { &*text }.to_string(); |
| 278 | let trimmed = line.trim(); |
| 279 | if !trimmed.is_empty() { |
| 280 | lines.push(trimmed.to_string()); |
| 281 | } |
| 282 | } |
| 283 | |
| 284 | Ok(lines.join("\n")) |
| 285 | } |
| 286 | |
| 287 | fn vision_error_suffix(error: *mut NSError) -> String { |
| 288 | if error.is_null() { |
| 289 | return String::new(); |
| 290 | } |
| 291 | let description: *mut NSString = unsafe { msg_send![error, localizedDescription] }; |
| 292 | if description.is_null() { |
| 293 | String::new() |
| 294 | } else { |
| 295 | format!(": {}", unsafe { &*description }) |
| 296 | } |
| 297 | } |
| 298 | } |
| 299 | |
| 300 | #[cfg(test)] |
| 301 | mod tests { |
| 302 | use super::*; |
| 303 | use std::fs; |
| 304 | use tempfile::tempdir; |
| 305 | |
| 306 | /// Resolve the checked-in OCR fixture path. The image lives at |
| 307 | /// `crates/tui/tests/fixtures/ocr_hello.png` (300x100 grayscale, |
| 308 | /// "HELLO OCR" rendered in Helvetica) and is committed for the |
| 309 | /// happy-path round-trip below. |
| 310 | fn ocr_fixture_path() -> std::path::PathBuf { |
| 311 | std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/ocr_hello.png") |
| 312 | } |
| 313 | |
| 314 | #[test] |
| 315 | fn tool_metadata_marks_image_ocr_read_only_and_parallel() { |
| 316 | let tool = ImageOcrTool; |
| 317 | assert_eq!(tool.name(), "image_ocr"); |
| 318 | assert!(tool.supports_parallel()); |
| 319 | let caps = tool.capabilities(); |
| 320 | assert!(caps.contains(&ToolCapability::ReadOnly)); |
| 321 | assert!(!caps.contains(&ToolCapability::WritesFiles)); |
| 322 | } |
| 323 | |
| 324 | #[tokio::test] |
| 325 | async fn image_ocr_rejects_missing_path() { |
| 326 | let tmp = tempdir().expect("tempdir"); |
| 327 | let ctx = ToolContext::new(tmp.path().to_path_buf()); |
| 328 | let err = ImageOcrTool |
| 329 | .execute(json!({"path": "definitely-not-here.png"}), &ctx) |
| 330 | .await |
| 331 | .expect_err("nonexistent path must reject before tesseract spawn"); |
| 332 | let msg = err.to_string(); |
| 333 | assert!( |
| 334 | msg.contains("does not exist"), |
| 335 | "error must call out missing path; got {msg}" |
| 336 | ); |
| 337 | } |
| 338 | |
| 339 | #[tokio::test] |
| 340 | async fn image_ocr_recovers_hello_from_fixture_image() { |
| 341 | if !ocr_available() { |
| 342 | // Tool wouldn't be registered without a local OCR backend — mirror |
| 343 | // that here so the suite stays green on CI images that |
| 344 | // intentionally omit OCR tooling. |
| 345 | return; |
| 346 | } |
| 347 | let fixture = ocr_fixture_path(); |
| 348 | if !fixture.exists() { |
| 349 | // Fixture not committed (sparse / shallow checkout). Skip |
| 350 | // silently rather than failing the suite. |
| 351 | return; |
| 352 | } |
| 353 | let tmp = tempdir().expect("tempdir"); |
| 354 | // Stage the fixture under the workspace so the path resolver |
| 355 | // accepts the relative input — keeps the test independent of |
| 356 | // the workspace boundary check inside `resolve_path`. |
| 357 | let staged = tmp.path().join("ocr_hello.png"); |
| 358 | fs::copy(&fixture, &staged).unwrap(); |
| 359 | let ctx = ToolContext::new(tmp.path().to_path_buf()); |
| 360 | let result = match ImageOcrTool |
| 361 | .execute(json!({"path": "ocr_hello.png"}), &ctx) |
| 362 | .await |
| 363 | { |
| 364 | Ok(result) => result, |
| 365 | Err(err) => { |
| 366 | // Backend probe can still disagree with a live OCR run |
| 367 | // (restricted Vision, broken tesseract install, sandbox). |
| 368 | // Name promises coverage only when the backend works. |
| 369 | let msg = err.to_string(); |
| 370 | let _skip_reason = format!("OCR backend probe passed but execute failed: {msg}"); |
| 371 | let _ = &_skip_reason; |
| 372 | return; |
| 373 | } |
| 374 | }; |
| 375 | assert!(result.success); |
| 376 | // Tesseract reliably recovers "HELLO OCR" from the rendered |
| 377 | // PNG; allow either spacing variant. |
| 378 | let normalised = result.content.to_uppercase(); |
| 379 | assert!( |
| 380 | normalised.contains("HELLO") && normalised.contains("OCR"), |
| 381 | "expected OCR to recover HELLO OCR; got {:?}", |
| 382 | result.content |
| 383 | ); |
| 384 | } |
| 385 | } |
| 386 |