| 1 | #import <Cocoa/Cocoa.h> |
| 2 | #import <Vision/Vision.h> |
| 3 | #include <fcntl.h> |
| 4 | #include <sys/stat.h> |
| 5 | #include <unistd.h> |
| 6 | |
| 7 | // Text recognition enriches an existing window observation. It neither reads |
| 8 | // the desktop nor creates accessibility roles or element identities. |
| 9 | static NSDictionary *cuOCRUnavailable(NSString *reason) { |
| 10 | return @{ @"status":@"unavailable", @"engine":@"apple_vision", @"reason":reason, @"blocks":@[] }; |
| 11 | } |
| 12 | |
| 13 | static NSDictionary *cuOCRPixelBounds(CGRect box, size_t width, size_t height) { |
| 14 | // Vision uses a normalized lower-left origin; raster targets use upper-left. |
| 15 | double x=MAX(0,MIN(width,floor(box.origin.x*width))); |
| 16 | double y=MAX(0,MIN(height,floor((1-CGRectGetMaxY(box))*height))); |
| 17 | double right=MAX(x,MIN(width,ceil(CGRectGetMaxX(box)*width))); |
| 18 | double bottom=MAX(y,MIN(height,ceil((1-box.origin.y)*height))); |
| 19 | return @{ @"x":@(x), @"y":@(y), @"w":@(right-x), @"h":@(bottom-y) }; |
| 20 | } |
| 21 | |
| 22 | static NSDictionary *cuRecognizeText(NSString *file) { |
| 23 | if(![file isKindOfClass:NSString.class] || !file.isAbsolutePath) |
| 24 | return cuOCRUnavailable(@"OCR needs the captured window image"); |
| 25 | int fd=open(file.fileSystemRepresentation,O_RDONLY|O_NOFOLLOW|O_CLOEXEC); |
| 26 | struct stat st; |
| 27 | if(fd<0) return cuOCRUnavailable(@"The captured window image could not be opened"); |
| 28 | if(fstat(fd,&st)!=0 || !S_ISREG(st.st_mode) || st.st_size<=0 || st.st_size>64*1024*1024) { |
| 29 | close(fd); return cuOCRUnavailable(@"The captured window image is empty or exceeds the 64 MiB OCR limit"); |
| 30 | } |
| 31 | // Read the same descriptor we checked; do not reopen a replaceable path. |
| 32 | NSMutableData *bytes=[NSMutableData dataWithLength:(NSUInteger)st.st_size]; |
| 33 | size_t offset=0; |
| 34 | while(offset<bytes.length) { |
| 35 | ssize_t count=read(fd,(char *)bytes.mutableBytes+offset,bytes.length-offset); |
| 36 | if(count<0 && errno==EINTR) continue; |
| 37 | if(count<=0) { close(fd); return cuOCRUnavailable(@"The captured window image could not be read"); } |
| 38 | offset+=(size_t)count; |
| 39 | } |
| 40 | close(fd); |
| 41 | NSBitmapImageRep *bitmap=[[NSBitmapImageRep alloc] initWithData:bytes]; |
| 42 | CGImageRef image=bitmap.CGImage; |
| 43 | if(!image) return cuOCRUnavailable(@"The captured window image could not be decoded"); |
| 44 | size_t width=CGImageGetWidth(image),height=CGImageGetHeight(image); |
| 45 | if(!width || !height || width>50000000/height) |
| 46 | return cuOCRUnavailable(@"The captured window image exceeds the 50 megapixel OCR limit"); |
| 47 | @try { |
| 48 | VNRecognizeTextRequest *request=[VNRecognizeTextRequest new]; |
| 49 | request.recognitionLevel=VNRequestTextRecognitionLevelAccurate; |
| 50 | // Preserve literal UI strings, including code, instead of correcting words. |
| 51 | request.usesLanguageCorrection=NO; |
| 52 | if(@available(macOS 13.0,*)) request.automaticallyDetectsLanguage=YES; |
| 53 | VNImageRequestHandler *handler=[[VNImageRequestHandler alloc] initWithCGImage:image options:@{}]; |
| 54 | NSError *error=nil; |
| 55 | if(![handler performRequests:@[request] error:&error]) |
| 56 | return cuOCRUnavailable([NSString stringWithFormat:@"Apple Vision could not recognize this image: %@",error.localizedDescription?:@"unknown error"]); |
| 57 | NSMutableArray *blocks=[NSMutableArray array]; |
| 58 | NSUInteger characters=0; |
| 59 | BOOL truncated=NO; |
| 60 | for(VNRecognizedTextObservation *observation in request.results) { |
| 61 | VNRecognizedText *candidate=[observation topCandidates:1].firstObject; |
| 62 | if(!candidate.string.length) continue; |
| 63 | if(blocks.count>=256 || characters+candidate.string.length>16000) { truncated=YES; break; } |
| 64 | CGRect box=observation.boundingBox; |
| 65 | if(!isfinite(box.origin.x) || !isfinite(box.origin.y) || !isfinite(box.size.width) || !isfinite(box.size.height)) continue; |
| 66 | NSDictionary *bounds=cuOCRPixelBounds(box,width,height); |
| 67 | if([bounds[@"w"] doubleValue]<=0 || [bounds[@"h"] doubleValue]<=0) continue; |
| 68 | [blocks addObject:@{ @"text":candidate.string, @"confidence":@(candidate.confidence), @"bounds":bounds }]; |
| 69 | characters+=candidate.string.length; |
| 70 | } |
| 71 | return @{ @"status":@"ok", @"engine":@"apple_vision", @"coordinate_space":@"raster_pixels", |
| 72 | @"pixels":@{ @"w":@(width), @"h":@(height) }, @"blocks":blocks, |
| 73 | @"truncated":@(truncated), @"character_count":@(characters) }; |
| 74 | } @catch(NSException *error) { |
| 75 | return cuOCRUnavailable([NSString stringWithFormat:@"Apple Vision is unavailable: %@",error.reason?:error.name]); |
| 76 | } |
| 77 | } |
| 78 |