| 1 | import { test } from 'node:test'; |
| 2 | import assert from 'node:assert/strict'; |
| 3 | import fs from 'node:fs'; |
| 4 | import os from 'node:os'; |
| 5 | import path from 'node:path'; |
| 6 | import { spawnSync } from 'node:child_process'; |
| 7 | import { create } from '../src/backends/darwin.mjs'; |
| 8 | |
| 9 | function fixture(t, options = {}) { |
| 10 | const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-ocr-')); |
| 11 | const saved = [process.env.CODEWHALE_CU_APP_BUNDLE, process.env.CODEWHALE_CU_RECORDINGS_DIR]; |
| 12 | t.after(() => { |
| 13 | for (const [index, name] of ['CODEWHALE_CU_APP_BUNDLE', 'CODEWHALE_CU_RECORDINGS_DIR'].entries()) { |
| 14 | if (saved[index] === undefined) delete process.env[name]; else process.env[name] = saved[index]; |
| 15 | } |
| 16 | fs.rmSync(root, { recursive: true, force: true }); |
| 17 | }); |
| 18 | fs.mkdirSync(path.join(root, 'Contents', 'MacOS'), { recursive: true }); |
| 19 | fs.writeFileSync(path.join(root, 'Contents', 'MacOS', 'accessibility'), ''); |
| 20 | process.env.CODEWHALE_CU_APP_BUNDLE = root; |
| 21 | process.env.CODEWHALE_CU_RECORDINGS_DIR = path.join(root, 'captures'); |
| 22 | const observed = { found: true, pid: 731, bundle_id: 'test.ocr', elements: [{ index: 0, role: 'AXButton', label: 'Save', actions: ['AXPress'], windowIndex: 1, path: [0] }] }; |
| 23 | if (options.missingIdentity) delete observed.pid; |
| 24 | const calls = []; |
| 25 | const backend = create({ exec: { async run(cmd, args) { |
| 26 | if (cmd === 'screencapture') { |
| 27 | calls.push({ tool: 'screencapture', args }); |
| 28 | if (options.captureFailure) return { code: 1, stderr: 'Screen Recording permission denied', stdout: '' }; |
| 29 | // A real PNG signature and IHDR: the backend identifies the raster |
| 30 | // format before trusting its dimensions, so a headerless stub is not a |
| 31 | // faithful stand-in for what screencapture writes. |
| 32 | const header = Buffer.alloc(24); |
| 33 | Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]).copy(header, 0); |
| 34 | header.writeUInt32BE(13, 8); header.write('IHDR', 12, 'ascii'); |
| 35 | header.writeUInt32BE(200, 16); header.writeUInt32BE(100, 20); |
| 36 | fs.writeFileSync(args.at(-1), header); |
| 37 | return { code: 0, stdout: '', stderr: '' }; |
| 38 | } |
| 39 | const request = JSON.parse(args[0]); calls.push(request); |
| 40 | let body; |
| 41 | if (request.tool === 'get_app_state') body = observed; |
| 42 | else if (request.tool === 'input_capabilities') body = { window_ocr: options.oldHelper ? 0 : 1 }; |
| 43 | else if (request.tool === 'window_info') body = { window_id: 901, points: { x: -400, y: 200, w: 100, h: 50 } }; |
| 44 | else if (request.tool === 'displays') body = [{ index: 1, scale: 2 }]; |
| 45 | else if (request.tool === 'recognize_text') { |
| 46 | if (options.cancelled) return { code: -1, stdout: '', stderr: '', aborted: true }; |
| 47 | body = options.ocr ?? { status: 'ok', engine: 'apple_vision', coordinate_space: 'raster_pixels', pixels: { w: 200, h: 100 }, blocks: [{ text: 'Invoice total', confidence: 0.92, bounds: { x: 20, y: 10, w: 40, h: 20 } }] }; |
| 48 | } else assert.fail(`unexpected native tool ${request.tool}`); |
| 49 | return { code: 0, stdout: JSON.stringify(body), stderr: '' }; |
| 50 | } } }); |
| 51 | return { backend, calls, observed, root }; |
| 52 | } |
| 53 | |
| 54 | test('default app state remains AX-only without capture or OCR work', async t => { |
| 55 | const { backend, calls, observed } = fixture(t); |
| 56 | assert.deepEqual(await backend.get_app_state({ app_ref: { pid: 731 } }), observed); |
| 57 | assert.deepEqual(calls.map(call => call.tool), ['get_app_state']); |
| 58 | }); |
| 59 | |
| 60 | test('opt-in OCR captures the resolved selected window and exposes raster-pixel text targets', async t => { |
| 61 | const { backend, calls, observed } = fixture(t); |
| 62 | const result = await backend.get_app_state({ app_ref: { name: 'Test' }, window_id: 1, include_ocr: true }); |
| 63 | assert.deepEqual(result.elements, observed.elements, 'AX roles and identity remain unchanged'); |
| 64 | const window = calls.find(call => call.tool === 'window_info'); |
| 65 | assert.deepEqual(window.args.app_ref, { pid: 731, bundle_id: 'test.ocr' }); |
| 66 | assert.equal(window.args.window_id, 1); |
| 67 | const capture = calls.find(call => call.tool === 'screencapture'); |
| 68 | assert.deepEqual(capture.args.slice(0, -1), ['-x', '-t', 'png', '-o', '-l', '901']); |
| 69 | assert.equal(result.ocr.status, 'ok'); |
| 70 | assert.deepEqual(result.ocr.blocks[0].target, { type: 'coordinate', x: 40, y: 20 }); |
| 71 | assert.equal(result.ocr.blocks[0].role, undefined, 'recognized text is not an accessibility control'); |
| 72 | assert.deepEqual(result.ocr.raster.points, { x: -400, y: 200, w: 100, h: 50 }); |
| 73 | assert.equal(result.ocr.raster.scale, 2); |
| 74 | assert.equal(calls.find(call => call.tool === 'recognize_text').args.file, result.ocr.raster.file); |
| 75 | }); |
| 76 | |
| 77 | test('capture permission failure preserves valid AX state with an explicit OCR diagnosis', async t => { |
| 78 | const { backend, calls, observed } = fixture(t, { captureFailure: true }); |
| 79 | const result = await backend.get_app_state({ app_ref: { pid: 731 }, include_ocr: true }); |
| 80 | assert.deepEqual(result.elements, observed.elements); |
| 81 | assert.equal(result.ocr.status, 'unavailable'); |
| 82 | assert.match(result.ocr.reason, /Screen Recording permission denied/); |
| 83 | assert.equal(result.ocr.raster, undefined); |
| 84 | assert.ok(!calls.some(call => call.tool === 'recognize_text')); |
| 85 | }); |
| 86 | |
| 87 | test('OCR failure keeps AX state and captured geometry but never invents text targets', async t => { |
| 88 | const { backend, observed } = fixture(t, { ocr: { status: 'unavailable', reason: 'Apple Vision unavailable' } }); |
| 89 | const result = await backend.get_app_state({ app_ref: { pid: 731 }, include_ocr: true }); |
| 90 | assert.deepEqual(result.elements, observed.elements); |
| 91 | assert.equal(result.ocr.reason, 'Apple Vision unavailable'); |
| 92 | assert.deepEqual(result.ocr.blocks, []); |
| 93 | assert.deepEqual(result.ocr.raster.pixels, { w: 200, h: 100 }, 'MCP can bind the new capture even when OCR fails'); |
| 94 | }); |
| 95 | |
| 96 | test('OCR image dimension mismatch fails without discarding AX state', async t => { |
| 97 | const { backend, observed } = fixture(t, { ocr: { status: 'ok', pixels: { w: 400, h: 200 }, blocks: [] } }); |
| 98 | const result = await backend.get_app_state({ app_ref: { pid: 731 }, include_ocr: true }); |
| 99 | assert.deepEqual(result.elements, observed.elements); |
| 100 | assert.equal(result.ocr.status, 'unavailable'); |
| 101 | assert.match(result.ocr.reason, /mismatched image dimensions/); |
| 102 | }); |
| 103 | |
| 104 | test('missing resolved identity cannot redirect optional OCR to the foreground application', async t => { |
| 105 | const { backend, calls } = fixture(t, { missingIdentity: true }); |
| 106 | const result = await backend.get_app_state({ app_ref: { name: 'Test' }, include_ocr: true }); |
| 107 | assert.equal(result.ocr.status, 'unavailable'); |
| 108 | assert.match(result.ocr.reason, /exact process identity/); |
| 109 | assert.deepEqual(calls.map(call => call.tool), ['get_app_state']); |
| 110 | }); |
| 111 | |
| 112 | test('an older helper is refused before it can capture a different window', async t => { |
| 113 | const { backend, calls } = fixture(t, { oldHelper: true }); |
| 114 | const result = await backend.get_app_state({ app_ref: { pid: 731 }, window_id: 1, include_ocr: true }); |
| 115 | assert.equal(result.ocr.status, 'unavailable'); |
| 116 | assert.match(result.ocr.reason, /helper needs an update/); |
| 117 | assert.deepEqual(calls.map(call => call.tool), ['get_app_state', 'input_capabilities']); |
| 118 | }); |
| 119 | |
| 120 | test('OCR cancellation remains cancellation', async t => { |
| 121 | const { backend } = fixture(t, { cancelled: true }); |
| 122 | await assert.rejects(backend.get_app_state({ app_ref: { pid: 731 }, include_ocr: true }), error => error.code === 'cancelled'); |
| 123 | }); |
| 124 | |
| 125 | test('Apple Vision recognizes a generated image with correct raster bounds and rejects invalid images', { skip: process.platform !== 'darwin' }, t => { |
| 126 | const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cu-vision-test-')); |
| 127 | t.after(() => fs.rmSync(root, { recursive: true, force: true })); |
| 128 | const source = path.join(root, 'probe.m'), binary = path.join(root, 'probe'), image = path.join(root, 'text.png'); |
| 129 | fs.writeFileSync(source, `#import ${JSON.stringify(path.resolve('src/backends/darwin-ocr.h'))} |
| 130 | int main(int argc,const char **argv) { @autoreleasepool { |
| 131 | NSString *file=[NSString stringWithUTF8String:argv[1]]; |
| 132 | NSBitmapImageRep *bitmap=[[NSBitmapImageRep alloc] initWithBitmapDataPlanes:NULL pixelsWide:1000 pixelsHigh:320 bitsPerSample:8 samplesPerPixel:4 hasAlpha:YES isPlanar:NO colorSpaceName:NSDeviceRGBColorSpace bytesPerRow:0 bitsPerPixel:0]; |
| 133 | [NSGraphicsContext saveGraphicsState]; |
| 134 | [NSGraphicsContext setCurrentContext:[NSGraphicsContext graphicsContextWithBitmapImageRep:bitmap]]; |
| 135 | [NSColor.whiteColor setFill]; NSRectFill(NSMakeRect(0,0,1000,320)); |
| 136 | [@"Codewhale OCR 7319" drawAtPoint:NSMakePoint(50,220) withAttributes:@{NSFontAttributeName:[NSFont systemFontOfSize:48],NSForegroundColorAttributeName:NSColor.blackColor}]; |
| 137 | [@"Invoice total 42.50" drawAtPoint:NSMakePoint(50,70) withAttributes:@{NSFontAttributeName:[NSFont systemFontOfSize:40],NSForegroundColorAttributeName:NSColor.blackColor}]; |
| 138 | [NSGraphicsContext restoreGraphicsState]; |
| 139 | [[bitmap representationUsingType:NSBitmapImageFileTypePNG properties:@{}] writeToFile:file atomically:YES]; |
| 140 | NSDictionary *recognized=cuRecognizeText(file); |
| 141 | [@"invalid image" writeToFile:file atomically:YES encoding:NSUTF8StringEncoding error:nil]; |
| 142 | NSDictionary *result=@{@"recognized":recognized,@"invalid":cuRecognizeText(file),@"missing":cuRecognizeText([file stringByAppendingString:@".missing"]),@"bounds":cuOCRPixelBounds(CGRectMake(.125,.25,.25,.5),1000,800),@"clipped":cuOCRPixelBounds(CGRectMake(-.5,-.5,2,2),1000,800)}; |
| 143 | NSData *json=[NSJSONSerialization dataWithJSONObject:result options:0 error:nil]; |
| 144 | puts([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding].UTF8String); |
| 145 | } return 0; }`); |
| 146 | const build = spawnSync('clang', ['-fobjc-arc', '-Os', '-framework', 'Cocoa', '-framework', 'Vision', source, '-o', binary], { encoding: 'utf8' }); |
| 147 | assert.equal(build.status, 0, build.stderr); |
| 148 | const run = spawnSync(binary, [image], { encoding: 'utf8', timeout: 20000 }); |
| 149 | assert.equal(run.status, 0, run.stderr); |
| 150 | const result = JSON.parse(run.stdout); |
| 151 | assert.equal(result.recognized.status, 'ok', JSON.stringify(result.recognized)); |
| 152 | const first = result.recognized.blocks.find(block => block.text === 'Codewhale OCR 7319'); |
| 153 | const second = result.recognized.blocks.find(block => block.text === 'Invoice total 42.50'); |
| 154 | assert.ok(first && second, JSON.stringify(result.recognized)); |
| 155 | assert.ok(first.confidence > 0.8 && second.confidence > 0.8); |
| 156 | assert.ok(first.bounds.x >= 40 && first.bounds.x < 65 && first.bounds.y >= 40 && first.bounds.y < 100); |
| 157 | assert.ok(second.bounds.y > 200 && second.bounds.y < 260, 'Vision lower-left coordinates became top-left raster coordinates'); |
| 158 | assert.deepEqual(result.bounds, { x: 125, y: 200, w: 250, h: 400 }); |
| 159 | assert.deepEqual(result.clipped, { x: 0, y: 0, w: 1000, h: 800 }); |
| 160 | assert.equal(result.invalid.status, 'unavailable'); |
| 161 | assert.equal(result.missing.status, 'unavailable'); |
| 162 | }); |
| 163 |