| 1 | """Run Codewhale through a Verifiers v0.2 interception endpoint. |
| 2 | |
| 3 | The harness owns an isolated Codewhale home for every rollout, forwards only |
| 4 | the interception secret supplied by Verifiers, and retains a bounded terminal |
| 5 | receipt rather than copying raw program output into trace metadata. |
| 6 | """ |
| 7 | |
| 8 | from __future__ import annotations |
| 9 | |
| 10 | import hashlib |
| 11 | import json |
| 12 | import logging |
| 13 | import re |
| 14 | import shlex |
| 15 | from collections import Counter |
| 16 | from typing import Any, Literal |
| 17 | |
| 18 | from verifiers.v1.clients import ModelContext |
| 19 | from verifiers.v1.harness import Harness, HarnessConfig |
| 20 | from verifiers.v1.runtimes import ProgramResult, Runtime |
| 21 | from verifiers.v1.trace import Trace |
| 22 | |
| 23 | logger = logging.getLogger(__name__) |
| 24 | |
| 25 | INSTALL_DIR = "/tmp/vf-codewhale" |
| 26 | DEFAULT_BINARY = f"{INSTALL_DIR}/bin/codewhale" |
| 27 | RELEASE_ROOT = "https://github.com/Hmbown/CodeWhale/releases/download" |
| 28 | STREAM_SCHEMA = "codewhale.exec-stream" |
| 29 | STREAM_SCHEMA_VERSION = 1 |
| 30 | MAX_TERMINAL_RECEIPT_BYTES = 8_192 |
| 31 | MAX_TERMINAL_STRING_CHARS = 512 |
| 32 | _VERSION = re.compile(r"^[0-9]+\.[0-9]+\.[0-9]+(?:[-+][0-9A-Za-z.-]+)?$") |
| 33 | _TOOL = re.compile(r"^[a-z0-9][a-z0-9_-]*$") |
| 34 | _SHA256 = re.compile(r"^sha256:[0-9a-f]{64}$") |
| 35 | _EVENT_TYPES = { |
| 36 | "content", |
| 37 | "tool_use", |
| 38 | "tool_result", |
| 39 | "sandbox_denied", |
| 40 | "workflow_event", |
| 41 | "session_capture", |
| 42 | "turn_usage", |
| 43 | "metadata", |
| 44 | "done", |
| 45 | "error", |
| 46 | } |
| 47 | _TERMINAL_FIELDS = { |
| 48 | "receipt_kind", |
| 49 | "provider", |
| 50 | "provider_id", |
| 51 | "model", |
| 52 | "route_source", |
| 53 | "input_tokens", |
| 54 | "output_tokens", |
| 55 | "prompt_cache_hit_tokens", |
| 56 | "prompt_cache_miss_tokens", |
| 57 | "prompt_cache_write_tokens", |
| 58 | "reasoning_tokens", |
| 59 | "duration_ms", |
| 60 | "retry_count", |
| 61 | "approval_posture", |
| 62 | "sandbox_posture", |
| 63 | "binary_sha256", |
| 64 | "config_sha256", |
| 65 | "prompt_sha256", |
| 66 | "tool_catalog_sha256", |
| 67 | "visible_final_answer_chars", |
| 68 | "message_count", |
| 69 | "status", |
| 70 | "termination_reason", |
| 71 | "error_category", |
| 72 | } |
| 73 | |
| 74 | |
| 75 | class CodewhaleHarnessConfig(HarnessConfig): |
| 76 | version: str = "0.9.1" |
| 77 | """Codewhale release to install, pinned for reproducible rollouts.""" |
| 78 | |
| 79 | binary_path: str | None = None |
| 80 | """Preinstalled facade inside the runtime; useful for local candidate testing.""" |
| 81 | |
| 82 | max_turns: int | None = None |
| 83 | """Optional model-step ceiling; omitted rollouts are unlimited.""" |
| 84 | |
| 85 | sandbox: Literal["auto", "read-only", "workspace-write", "external-sandbox"] = ( |
| 86 | "auto" |
| 87 | ) |
| 88 | """`auto` keeps subprocess runs workspace-bound and trusts isolated runtimes.""" |
| 89 | |
| 90 | |
| 91 | class CodewhaleHarness(Harness[CodewhaleHarnessConfig]): |
| 92 | APPENDS_SYSTEM_PROMPT = True |
| 93 | SUPPORTS_MCP = True |
| 94 | SUPPORTS_USER_SIM = False |
| 95 | |
| 96 | def _validate_config(self) -> None: |
| 97 | if not _VERSION.fullmatch(self.config.version): |
| 98 | raise ValueError("version must be a semantic release identifier") |
| 99 | if ( |
| 100 | self.config.max_turns is not None |
| 101 | and not 1 <= self.config.max_turns <= 10_000 |
| 102 | ): |
| 103 | raise ValueError("max_turns must be between 1 and 10000") |
| 104 | invalid = [ |
| 105 | tool |
| 106 | for tool in self.config.disabled_tools or [] |
| 107 | if not _TOOL.fullmatch(tool) |
| 108 | ] |
| 109 | if invalid: |
| 110 | raise ValueError( |
| 111 | "disabled_tools must use Codewhale catalog identifiers: " |
| 112 | + ", ".join(repr(tool) for tool in invalid) |
| 113 | ) |
| 114 | |
| 115 | @property |
| 116 | def binary(self) -> str: |
| 117 | configured = (self.config.binary_path or "").strip() |
| 118 | return configured or DEFAULT_BINARY |
| 119 | |
| 120 | async def setup(self, runtime: Runtime) -> None: |
| 121 | self._validate_config() |
| 122 | if self.config.binary_path: |
| 123 | logger.info("codewhale: verifying preinstalled %s", self.binary) |
| 124 | result = await runtime.run([self.binary, "--version"], {}) |
| 125 | version_text = f"{result.stdout}\n{result.stderr}" |
| 126 | if result.exit_code != 0 or not _has_version( |
| 127 | version_text, self.config.version |
| 128 | ): |
| 129 | raise RuntimeError( |
| 130 | "configured Codewhale binary is unavailable or does not report " |
| 131 | f"version {self.config.version}" |
| 132 | ) |
| 133 | return |
| 134 | |
| 135 | logger.info("codewhale: ensuring Codewhale %s is installed", self.config.version) |
| 136 | script = _install_script(self.config.version) |
| 137 | result = await runtime.run(["sh", "-c", script], {}) |
| 138 | if result.exit_code != 0: |
| 139 | raise RuntimeError( |
| 140 | "Codewhale install failed: " |
| 141 | + (result.stderr or result.stdout).strip()[-500:] |
| 142 | ) |
| 143 | |
| 144 | async def launch( |
| 145 | self, |
| 146 | ctx: ModelContext, |
| 147 | trace: Trace, |
| 148 | runtime: Runtime, |
| 149 | endpoint: str, |
| 150 | secret: str, |
| 151 | mcp_urls: dict[str, str], |
| 152 | ) -> ProgramResult: |
| 153 | self._validate_config() |
| 154 | system, prompt = self.resolve_prompt(trace.task.data) |
| 155 | trace_key = hashlib.sha256(str(trace.id).encode()).hexdigest()[:32] |
| 156 | home = f".vf-codewhale/{trace_key}" |
| 157 | mcp_path = f"{home}/mcp.json" |
| 158 | mcp = {"servers": {name: {"url": url} for name, url in mcp_urls.items()}} |
| 159 | await runtime.write( |
| 160 | mcp_path, |
| 161 | json.dumps(mcp, sort_keys=True, separators=(",", ":")).encode(), |
| 162 | ) |
| 163 | |
| 164 | endpoint = endpoint.rstrip("/") |
| 165 | env = { |
| 166 | **self.config.resolved_env, |
| 167 | "CODEWHALE_HOME": home, |
| 168 | "CODEWHALE_PROVIDER": "openai", |
| 169 | "DEEPSEEK_PROVIDER": "openai", |
| 170 | "CODEWHALE_MODEL": ctx.model, |
| 171 | "DEEPSEEK_MODEL": ctx.model, |
| 172 | "OPENAI_MODEL": ctx.model, |
| 173 | "OPENAI_BASE_URL": endpoint, |
| 174 | "OPENAI_API_KEY": secret, |
| 175 | "CODEWHALE_MCP_CONFIG": mcp_path, |
| 176 | "DEEPSEEK_MCP_CONFIG": mcp_path, |
| 177 | "CODEWHALE_TELEMETRY": "false", |
| 178 | "DEEPSEEK_TELEMETRY": "false", |
| 179 | "CODEWHALE_MEMORY": "false", |
| 180 | "NO_COLOR": "1", |
| 181 | } |
| 182 | if endpoint.startswith("http://") or any( |
| 183 | url.startswith("http://") for url in mcp_urls.values() |
| 184 | ): |
| 185 | # Verifiers' interception and colocated MCP endpoints are often |
| 186 | # ephemeral HTTP services inside an already-isolated runtime. |
| 187 | env["CODEWHALE_ALLOW_INSECURE_HTTP"] = "1" |
| 188 | |
| 189 | sandbox = self.config.sandbox |
| 190 | if sandbox == "auto": |
| 191 | sandbox = ( |
| 192 | "workspace-write" |
| 193 | if runtime.type == "subprocess" |
| 194 | else "external-sandbox" |
| 195 | ) |
| 196 | argv = [ |
| 197 | self.binary, |
| 198 | "--provider", |
| 199 | "openai", |
| 200 | "--model", |
| 201 | ctx.model, |
| 202 | "--telemetry", |
| 203 | "false", |
| 204 | "--workspace", |
| 205 | ".", |
| 206 | "--skip-onboarding", |
| 207 | "--no-project-config", |
| 208 | "exec", |
| 209 | "--auto", |
| 210 | "--sandbox", |
| 211 | sandbox, |
| 212 | "--output-format", |
| 213 | "stream-json", |
| 214 | ] |
| 215 | if self.config.max_turns is not None: |
| 216 | argv.extend(["--max-turns", str(self.config.max_turns)]) |
| 217 | if self.config.disabled_tools: |
| 218 | argv.extend(["--disallowed-tools", ",".join(self.config.disabled_tools)]) |
| 219 | if system: |
| 220 | argv.extend(["--append-system-prompt", system]) |
| 221 | argv.extend(["--", str(prompt or "")]) |
| 222 | |
| 223 | result = await runtime.run_program(argv, env) |
| 224 | if result.exit_code == 0: |
| 225 | receipt = _parse_stream_receipt(result.stdout) |
| 226 | terminal = receipt["terminal"] |
| 227 | if terminal.get("provider") != "openai": |
| 228 | raise RuntimeError("Codewhale terminal receipt did not use provider openai") |
| 229 | if terminal.get("model") != ctx.model: |
| 230 | raise RuntimeError("Codewhale terminal receipt model did not match rollout") |
| 231 | if terminal.get("approval_posture") != "auto_tools": |
| 232 | raise RuntimeError("Codewhale terminal receipt did not confirm auto tools") |
| 233 | if terminal.get("sandbox_posture") != sandbox: |
| 234 | raise RuntimeError("Codewhale terminal receipt sandbox did not match launch") |
| 235 | if receipt["events"].get("error", 0) != 0: |
| 236 | raise RuntimeError("Codewhale successful run contained an error event") |
| 237 | if terminal.get("status") != "completed": |
| 238 | raise RuntimeError("Codewhale terminal receipt did not report completion") |
| 239 | if terminal.get("termination_reason") != "resolved": |
| 240 | raise RuntimeError("Codewhale terminal receipt was not resolved") |
| 241 | trace.info["codewhale"] = receipt |
| 242 | return result |
| 243 | |
| 244 | |
| 245 | def _has_version(output: str, version: str) -> bool: |
| 246 | return ( |
| 247 | re.search( |
| 248 | rf"(?<![0-9A-Za-z.+-]){re.escape(version)}(?![0-9A-Za-z.+-])", |
| 249 | output, |
| 250 | ) |
| 251 | is not None |
| 252 | ) |
| 253 | |
| 254 | |
| 255 | def _bounded_terminal(meta: dict[str, Any]) -> dict[str, Any]: |
| 256 | terminal: dict[str, Any] = {} |
| 257 | for key in _TERMINAL_FIELDS: |
| 258 | if key not in meta or meta[key] is None: |
| 259 | continue |
| 260 | value = meta[key] |
| 261 | if isinstance(value, str): |
| 262 | if len(value) > MAX_TERMINAL_STRING_CHARS: |
| 263 | raise RuntimeError("Codewhale terminal receipt exceeded its string bound") |
| 264 | elif isinstance(value, int) and not isinstance(value, bool): |
| 265 | if value < 0 or value > 2**63 - 1: |
| 266 | raise RuntimeError("Codewhale terminal receipt contained an invalid count") |
| 267 | else: |
| 268 | raise RuntimeError("Codewhale terminal receipt contained a non-scalar field") |
| 269 | terminal[key] = value |
| 270 | encoded = json.dumps(terminal, sort_keys=True, separators=(",", ":")).encode() |
| 271 | if len(encoded) > MAX_TERMINAL_RECEIPT_BYTES: |
| 272 | raise RuntimeError("Codewhale terminal receipt exceeded its total bound") |
| 273 | return terminal |
| 274 | |
| 275 | |
| 276 | def _install_script(version: str) -> str: |
| 277 | version_q = shlex.quote(version) |
| 278 | install_q = shlex.quote(INSTALL_DIR) |
| 279 | release_q = shlex.quote(RELEASE_ROOT) |
| 280 | return f""" |
| 281 | set -eu |
| 282 | version={version_q} |
| 283 | install_dir={install_q} |
| 284 | release_root={release_q} |
| 285 | if [ "$(uname -s)" != Linux ]; then |
| 286 | echo "automatic Codewhale installation supports Linux runtimes; set binary_path" >&2 |
| 287 | exit 1 |
| 288 | fi |
| 289 | case "$(uname -m)" in |
| 290 | x86_64|amd64) platform=linux-x64 ;; |
| 291 | aarch64|arm64) platform=linux-arm64 ;; |
| 292 | *) echo "unsupported Codewhale runtime architecture: $(uname -m)" >&2; exit 1 ;; |
| 293 | esac |
| 294 | if ! command -v curl >/dev/null 2>&1 \ |
| 295 | || ! command -v sha256sum >/dev/null 2>&1 \ |
| 296 | || ! command -v flock >/dev/null 2>&1; then |
| 297 | if command -v apt-get >/dev/null 2>&1; then |
| 298 | apt-get update -qq |
| 299 | apt-get install -y -qq curl ca-certificates coreutils util-linux >/dev/null |
| 300 | elif command -v apk >/dev/null 2>&1; then |
| 301 | apk add --no-cache curl ca-certificates coreutils util-linux >/dev/null |
| 302 | else |
| 303 | echo "Codewhale install needs curl, sha256sum, and flock" >&2 |
| 304 | exit 1 |
| 305 | fi |
| 306 | fi |
| 307 | mkdir -p "$install_dir" |
| 308 | exec 9>"$install_dir/install.lock" |
| 309 | flock 9 |
| 310 | mkdir -p "$install_dir/bin" |
| 311 | if [ -f "$install_dir/bin/.version" ] \ |
| 312 | && [ "$(cat "$install_dir/bin/.version")" = "$version" ] \ |
| 313 | && (cd "$install_dir/bin" && sha256sum -c .sha256 >/dev/null 2>&1); then |
| 314 | exit 0 |
| 315 | fi |
| 316 | tmp="$(mktemp -d "$install_dir/install.XXXXXX")" |
| 317 | trap 'rm -rf "$tmp"' EXIT HUP INT TERM |
| 318 | base="$release_root/v$version" |
| 319 | curl -fsSL "$base/codewhale-artifacts-sha256.txt" -o "$tmp/manifest" |
| 320 | for pair in \ |
| 321 | "codewhale-$platform:codewhale" \ |
| 322 | "codew-$platform:codew" \ |
| 323 | "codewhale-tui-$platform:codewhale-tui" |
| 324 | do |
| 325 | asset="${{pair%%:*}}" |
| 326 | target="${{pair#*:}}" |
| 327 | curl -fsSL "$base/$asset" -o "$tmp/$asset" |
| 328 | expected="$(awk -v asset="$asset" '$2 == asset {{ print $1; exit }}' "$tmp/manifest")" |
| 329 | actual="$(sha256sum "$tmp/$asset" | awk '{{print $1}}')" |
| 330 | if [ -z "$expected" ] || [ "$actual" != "$expected" ]; then |
| 331 | echo "Codewhale checksum verification failed for $asset" >&2 |
| 332 | exit 1 |
| 333 | fi |
| 334 | cp "$tmp/$asset" "$install_dir/bin/$target.tmp.$$" |
| 335 | chmod 0755 "$install_dir/bin/$target.tmp.$$" |
| 336 | mv -f "$install_dir/bin/$target.tmp.$$" "$install_dir/bin/$target" |
| 337 | done |
| 338 | (cd "$install_dir/bin" && sha256sum codewhale codew codewhale-tui > .sha256.tmp) |
| 339 | mv -f "$install_dir/bin/.sha256.tmp" "$install_dir/bin/.sha256" |
| 340 | printf '%s' "$version" > "$install_dir/bin/.version.tmp" |
| 341 | mv -f "$install_dir/bin/.version.tmp" "$install_dir/bin/.version" |
| 342 | """ |
| 343 | |
| 344 | |
| 345 | def _parse_stream_receipt(stdout: str) -> dict[str, Any]: |
| 346 | counts: Counter[str] = Counter() |
| 347 | terminal: dict[str, Any] | None = None |
| 348 | ordered_types: list[str] = [] |
| 349 | for line_number, line in enumerate(stdout.splitlines(), start=1): |
| 350 | if not line.strip(): |
| 351 | continue |
| 352 | try: |
| 353 | event = json.loads(line) |
| 354 | except json.JSONDecodeError as error: |
| 355 | raise RuntimeError( |
| 356 | f"Codewhale stream-json line {line_number} was not valid JSON" |
| 357 | ) from error |
| 358 | if not isinstance(event, dict): |
| 359 | raise RuntimeError( |
| 360 | f"Codewhale stream-json line {line_number} was not an object" |
| 361 | ) |
| 362 | if event.get("schema") != STREAM_SCHEMA or event.get( |
| 363 | "schema_version" |
| 364 | ) != STREAM_SCHEMA_VERSION: |
| 365 | raise RuntimeError("Codewhale stream-json schema did not match v0.9.1") |
| 366 | event_type = event.get("type") |
| 367 | if event_type not in _EVENT_TYPES: |
| 368 | raise RuntimeError("Codewhale stream-json contained an unknown event type") |
| 369 | counts[event_type] += 1 |
| 370 | ordered_types.append(event_type) |
| 371 | if event_type == "metadata": |
| 372 | if terminal is not None: |
| 373 | raise RuntimeError("Codewhale emitted more than one terminal metadata receipt") |
| 374 | meta = event.get("meta") |
| 375 | if not isinstance(meta, dict) or meta.get("receipt_kind") != "terminal": |
| 376 | raise RuntimeError("Codewhale metadata event was not a terminal receipt") |
| 377 | terminal = _bounded_terminal(meta) |
| 378 | |
| 379 | if terminal is None: |
| 380 | raise RuntimeError("Codewhale stream-json omitted terminal metadata") |
| 381 | if counts["done"] != 1 or not ordered_types or ordered_types[-1] != "done": |
| 382 | raise RuntimeError("Codewhale stream-json did not end with exactly one done event") |
| 383 | if ordered_types[-2:-1] != ["metadata"]: |
| 384 | raise RuntimeError("Codewhale terminal metadata did not immediately precede done") |
| 385 | for field in ["binary_sha256", "prompt_sha256"]: |
| 386 | if not isinstance(terminal.get(field), str) or not _SHA256.fullmatch( |
| 387 | terminal[field] |
| 388 | ): |
| 389 | raise RuntimeError(f"Codewhale terminal receipt omitted a valid {field}") |
| 390 | return { |
| 391 | "schema": STREAM_SCHEMA, |
| 392 | "schema_version": STREAM_SCHEMA_VERSION, |
| 393 | "events": dict(sorted(counts.items())), |
| 394 | "terminal": terminal, |
| 395 | } |
| 396 |