| 1 | #!/usr/bin/env bash |
| 2 | # app-server runtime API release smoke. |
| 3 | # |
| 4 | # Two independent checks, both safe to run before every release: |
| 5 | # |
| 6 | # 1. app-server stdio probe (always): drives `codewhale app-server --stdio` |
| 7 | # with JSON-RPC health/capabilities requests and asserts the control |
| 8 | # surface answers. Spends no model tokens, makes no network calls, and runs |
| 9 | # against a throwaway config so it never reads the maintainer's real keys. |
| 10 | # |
| 11 | # 2. provider/model matrix (opt-in, --matrix): discovers configured providers |
| 12 | # from `codewhale auth list`, maps each to a cheap sentinel model, and |
| 13 | # either prints the plan (default, dry-run) or runs a tiny `exec` prompt per |
| 14 | # provider (--real). `auth list` reports presence flags only; secrets are |
| 15 | # never printed and exec output is passed through a redactor. |
| 16 | # |
| 17 | # Usage: |
| 18 | # scripts/release/app-server-smoke.sh # stdio probe only |
| 19 | # scripts/release/app-server-smoke.sh --matrix # + print provider matrix (dry-run) |
| 20 | # scripts/release/app-server-smoke.sh --matrix --real # + exec a sentinel per provider |
| 21 | # scripts/release/app-server-smoke.sh --matrix --provider deepseek --provider zai |
| 22 | # scripts/release/app-server-smoke.sh --bin ./target/release/codewhale |
| 23 | # |
| 24 | # Binary resolution order: --bin <path>, $CODEWHALE_BIN, ./target/release/codewhale, PATH. |
| 25 | # Per-provider cheap-model override: SMOKE_MODEL_<SLUG> with slug upper-cased and |
| 26 | # '-' replaced by '_', e.g. SMOKE_MODEL_XIAOMI_MIMO=mimo-7b. |
| 27 | |
| 28 | set -euo pipefail |
| 29 | |
| 30 | PASS=0 |
| 31 | FAIL=0 |
| 32 | BIN="${CODEWHALE_BIN:-}" |
| 33 | DO_MATRIX=0 |
| 34 | DRY_RUN=1 |
| 35 | SENTINEL="${SMOKE_SENTINEL:-Reply with exactly the single word: pong}" |
| 36 | EXEC_TIMEOUT="${SMOKE_EXEC_TIMEOUT:-60}" |
| 37 | declare -a ONLY_PROVIDERS=() |
| 38 | |
| 39 | # Best-effort CHEAP models per provider slot. These are intentionally overridable |
| 40 | # (SMOKE_MODEL_<SLUG>) because there is no committed route-effective model |
| 41 | # inventory yet (#3205); unmapped configured providers fail loudly in --real mode |
| 42 | # instead of guessing. Keep this list conservative: only add a provider here when |
| 43 | # its mapped id is a genuinely cheap/small model. Providers without a verified |
| 44 | # cheap default (e.g. arcee, openrouter, xiaomi-mimo, openai-codex) are left |
| 45 | # unmapped on purpose and must be given a model per run via SMOKE_MODEL_<SLUG>. |
| 46 | default_model_for() { |
| 47 | case "$1" in |
| 48 | deepseek) echo "deepseek-chat" ;; |
| 49 | zai) echo "glm-4-flash" ;; |
| 50 | moonshot) echo "moonshot-v1-8k" ;; |
| 51 | openai) echo "gpt-4o-mini" ;; |
| 52 | *) echo "" ;; |
| 53 | esac |
| 54 | } |
| 55 | |
| 56 | log() { printf '\033[1;34m>>> %s\033[0m\n' "$*"; } |
| 57 | pass() { printf '\033[1;32m \xe2\x9c\x93 %s\033[0m\n' "$*"; PASS=$((PASS + 1)); } |
| 58 | fail() { printf '\033[1;31m \xe2\x9c\x97 %s\033[0m\n' "$*"; FAIL=$((FAIL + 1)); } |
| 59 | note() { printf ' %s\n' "$*"; } |
| 60 | |
| 61 | usage() { sed -n '2,33p' "$0"; } |
| 62 | |
| 63 | # Mask anything that looks like a credential. Defense in depth: the CLI does not |
| 64 | # print secrets, but exec output is untrusted text. |
| 65 | redact() { |
| 66 | sed -E \ |
| 67 | -e 's/(sk-[A-Za-z0-9]{2})[A-Za-z0-9_-]+/\1…REDACTED/g' \ |
| 68 | -e 's/(Bearer +)[A-Za-z0-9._-]+/\1REDACTED/g' \ |
| 69 | -e 's/(([Aa][Pp][Ii][_-]?[Kk][Ee][Yy]|[Tt][Oo][Kk][Ee][Nn]|[Ss][Ee][Cc][Rr][Ee][Tt])["'"'"' :=]+)[^"'"'"' ,}]+/\1REDACTED/g' |
| 70 | } |
| 71 | |
| 72 | parse_args() { |
| 73 | while [[ $# -gt 0 ]]; do |
| 74 | case "$1" in |
| 75 | --matrix) DO_MATRIX=1 ;; |
| 76 | --real) DRY_RUN=0 ;; |
| 77 | --dry-run) DRY_RUN=1 ;; |
| 78 | --provider) shift; ONLY_PROVIDERS+=("${1:?--provider needs a value}") ;; |
| 79 | --bin) shift; BIN="${1:?--bin needs a path}" ;; |
| 80 | -h|--help) usage; exit 0 ;; |
| 81 | *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;; |
| 82 | esac |
| 83 | shift |
| 84 | done |
| 85 | } |
| 86 | |
| 87 | resolve_bin() { |
| 88 | if [[ -n "$BIN" ]]; then |
| 89 | [[ -x "$BIN" ]] || { echo "codewhale binary not executable: $BIN" >&2; exit 2; } |
| 90 | return |
| 91 | fi |
| 92 | if [[ -x "./target/release/codewhale" ]]; then |
| 93 | BIN="./target/release/codewhale" |
| 94 | elif command -v codewhale >/dev/null 2>&1; then |
| 95 | BIN="$(command -v codewhale)" |
| 96 | else |
| 97 | echo "could not find a codewhale binary." >&2 |
| 98 | echo " build one: cargo build -p codewhale-cli --release" >&2 |
| 99 | echo " or pass: --bin <path> / CODEWHALE_BIN=<path>" >&2 |
| 100 | exit 2 |
| 101 | fi |
| 102 | } |
| 103 | |
| 104 | # ── Check 1: app-server stdio control surface ──────────────────────────────── |
| 105 | |
| 106 | stdio_probe() { |
| 107 | log "=== app-server stdio probe (no model tokens) ===" |
| 108 | local tmp out |
| 109 | tmp="$(mktemp -d)" |
| 110 | # Throwaway config keeps the probe hermetic: no real keys read, state.db and |
| 111 | # events.jsonl land in the temp dir. |
| 112 | : >"$tmp/config.toml" |
| 113 | |
| 114 | out="$(printf '%s\n' \ |
| 115 | '{"jsonrpc":"2.0","id":1,"method":"healthz"}' \ |
| 116 | '{"jsonrpc":"2.0","id":2,"method":"capabilities"}' \ |
| 117 | '{"jsonrpc":"2.0","id":3,"method":"app/capabilities"}' \ |
| 118 | '{"jsonrpc":"2.0","id":4,"method":"prompt/capabilities"}' \ |
| 119 | '{"jsonrpc":"2.0","id":5,"method":"thread/capabilities"}' \ |
| 120 | '{"jsonrpc":"2.0","id":6,"method":"shutdown"}' \ |
| 121 | | "$BIN" app-server --stdio --config "$tmp/config.toml" 2>/dev/null || true)" |
| 122 | rm -rf "$tmp" |
| 123 | |
| 124 | if [[ -z "$out" ]]; then |
| 125 | fail "app-server --stdio produced no output" |
| 126 | return |
| 127 | fi |
| 128 | |
| 129 | probe_assert "$out" '"status":"ok"' "healthz reports ok" |
| 130 | probe_assert "$out" '"thread/request"' "capabilities advertise thread/* methods" |
| 131 | probe_assert "$out" '"prompt/run"' "capabilities advertise prompt/run" |
| 132 | probe_assert "$out" '"transport":"stdio+http"' "app/capabilities reports transport" |
| 133 | probe_assert "$out" '"prompt/request"' "prompt/capabilities lists prompt/request" |
| 134 | probe_assert "$out" '"thread/goal/set"' "thread/capabilities lists goal methods" |
| 135 | } |
| 136 | |
| 137 | probe_assert() { |
| 138 | local haystack="$1" needle="$2" desc="$3" |
| 139 | if printf '%s' "$haystack" | grep -qF "$needle"; then |
| 140 | pass "$desc" |
| 141 | else |
| 142 | fail "$desc (missing: $needle)" |
| 143 | fi |
| 144 | } |
| 145 | |
| 146 | # ── Check 2: provider/model matrix ─────────────────────────────────────────── |
| 147 | |
| 148 | # Echo configured provider slugs (active != missing) from `codewhale auth list`. |
| 149 | configured_providers() { |
| 150 | "$BIN" auth list 2>/dev/null \ |
| 151 | | awk 'NR > 1 && NF >= 2 && $NF != "missing" { print $1 }' |
| 152 | } |
| 153 | |
| 154 | model_for() { |
| 155 | local slug="$1" var |
| 156 | var="SMOKE_MODEL_$(printf '%s' "$slug" | tr '[:lower:]-' '[:upper:]_')" |
| 157 | if [[ -n "${!var:-}" ]]; then |
| 158 | printf '%s' "${!var}" |
| 159 | else |
| 160 | default_model_for "$slug" |
| 161 | fi |
| 162 | } |
| 163 | |
| 164 | want_provider() { |
| 165 | [[ ${#ONLY_PROVIDERS[@]} -eq 0 ]] && return 0 |
| 166 | local p |
| 167 | for p in "${ONLY_PROVIDERS[@]}"; do |
| 168 | [[ "$p" == "$1" ]] && return 0 |
| 169 | done |
| 170 | return 1 |
| 171 | } |
| 172 | |
| 173 | run_matrix() { |
| 174 | if [[ $DRY_RUN -eq 1 ]]; then |
| 175 | log "=== provider/model matrix (dry-run) ===" |
| 176 | else |
| 177 | log "=== provider/model matrix (real exec) ===" |
| 178 | fi |
| 179 | |
| 180 | local -a providers=() |
| 181 | local p |
| 182 | while IFS= read -r p; do |
| 183 | [[ -z "$p" ]] && continue |
| 184 | want_provider "$p" && providers+=("$p") |
| 185 | done < <(configured_providers) |
| 186 | |
| 187 | if [[ ${#providers[@]} -eq 0 ]]; then |
| 188 | note "no configured providers discovered (auth list); nothing to test" |
| 189 | return |
| 190 | fi |
| 191 | |
| 192 | local slug model |
| 193 | for slug in "${providers[@]}"; do |
| 194 | model="$(model_for "$slug")" |
| 195 | if [[ -z "$model" ]]; then |
| 196 | if [[ $DRY_RUN -eq 1 ]]; then |
| 197 | note "$slug -> (UNMAPPED; set SMOKE_MODEL_$(printf '%s' "$slug" | tr '[:lower:]-' '[:upper:]_'))" |
| 198 | else |
| 199 | fail "$slug has no cheap-model mapping (set SMOKE_MODEL_$(printf '%s' "$slug" | tr '[:lower:]-' '[:upper:]_')=<model>)" |
| 200 | fi |
| 201 | continue |
| 202 | fi |
| 203 | |
| 204 | if [[ $DRY_RUN -eq 1 ]]; then |
| 205 | note "$slug -> $model [$BIN --provider $slug --model $model exec \"<sentinel>\"]" |
| 206 | continue |
| 207 | fi |
| 208 | |
| 209 | local out rc=0 |
| 210 | if command -v timeout >/dev/null 2>&1; then |
| 211 | out="$(timeout "$EXEC_TIMEOUT" "$BIN" --provider "$slug" --model "$model" exec "$SENTINEL" 2>&1)" || rc=$? |
| 212 | else |
| 213 | out="$("$BIN" --provider "$slug" --model "$model" exec "$SENTINEL" 2>&1)" || rc=$? |
| 214 | fi |
| 215 | if [[ $rc -eq 0 ]]; then |
| 216 | pass "$slug/$model exec ok" |
| 217 | note "$(printf '%s' "$out" | redact | tail -n 1)" |
| 218 | else |
| 219 | fail "$slug/$model exec failed (rc=$rc)" |
| 220 | note "$(printf '%s' "$out" | redact | tail -n 3)" |
| 221 | fi |
| 222 | done |
| 223 | } |
| 224 | |
| 225 | main() { |
| 226 | parse_args "$@" |
| 227 | resolve_bin |
| 228 | log "Using binary: $BIN" |
| 229 | |
| 230 | stdio_probe |
| 231 | if [[ $DO_MATRIX -eq 1 ]]; then |
| 232 | run_matrix |
| 233 | else |
| 234 | note "(provider/model matrix skipped; pass --matrix to enable)" |
| 235 | fi |
| 236 | |
| 237 | echo "" |
| 238 | log "Results: $PASS passed, $FAIL failed" |
| 239 | [[ $FAIL -eq 0 ]] |
| 240 | } |
| 241 | |
| 242 | main "$@" |
| 243 |