| 1 | #!/usr/bin/env node |
| 2 | |
| 3 | const assert = require("node:assert/strict"); |
| 4 | const fs = require("node:fs"); |
| 5 | const path = require("node:path"); |
| 6 | const vm = require("node:vm"); |
| 7 | |
| 8 | const repoRoot = path.resolve(__dirname, "..", ".."); |
| 9 | const { |
| 10 | allAssetNames, |
| 11 | allReleaseAssetNames, |
| 12 | BUNDLE_ASSET_NAMES, |
| 13 | LEGACY_TUI_BRIDGE_ASSET_NAMES, |
| 14 | } = require(path.join(repoRoot, "npm", "codewhale", "scripts", "artifacts")); |
| 15 | |
| 16 | function read(relativePath) { |
| 17 | return fs.readFileSync(path.join(repoRoot, relativePath), "utf8"); |
| 18 | } |
| 19 | |
| 20 | function valuesForKey(source, key) { |
| 21 | const expression = new RegExp(`^\\s+${key}:\\s+([^#\\s]+)\\s*$`, "gm"); |
| 22 | return [...source.matchAll(expression)].map((match) => match[1]); |
| 23 | } |
| 24 | |
| 25 | function namedStep(source, name) { |
| 26 | const marker = ` - name: ${name}\n`; |
| 27 | const start = source.indexOf(marker); |
| 28 | assert.notEqual(start, -1, `missing workflow step: ${name}`); |
| 29 | const next = source.indexOf("\n - ", start + marker.length); |
| 30 | return source.slice(start, next === -1 ? source.length : next); |
| 31 | } |
| 32 | |
| 33 | const ci = read(".github/workflows/ci.yml"); |
| 34 | const nightly = read(".github/workflows/nightly.yml"); |
| 35 | const candidate = read(".github/workflows/release-candidate.yml"); |
| 36 | const artifacts = read(".github/workflows/release-artifacts.yml"); |
| 37 | const release = read(".github/workflows/release.yml"); |
| 38 | const republish = read(".github/workflows/release-republish.yml"); |
| 39 | const releaseDockerfile = read("packaging/docker/Dockerfile.release"); |
| 40 | const cnb = read(".cnb.yml"); |
| 41 | const bundles = read("scripts/release/create-release-bundles.sh"); |
| 42 | const archiveInstaller = read("scripts/release/install.sh"); |
| 43 | const cliDispatcher = read("crates/cli/src/lib.rs"); |
| 44 | const runbook = read("docs/RELEASE_RUNBOOK.md"); |
| 45 | |
| 46 | const ciTestJob = ci.slice(ci.indexOf("\n test:\n")); |
| 47 | assert.match( |
| 48 | ciTestJob, |
| 49 | /matrix\.os == 'ubuntu-latest'.*github\.event_name == 'pull_request'/, |
| 50 | "heavy pull requests must select the real Ubuntu test lane", |
| 51 | ); |
| 52 | assert.match( |
| 53 | ciTestJob, |
| 54 | /cargo nextest run --workspace --all-features --locked --profile ci/, |
| 55 | "the Ubuntu pull-request lane must run workspace nextest", |
| 56 | ); |
| 57 | assert.match( |
| 58 | ciTestJob, |
| 59 | /name: Linux test location \(CNB\)/, |
| 60 | "the non-PR CNB fallback must be named explicitly", |
| 61 | ); |
| 62 | |
| 63 | const npmSmokeJob = ci.match(/^ npm-wrapper-smoke:\n([\s\S]*?)(?=^ \S)/m)?.[1]; |
| 64 | assert.ok(npmSmokeJob, "CI must retain the required npm-wrapper job"); |
| 65 | assert.match( |
| 66 | namedStep(npmSmokeJob, "Build wrapper binaries"), |
| 67 | /run: cargo build --release --locked -p codewhale-cli -p codewhale-tui/, |
| 68 | ); |
| 69 | assert.match( |
| 70 | namedStep(npmSmokeJob, "Smoke wrapper install and delegated entrypoints"), |
| 71 | /run: node scripts\/release\/npm-wrapper-smoke\.js/, |
| 72 | ); |
| 73 | const npmSmokeSteps = npmSmokeJob.split(/(?=^ - )/m).slice(1); |
| 74 | // Exercise the workflow's actual Boolean guards. A successful location echo |
| 75 | // must never substitute for the build/install smoke on a heavy pull request. |
| 76 | const npmSmokeCases = [ |
| 77 | // name, event, heavy, OS, trusted, cache success, execute, Linux deps, CNB |
| 78 | ["own PR", "pull_request", true, "ubuntu-latest", true, true, true, true, false], |
| 79 | ["fork PR", "pull_request", true, "ubuntu-latest", false, true, true, true, false], |
| 80 | ["PR cache failure", "pull_request", true, "ubuntu-latest", false, false, true, true, false], |
| 81 | ["light PR", "pull_request", false, "ubuntu-latest", true, true, false, false, false], |
| 82 | ["manual Ubuntu", "workflow_dispatch", true, "ubuntu-latest", true, true, true, true, false], |
| 83 | ["main Ubuntu", "push", true, "ubuntu-latest", true, true, false, false, true], |
| 84 | ["main macOS", "push", true, "macos-latest", true, true, true, false, false], |
| 85 | ["main Windows", "push", true, "windows-latest", true, true, true, false, false], |
| 86 | ["light main", "push", false, "ubuntu-latest", true, true, false, false, false], |
| 87 | ["schedule", "schedule", true, "ubuntu-latest", true, true, false, false, false], |
| 88 | ]; |
| 89 | for (const [label, event, heavy, os, trusted, cache, execute, linuxDeps, cnb] of npmSmokeCases) { |
| 90 | const context = { |
| 91 | needs: { changes: { outputs: { heavy: String(heavy), trusted: String(trusted) } } }, |
| 92 | github: { event_name: event }, |
| 93 | matrix: { os }, |
| 94 | steps: { sccache: { outcome: cache ? "success" : "failure" } }, |
| 95 | }; |
| 96 | const jobGuard = npmSmokeJob.match(/^ if: (.+)$/m)?.[1]; |
| 97 | assert.ok(jobGuard, "the wrapper job must retain its event guard"); |
| 98 | const jobEnabled = vm.runInNewContext(jobGuard, context); |
| 99 | for (const step of npmSmokeSteps) { |
| 100 | const name = step.match(/^ - (?:name|uses): (.+)$/m)?.[1]; |
| 101 | const guard = step.match(/^ if: (.+)$/m)?.[1]; |
| 102 | assert.ok(name && guard, "every wrapper step must have an explicit guard"); |
| 103 | let expected = execute; |
| 104 | if (name === "Skip npm wrapper smoke for light change") expected = !heavy; |
| 105 | else if (name === "Install Linux system dependencies") expected = linuxDeps; |
| 106 | else if (name === "Linux smoke location") expected = cnb; |
| 107 | else if (name === "Enable sccache" || name === "sccache stats") expected = execute && cache; |
| 108 | assert.equal( |
| 109 | Boolean(jobEnabled && vm.runInNewContext(guard, context)), |
| 110 | expected, |
| 111 | `${label}: ${name} must ${expected ? "execute" : "stay skipped"}`, |
| 112 | ); |
| 113 | } |
| 114 | } |
| 115 | console.log(`Wrapper CI guards OK: ${npmSmokeCases.length} event cases, ${npmSmokeSteps.length} steps each.`); |
| 116 | |
| 117 | assert.match(ci, /^ workflow_dispatch:\n inputs:\n expected_sha:/m); |
| 118 | const manualForceBlock = ci.match( |
| 119 | /if \[\[ "\$\{EVENT_NAME\}" == "workflow_dispatch" \]\]; then([\s\S]*?)\n\s+if \[\[ "\$\{EVENT_NAME\}" == "schedule" \]\]; then/, |
| 120 | ); |
| 121 | assert.ok(manualForceBlock, "CI must have a dedicated manual-dispatch force-full branch"); |
| 122 | for (const output of ["heavy", "workflow", "mobile", "actions"]) { |
| 123 | assert.match(manualForceBlock[1], new RegExp(`echo "${output}=true"`)); |
| 124 | } |
| 125 | assert.match(manualForceBlock[1], /#EXPECTED_SHA.*-ne 40/s); |
| 126 | assert.match(manualForceBlock[1], /actual.*EXPECTED_SHA/s); |
| 127 | const expectedNightlyTargets = [ |
| 128 | "x86_64-unknown-linux-gnu", |
| 129 | "aarch64-unknown-linux-musl", |
| 130 | "x86_64-apple-darwin", |
| 131 | "aarch64-apple-darwin", |
| 132 | "x86_64-pc-windows-msvc", |
| 133 | "aarch64-pc-windows-msvc", |
| 134 | ].sort(); |
| 135 | assert.deepEqual([...new Set(valuesForKey(nightly, "target"))].sort(), expectedNightlyTargets); |
| 136 | assert.deepEqual( |
| 137 | [ |
| 138 | ...valuesForKey(nightly, "primary_artifact"), |
| 139 | ...valuesForKey(nightly, "alias_artifact"), |
| 140 | ].sort(), |
| 141 | [ |
| 142 | "codewhale-linux-x64", |
| 143 | "codew-linux-x64", |
| 144 | "codewhale-linux-arm64", |
| 145 | "codew-linux-arm64", |
| 146 | "codewhale-macos-x64", |
| 147 | "codew-macos-x64", |
| 148 | "codewhale-macos-arm64", |
| 149 | "codew-macos-arm64", |
| 150 | "codewhale-windows-x64.exe", |
| 151 | "codew-windows-x64.exe", |
| 152 | "codewhale-windows-arm64.exe", |
| 153 | "codew-windows-arm64.exe", |
| 154 | ].sort(), |
| 155 | ); |
| 156 | assert.match( |
| 157 | nightly, |
| 158 | /cargo build --release --locked --target \$\{\{ matrix\.target \}\} -p codewhale-cli/, |
| 159 | ); |
| 160 | assert.match(nightly, /startsWith\(matrix\.target, 'x86_64-'\).*runner\.arch == 'X64'/s); |
| 161 | assert.match(nightly, /startsWith\(matrix\.target, 'aarch64-'\).*runner\.arch == 'ARM64'/s); |
| 162 | const nightlyArmMuslSetup = namedStep(nightly, "Install Linux ARM64 musl toolchain"); |
| 163 | assert.match(nightlyArmMuslSetup, /matrix\.target == 'aarch64-unknown-linux-musl'/); |
| 164 | assert.match(nightlyArmMuslSetup, /apt-get install -y binutils musl-tools/); |
| 165 | assert.match(nightlyArmMuslSetup, /rustup target add --toolchain stable aarch64-unknown-linux-musl/); |
| 166 | const nightlyArmStaticSmoke = namedStep( |
| 167 | nightly, |
| 168 | "Verify static Linux ARM64 binary and launch", |
| 169 | ); |
| 170 | assert.match( |
| 171 | nightlyArmStaticSmoke, |
| 172 | /matrix\.target == 'aarch64-unknown-linux-musl' && runner\.arch == 'ARM64'/, |
| 173 | ); |
| 174 | assert.match(nightlyArmStaticSmoke, /readelf -l "\$\{bin_path\}"/); |
| 175 | assert.match(nightlyArmStaticSmoke, /grep -Fq 'INTERP'/); |
| 176 | assert.match(nightlyArmStaticSmoke, /"\$\{bin_path\}" --version/); |
| 177 | assert.doesNotMatch(nightly, /codewhale-tui/); |
| 178 | assert.doesNotMatch(nightly, /target\/[^\n]*\/codew(?:\.exe)?/); |
| 179 | assert.match(nightly, /cp "\$\{bin_path\}" "\$\{dir\}\/\$\{artifact\}"/); |
| 180 | assert.match(nightly, /cmp -s[\s\S]*nightly-primary[\s\S]*nightly-alias/); |
| 181 | assert.equal((nightly.match(/retention-days: 14/g) || []).length, 2); |
| 182 | |
| 183 | assert.match(candidate, /^ workflow_dispatch:\n inputs:\n expected_sha:/m); |
| 184 | assert.doesNotMatch(candidate, /^ (push|pull_request|schedule):/m); |
| 185 | assert.match(candidate, /uses: \.\/\.github\/workflows\/release-artifacts\.yml/); |
| 186 | assert.match(candidate, /source_sha: \$\{\{ needs\.resolve\.outputs\.sha \}\}/); |
| 187 | assert.match(candidate, /^ web:\n/m); |
| 188 | assert.doesNotMatch( |
| 189 | candidate, |
| 190 | /ref: \$\{\{ needs\.resolve\.outputs\.sha \}\}/, |
| 191 | "candidate jobs must checkout GITHUB_SHA, not interpolate the dispatch SHA into ref", |
| 192 | ); |
| 193 | assert.match(candidate, /cache-dependency-path: web\/package-lock\.json/); |
| 194 | assert.match(candidate, /package-manager-cache: false/); |
| 195 | assert.match(candidate, /working-directory: web/); |
| 196 | for (const workflow of [candidate, read(".github/workflows/web.yml")]) { |
| 197 | for (const command of ["npm ci", "npm test", "npm run check"]) { |
| 198 | assert.ok(workflow.includes(`run: ${command}\n`), `missing web gate: ${command}`); |
| 199 | } |
| 200 | } |
| 201 | // Both workflows use this gate. Preserve every check and its order: checking |
| 202 | // committed facts after prebuild could silently repair drift before testing it. |
| 203 | assert.deepEqual(JSON.parse(read("web/package.json")).scripts.check.split(" && "), [ |
| 204 | "npm run check:facts", |
| 205 | "npm run check:latest-release", |
| 206 | "npm run prebuild", |
| 207 | "npm run check:docs", |
| 208 | "npm run check:tokens", |
| 209 | "npm run lint", |
| 210 | "tsc --noEmit", |
| 211 | "npm run build", |
| 212 | ]); |
| 213 | assert.match(candidate, /^ needs: \[resolve, web\]$/m); |
| 214 | assert.match(candidate, /needs\.web\.result == 'success'/); |
| 215 | |
| 216 | for (const [label, workflow] of [ |
| 217 | ["release candidate", candidate], |
| 218 | ["shared artifact", artifacts], |
| 219 | ]) { |
| 220 | for (const forbidden of [ |
| 221 | /contents:\s*write/, |
| 222 | /packages:\s*write/, |
| 223 | /softprops\/action-gh-release/, |
| 224 | /docker\/login-action/, |
| 225 | /docker\/build-push-action/, |
| 226 | /\bgh release\b/, |
| 227 | /\bnpm publish\b/, |
| 228 | /\bcargo publish\b/, |
| 229 | /\bgit push\b/, |
| 230 | ]) { |
| 231 | assert.doesNotMatch(workflow, forbidden, `${label} workflow contains publication capability`); |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | for (const [label, workflow] of [ |
| 236 | ["release candidate", candidate], |
| 237 | ["shared artifact", artifacts], |
| 238 | ["public release", release], |
| 239 | ["release republish", republish], |
| 240 | ]) { |
| 241 | const remoteActions = [...workflow.matchAll(/^\s+(?:-\s+)?uses:\s+([^@\s]+)@([^#\s]+)/gm)] |
| 242 | .map((match) => ({ action: match[1], ref: match[2] })) |
| 243 | .filter(({ action }) => !action.startsWith("./")); |
| 244 | assert.ok(remoteActions.length > 0, `${label} workflow must exercise pinned actions`); |
| 245 | for (const { action, ref } of remoteActions) { |
| 246 | assert.match( |
| 247 | ref, |
| 248 | /^[0-9a-f]{40}$/, |
| 249 | `${label} action ${action} must use an audited full commit SHA`, |
| 250 | ); |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | const republishHomebrewJob = republish.match(/\n homebrew:\n([\s\S]*)$/); |
| 255 | assert.ok(republishHomebrewJob, "republish must retain a Homebrew recovery job"); |
| 256 | const republishHomebrewCheckout = namedStep( |
| 257 | republishHomebrewJob[0], |
| 258 | "Checkout release infrastructure", |
| 259 | ); |
| 260 | assert.match( |
| 261 | republishHomebrewCheckout, |
| 262 | /ref: \$\{\{ github\.event\.repository\.default_branch \}\}/, |
| 263 | "Homebrew recovery must use the repaired default-branch infrastructure", |
| 264 | ); |
| 265 | assert.doesNotMatch( |
| 266 | republishHomebrewCheckout, |
| 267 | /needs\.resolve\.outputs\.sha/, |
| 268 | "Homebrew recovery must not resurrect release-tag infrastructure", |
| 269 | ); |
| 270 | assert.match(republishHomebrewJob[0], /gh release download "\$\{\{ needs\.resolve\.outputs\.tag \}\}"/); |
| 271 | assert.match(republishHomebrewJob[0], /MANIFEST: \/tmp\/codewhale-artifacts-sha256\.txt/); |
| 272 | |
| 273 | assert.match(artifacts, /^ workflow_call:/m); |
| 274 | assert.match(artifacts, /^permissions:\n contents: read$/m); |
| 275 | const expectedTargets = [ |
| 276 | "x86_64-unknown-linux-musl", |
| 277 | "aarch64-unknown-linux-musl", |
| 278 | "aarch64-linux-android", |
| 279 | "x86_64-apple-darwin", |
| 280 | "aarch64-apple-darwin", |
| 281 | "x86_64-pc-windows-msvc", |
| 282 | "aarch64-pc-windows-msvc", |
| 283 | ].sort(); |
| 284 | assert.deepEqual([...new Set(valuesForKey(artifacts, "target"))].sort(), expectedTargets); |
| 285 | |
| 286 | const releaseMuslBuild = namedStep(artifacts, "Build static Linux binaries (musl)"); |
| 287 | assert.match(releaseMuslBuild, /endsWith\(matrix\.target, '-unknown-linux-musl'\)/); |
| 288 | assert.match(releaseMuslBuild, /apt-get install -y binutils musl-tools/); |
| 289 | assert.match(releaseMuslBuild, /rustup target add --toolchain stable \$\{\{ matrix\.target \}\}/); |
| 290 | assert.match( |
| 291 | releaseMuslBuild, |
| 292 | /cargo build --profile dist --locked --target \$\{\{ matrix\.target \}\} -p codewhale-cli/, |
| 293 | ); |
| 294 | const releaseStaticSmoke = namedStep( |
| 295 | artifacts, |
| 296 | "Verify static Linux binaries and launch on matching native runners", |
| 297 | ); |
| 298 | assert.match(releaseStaticSmoke, /endsWith\(matrix\.target, '-unknown-linux-musl'\)/); |
| 299 | assert.match( |
| 300 | releaseStaticSmoke, |
| 301 | /startsWith\(matrix\.target, 'aarch64-'\) && runner\.arch == 'ARM64'/, |
| 302 | ); |
| 303 | assert.match(releaseStaticSmoke, /readelf -l "\$\{bin_path\}"/); |
| 304 | assert.match(releaseStaticSmoke, /grep -Fq 'INTERP'/); |
| 305 | assert.match(releaseStaticSmoke, /"\$\{bin_path\}" --version/); |
| 306 | |
| 307 | const builtAssetNames = [ |
| 308 | ...valuesForKey(artifacts, "cli_artifact"), |
| 309 | ...valuesForKey(artifacts, "shim_artifact"), |
| 310 | ...valuesForKey(artifacts, "compat_tui_artifact"), |
| 311 | ]; |
| 312 | assert.equal(builtAssetNames.length, 21); |
| 313 | assert.deepEqual( |
| 314 | [...new Set(builtAssetNames)].sort(), |
| 315 | [ |
| 316 | ...allAssetNames().filter((name) => name !== "codewhale.bat"), |
| 317 | ...LEGACY_TUI_BRIDGE_ASSET_NAMES, |
| 318 | ].sort(), |
| 319 | ); |
| 320 | assert.match( |
| 321 | artifacts, |
| 322 | /stage_binary "\$\{\{ matrix\.cli_binary \}\}" "\$\{\{ matrix\.compat_tui_artifact \}\}"/, |
| 323 | "legacy TUI bridge assets must be staged from the one compiled codewhale binary", |
| 324 | ); |
| 325 | const bundleInvocations = [...bundles.matchAll( |
| 326 | /^bundle (\S+) \\\n\s+\S+ \S+ (tar\.gz|zip) (""|portable)$/gm, |
| 327 | )].map((match) => { |
| 328 | const variant = match[3] === "portable" ? "-portable" : ""; |
| 329 | return `codewhale-${match[1]}${variant}.${match[2]}`; |
| 330 | }); |
| 331 | assert.deepEqual(bundleInvocations.sort(), [...BUNDLE_ASSET_NAMES].sort()); |
| 332 | assert.match(artifacts, /aarch64-pc-windows-msvc/); |
| 333 | assert.match(artifacts, /aarch64-linux-android/); |
| 334 | assert.match(artifacts, /codew-windows-arm64\.exe/); |
| 335 | assert.match(artifacts, /CodeWhaleSetup\.exe/); |
| 336 | assert.match(artifacts, /assemble-release-assets\.js --verify release-assets/); |
| 337 | assert.match(artifacts, /CODEWHALE_SMOKE_ASSETS_DIR/); |
| 338 | assert.match(artifacts, /^ pin:\n/m); |
| 339 | assert.match(artifacts, /Require source_sha equals github\.sha/); |
| 340 | assert.doesNotMatch( |
| 341 | artifacts, |
| 342 | /ref: \$\{\{ inputs\.source_sha \}\}/, |
| 343 | "artifact jobs must checkout GITHUB_SHA, not interpolate the caller SHA into ref", |
| 344 | ); |
| 345 | assert.match(artifacts, /prefix-key: v1-\$\{\{ runner\.os \}\}-\$\{\{ runner\.arch \}\}-stable/); |
| 346 | assert.equal( |
| 347 | (artifacts.match(/package-manager-cache: false/g) || []).length, |
| 348 | 2, |
| 349 | "assemble and smoke must disable setup-node's implicit npm cache", |
| 350 | ); |
| 351 | const bundleStep = namedStep(artifacts, "Create and checksum platform archives"); |
| 352 | assert.match(bundleStep, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); |
| 353 | assert.match(bundleStep, /git show -s --format=%ct "\$\{SOURCE_SHA\}"/); |
| 354 | assert.match( |
| 355 | bundleStep, |
| 356 | /SOURCE_DATE_EPOCH="\$\{source_date_epoch\}"[\s\\]+bash scripts\/release\/create-release-bundles\.sh artifacts bundles/, |
| 357 | ); |
| 358 | assert.doesNotMatch(bundleStep, /inputs\.source_sha/); |
| 359 | assert.doesNotMatch(bundleStep, /\bdate\b/, "bundle timestamps must come from the pinned source commit, not wall-clock time"); |
| 360 | |
| 361 | const rustCacheBlocks = [...artifacts.matchAll(/uses: Swatinem\/rust-cache@[\s\S]*?(?=\n - )/g)].map( |
| 362 | (match) => match[0], |
| 363 | ); |
| 364 | assert.ok(rustCacheBlocks.length >= 1, "shared artifact workflow must pin rust-cache"); |
| 365 | for (const block of rustCacheBlocks) { |
| 366 | assert.doesNotMatch(block, /github\.(event|ref|sha)|inputs\./); |
| 367 | } |
| 368 | |
| 369 | const parity = release.match(/\n parity:\n([\s\S]*?)\n artifacts:\n/); |
| 370 | assert.ok(parity, "public release must retain a parity job"); |
| 371 | assert.doesNotMatch( |
| 372 | parity[1], |
| 373 | /ref: \$\{\{ needs\.resolve\.outputs\.sha \}\}/, |
| 374 | "parity must checkout GITHUB_SHA after resolve, not interpolate the tag SHA into ref", |
| 375 | ); |
| 376 | assert.match(parity[1], /prefix-key: v1-\$\{\{ runner\.os \}\}-\$\{\{ runner\.arch \}\}-stable/); |
| 377 | const parityRustCache = [...parity[1].matchAll(/uses: Swatinem\/rust-cache@[\s\S]*?(?=\n - )/g)].map( |
| 378 | (match) => match[0], |
| 379 | ); |
| 380 | assert.equal(parityRustCache.length, 1, "parity must pin exactly one rust-cache"); |
| 381 | assert.doesNotMatch(parityRustCache[0], /github\.(event|ref|sha)|inputs\./); |
| 382 | |
| 383 | assert.equal(allReleaseAssetNames().length, 34); |
| 384 | assert.match(release, /^ artifacts:\n/m); |
| 385 | assert.match(release, /uses: \.\/\.github\/workflows\/release-artifacts\.yml/); |
| 386 | assert.doesNotMatch(release, /^ (build|bundle|windows-installer):/m); |
| 387 | assert.match(release, /name: codewhale-release-assets\n\s+path: artifacts/); |
| 388 | assert.match(release, /files: artifacts\/\*/); |
| 389 | assert.equal( |
| 390 | (release.match(/ensure-release-assets-absent\.js/g) || []).length, |
| 391 | 2, |
| 392 | "public release must refuse existing assets before work and immediately before upload", |
| 393 | ); |
| 394 | assert.match(release, /overwrite_files:\s*false/); |
| 395 | assert.match(release, /fail_on_unmatched_files:\s*true/); |
| 396 | |
| 397 | assert.match(release, /^ docker-build:\n/m); |
| 398 | assert.match(release, /^ docker:\n/m); |
| 399 | assert.match(release, /runner: ubuntu-latest\n\s+platform: linux\/amd64/); |
| 400 | assert.match(release, /runner: ubuntu-24\.04-arm\n\s+platform: linux\/arm64/); |
| 401 | assert.match(release, /cli_artifact: codewhale-linux-x64/); |
| 402 | assert.match(release, /cli_artifact: codewhale-linux-arm64/); |
| 403 | assert.match(release, /shim_artifact: codew-linux-x64/); |
| 404 | assert.match(release, /shim_artifact: codew-linux-arm64/); |
| 405 | assert.doesNotMatch( |
| 406 | release, |
| 407 | /docker\/setup-qemu-action/, |
| 408 | "public container publication must not funnel both architectures through QEMU", |
| 409 | ); |
| 410 | const releaseDockerBytes = namedStep(release, "Verify native release bytes"); |
| 411 | assert.match(releaseDockerBytes, /CLI_ARTIFACT: \$\{\{ matrix\.cli_artifact \}\}/); |
| 412 | assert.match(releaseDockerBytes, /SHIM_ARTIFACT: \$\{\{ matrix\.shim_artifact \}\}/); |
| 413 | assert.match( |
| 414 | releaseDockerBytes, |
| 415 | /mv -- "docker-context\/bin\/\$\{CLI_ARTIFACT\}" docker-context\/bin\/codewhale/, |
| 416 | ); |
| 417 | assert.match( |
| 418 | releaseDockerBytes, |
| 419 | /mv -- "docker-context\/bin\/\$\{SHIM_ARTIFACT\}" docker-context\/bin\/codew/, |
| 420 | ); |
| 421 | assert.match(releaseDockerBytes, /cmp docker-context\/bin\/codewhale docker-context\/bin\/codew/); |
| 422 | const releaseDockerBuild = namedStep(release, "Assemble and push native image by digest"); |
| 423 | assert.match(releaseDockerBuild, /context: docker-context/); |
| 424 | assert.match(releaseDockerBuild, /file: infra\/packaging\/docker\/Dockerfile\.release/); |
| 425 | assert.match(releaseDockerBuild, /platforms: \$\{\{ matrix\.platform \}\}/); |
| 426 | assert.match(releaseDockerBuild, /provenance: mode=max/); |
| 427 | assert.match(releaseDockerBuild, /sbom: true/); |
| 428 | assert.match(releaseDockerBuild, /push-by-digest=true/); |
| 429 | const releaseDockerManifest = namedStep(release, "Publish multi-architecture manifest"); |
| 430 | assert.match(releaseDockerManifest, /Expected exactly two native image digests/); |
| 431 | assert.match(releaseDockerManifest, /docker buildx imagetools create/); |
| 432 | const releaseDockerSmoke = namedStep(release, "Verify and smoke published container"); |
| 433 | assert.match(releaseDockerSmoke, /linux\/amd64/); |
| 434 | assert.match(releaseDockerSmoke, /linux\/arm64/); |
| 435 | assert.match(releaseDockerSmoke, /--entrypoint codewhale/); |
| 436 | assert.match(releaseDockerSmoke, /--entrypoint codew/); |
| 437 | |
| 438 | const npmJob = release.match(/\n npm:\n([\s\S]*?)\n homebrew:\n/); |
| 439 | assert.ok(npmJob, "public release must retain a dedicated npm publication job"); |
| 440 | assert.match(npmJob[1], /^ needs: \[release, resolve\]$/m); |
| 441 | assert.match(npmJob[1], /needs\.release\.result == 'success'/); |
| 442 | assert.match(npmJob[1], /^ contents: read$/m); |
| 443 | assert.match(npmJob[1], /^ id-token: write$/m); |
| 444 | assert.match(npmJob[1], /ref: \$\{\{ needs\.resolve\.outputs\.sha \}\}/); |
| 445 | assert.match(npmJob[1], /fetch-depth: 0/); |
| 446 | assert.match(npmJob[1], /node-version: 24/); |
| 447 | assert.match(npmJob[1], /registry-url: https:\/\/registry\.npmjs\.org/); |
| 448 | assert.match(npmJob[1], /package-manager-cache: false/); |
| 449 | assert.match(npmJob[1], /npm install --global npm@12\.0\.2/); |
| 450 | const npmTagGate = namedStep(release, "Revalidate release tag before npm publish"); |
| 451 | const npmAssetGate = namedStep(release, "Revalidate public release assets"); |
| 452 | const npmPublish = namedStep(release, "Publish npm wrapper with trusted publishing"); |
| 453 | assert.match(npmTagGate, /verify-remote-tag\.sh/); |
| 454 | assert.match(npmAssetGate, /verify-release-assets\.sh/); |
| 455 | assert.match(npmAssetGate, /GH_TOKEN: \$\{\{ github\.token \}\}/); |
| 456 | assert.match(npmPublish, /working-directory: npm\/codewhale/); |
| 457 | assert.match(npmPublish, /GH_TOKEN: \$\{\{ github\.token \}\}/); |
| 458 | assert.match(npmPublish, /npm publish --access public/); |
| 459 | assert.doesNotMatch(npmJob[1], /NPM_TOKEN|NODE_AUTH_TOKEN|secrets\./); |
| 460 | assert.ok( |
| 461 | release.indexOf("Revalidate public release assets") < |
| 462 | release.indexOf("Publish npm wrapper with trusted publishing"), |
| 463 | "npm publication must follow the public exact-asset gate", |
| 464 | ); |
| 465 | |
| 466 | assert.match(releaseDockerfile, /^FROM debian:bookworm-slim$/m); |
| 467 | assert.match(releaseDockerfile, /ca-certificates/); |
| 468 | assert.match(releaseDockerfile, /libdbus-1-3/); |
| 469 | assert.match(releaseDockerfile, /COPY .*bin\/codewhale \/usr\/local\/bin\/codewhale/); |
| 470 | assert.match(releaseDockerfile, /COPY .*bin\/codew \/usr\/local\/bin\/codew/); |
| 471 | assert.match(releaseDockerfile, /^USER codewhale$/m); |
| 472 | assert.doesNotMatch( |
| 473 | releaseDockerfile, |
| 474 | /\bcargo\s+build\b|^FROM\s+rust:/m, |
| 475 | "release container assembly must reuse the already-verified release binaries", |
| 476 | ); |
| 477 | |
| 478 | assert.match(runbook, /release[- ]candidate/i); |
| 479 | assert.match(runbook, /expected_sha/); |
| 480 | assert.match(runbook, /34/); |
| 481 | assert.match(runbook, /does not create a tag/i); |
| 482 | assert.match(runbook, /explicit.*approval/i); |
| 483 | assert.match(runbook, /last[- ]useful[- ]log/i, "runbook must document the last-useful-log rule (#5496)"); |
| 484 | assert.match(runbook, /404 logs/i, "runbook must document the 404-log cancellation rule (#5496)"); |
| 485 | |
| 486 | const cnbRustGates = cnb.match( |
| 487 | /\.rust_workspace_gates_stage: &rust_workspace_gates_stage([\s\S]*?)\n\.linux_rust_gates:/, |
| 488 | ); |
| 489 | assert.ok(cnbRustGates, "CNB must retain the shared Rust workspace gate"); |
| 490 | assert.match( |
| 491 | cnbRustGates[1], |
| 492 | /timeout: 45m[\s\S]*export CARGO_BUILD_JOBS=1[\s\S]*export CARGO_PROFILE_TEST_DEBUG=0[\s\S]*cargo check --workspace --all-targets --locked[\s\S]*cargo clippy --workspace --all-targets --all-features --locked -- -D warnings[\s\S]*RUST_MIN_STACK=16777216 sh scripts\/with-hermetic-test-home.sh cargo test --workspace --all-features --locked/, |
| 493 | "CNB must serialize the memory-heavy Rust gate and preserve the workspace test stack contract", |
| 494 | ); |
| 495 | assert.doesNotMatch( |
| 496 | cnbRustGates[1], |
| 497 | /export (?:HOME|USERPROFILE|CODEWHALE_HOME)=/, |
| 498 | "CNB must reuse the shared test-home boundary without overriding legacy migration fixtures", |
| 499 | ); |
| 500 | |
| 501 | // Cover every test invocation, including named parity and narrow crate gates. |
| 502 | // These launchers protect production dependencies as well as cfg(test) code. |
| 503 | let hermeticInvocations = 0; |
| 504 | for (const [label, workflow, expected] of [["CI", ci, 5], ["release", release, 3], ["CNB", cnb, 3]]) { |
| 505 | const commands = workflow.split("\n").filter((line) => |
| 506 | !line.trimStart().startsWith("#") && /\bcargo (?:test|nextest run)\b/.test(line), |
| 507 | ); |
| 508 | assert.equal(commands.length, expected, `${label} must retain every Rust test invocation`); |
| 509 | for (const command of commands) { |
| 510 | assert.match(command, /sh scripts\/with-hermetic-test-home.sh cargo (?:test|nextest run)\b/, |
| 511 | `${label} Rust tests must use the shared test-home boundary`); |
| 512 | } |
| 513 | hermeticInvocations += commands.length; |
| 514 | } |
| 515 | for (const name of ["Run tests", "Run doctests"]) { |
| 516 | const step = namedStep(ciTestJob, name); |
| 517 | assert.match(step, /shell: bash/, `${name} must invoke the POSIX helper on Windows too`); |
| 518 | assert.match(step, /RUST_MIN_STACK: '16777216'/); |
| 519 | } |
| 520 | console.log(`Hermetic Rust workflow invocations OK: ${hermeticInvocations} checks passed.`); |
| 521 | |
| 522 | const nextest = read(".config/nextest.toml"); |
| 523 | const integrationGroup = nextest.search(/^filter = 'binary\(integration\)'$/m); |
| 524 | const telemetryGroup = nextest.indexOf( |
| 525 | "filter = 'binary(integration) & test(/^telemetry_contract::/)'", |
| 526 | ); |
| 527 | const execGroup = nextest.indexOf( |
| 528 | "filter = 'binary(integration) & test(/^exec_persistent_service::/)'", |
| 529 | ); |
| 530 | assert.ok(integrationGroup >= 0, "nextest must bound the integration binary"); |
| 531 | assert.ok( |
| 532 | telemetryGroup >= 0 && telemetryGroup < integrationGroup, |
| 533 | "telemetry-contract override must precede binary(integration); first matching group wins", |
| 534 | ); |
| 535 | assert.ok( |
| 536 | execGroup >= 0 && execGroup < integrationGroup, |
| 537 | "exec_persistent_service override must precede binary(integration); first matching group wins", |
| 538 | ); |
| 539 | assert.match(nextest, /exec-persistent-service = \{ max-threads = 1 \}/); |
| 540 | assert.equal( |
| 541 | (cnb.match(/^\s+- \*rust_workspace_gates_stage$/gm) || []).length, |
| 542 | 2, |
| 543 | "both CNB Rust pipelines must reuse the constrained workspace gate", |
| 544 | ); |
| 545 | |
| 546 | const cnbPreflight = cnb.match( |
| 547 | /\.linux_release_preflight: &linux_release_preflight([\s\S]*?)\nmain:/, |
| 548 | ); |
| 549 | assert.ok(cnbPreflight, "CNB must retain a dedicated release preflight"); |
| 550 | const cnbBuild = cnbPreflight[1].indexOf( |
| 551 | "cargo build --jobs 2 --release --locked -p codewhale-cli", |
| 552 | ); |
| 553 | const cnbAlias = cnbPreflight[1].indexOf( |
| 554 | "cp target/release/codewhale target/release/codew", |
| 555 | ); |
| 556 | const cnbSmoke = cnbPreflight[1].indexOf("node scripts/release/npm-wrapper-smoke.js"); |
| 557 | assert.ok(cnbBuild >= 0, "CNB release preflight must build the consolidated runtime"); |
| 558 | assert.ok(cnbAlias > cnbBuild, "CNB release preflight must materialize codew after the build"); |
| 559 | assert.ok(cnbSmoke > cnbAlias, "CNB release preflight must materialize codew before smoke"); |
| 560 | |
| 561 | const cnbTagRelease = cnb.match(/\$:\n tag_push:\n([\s\S]*)$/); |
| 562 | assert.ok(cnbTagRelease, "CNB must retain a tag release pipeline"); |
| 563 | const cnbTagStamp = cnbTagRelease[1].indexOf( |
| 564 | 'export CODEWHALE_BUILD_SHA="$commit_sha"', |
| 565 | ); |
| 566 | const cnbTagBuild = cnbTagRelease[1].indexOf( |
| 567 | "cargo build --jobs 2 --release --locked \\", |
| 568 | ); |
| 569 | const cnbTagVersionCheck = cnbTagRelease[1].indexOf( |
| 570 | "./scripts/release/check-versions.sh --require-dated-release", |
| 571 | ); |
| 572 | assert.ok(cnbTagVersionCheck >= 0, "CNB publication must reject undated source candidates"); |
| 573 | assert.ok(cnbTagVersionCheck < cnbTagBuild, "CNB must validate release notes before building public assets"); |
| 574 | assert.match(cnbTagRelease[1], /checkout_sha="\$\(git rev-parse 'HEAD\^\{commit\}'\)"/); |
| 575 | assert.match(cnbTagRelease[1], /commit_sha="\$\{CNB_COMMIT:-\$\{checkout_sha\}\}"/); |
| 576 | assert.match(cnbTagRelease[1], /CNB_COMMIT[\s\S]*does not match checkout[\s\S]*exit 1/); |
| 577 | assert.ok(cnbTagStamp >= 0, "CNB tag releases must stamp the consolidated runtime"); |
| 578 | assert.ok(cnbTagBuild > cnbTagStamp, "CNB tag releases must stamp before compiling"); |
| 579 | |
| 580 | assert.doesNotMatch( |
| 581 | archiveInstaller, |
| 582 | /cargo install codewhale --locked/, |
| 583 | "glibc recovery must name the published codewhale-cli crate", |
| 584 | ); |
| 585 | assert.equal( |
| 586 | (archiveInstaller.match(/cargo install codewhale-cli --locked/g) || []).length, |
| 587 | 2, |
| 588 | "both glibc recovery branches must name codewhale-cli", |
| 589 | ); |
| 590 | // The archive installer never overwrites an existing command: it validates the |
| 591 | // retired TUI path against the consolidated bytes and leaves upgrades to |
| 592 | // `codewhale update`, which migrates `codewhale-tui` beside the canonical pair. |
| 593 | assert.match( |
| 594 | archiveInstaller, |
| 595 | /legacy_tui="\$BIN_DIR\/codewhale-tui"[\s\S]*check_destination "\$SCRIPT_DIR\/codewhale" "\$legacy_tui"/, |
| 596 | "archive installs must validate the retired TUI path against consolidated bytes", |
| 597 | ); |
| 598 | assert.doesNotMatch( |
| 599 | archiveInstaller, |
| 600 | /install_binary "\$SCRIPT_DIR\/codewhale" "\$legacy_tui"/, |
| 601 | "archive installs must not overwrite an existing retired TUI command", |
| 602 | ); |
| 603 | assert.doesNotMatch( |
| 604 | cliDispatcher, |
| 605 | /codewhale_config::auto_model::classify/, |
| 606 | "the CLI dispatcher must leave auto routing to the provider-aware runtime", |
| 607 | ); |
| 608 | |
| 609 | // #5496: every release-lane job carries an explicit `timeout-minutes`. |
| 610 | // |
| 611 | // GitHub's default is 360 minutes, so an assigned-but-dead runner sits for six |
| 612 | // hours before anything reclaims it — observed on the v0.9.9 train as a job |
| 613 | // stuck `in_progress` with 404 logs. Timeouts are containment, not recovery: |
| 614 | // the runbook keeps the 404-log cancel/rerun rule for infrastructure failures. |
| 615 | // |
| 616 | // A job that calls a reusable workflow (`uses:`) cannot carry the key at all — |
| 617 | // GitHub rejects it — so the callee owns its own caps. That is why the artifact |
| 618 | // bounds live in release-artifacts.yml rather than in its callers. |
| 619 | function jobsWithoutTimeout(source) { |
| 620 | const lines = source.split("\n"); |
| 621 | const jobsAt = lines.findIndex((line) => /^jobs:\s*$/.test(line)); |
| 622 | assert.notEqual(jobsAt, -1, "workflow must declare jobs"); |
| 623 | const offenders = []; |
| 624 | for (let i = jobsAt + 1; i < lines.length; i += 1) { |
| 625 | const header = lines[i].match(/^ ([A-Za-z0-9_-]+):\s*$/); |
| 626 | if (!header) continue; |
| 627 | let reusable = false; |
| 628 | let capped = false; |
| 629 | for (let j = i + 1; j < lines.length; j += 1) { |
| 630 | if (/^ [A-Za-z0-9_-]+:\s*$/.test(lines[j])) break; |
| 631 | if (/^ uses:/.test(lines[j])) reusable = true; |
| 632 | if (/^ timeout-minutes:\s*\d+\s*$/.test(lines[j])) capped = true; |
| 633 | } |
| 634 | if (!reusable && !capped) offenders.push(header[1]); |
| 635 | } |
| 636 | return offenders; |
| 637 | } |
| 638 | |
| 639 | assert.deepEqual( |
| 640 | jobsWithoutTimeout("jobs:\n uncapped:\n runs-on: ubuntu-latest\n"), |
| 641 | ["uncapped"], |
| 642 | "jobsWithoutTimeout must detect an uncapped job", |
| 643 | ); |
| 644 | assert.deepEqual( |
| 645 | jobsWithoutTimeout("jobs:\n reusable:\n uses: ./.github/workflows/reusable.yml\n"), |
| 646 | [], |
| 647 | "jobsWithoutTimeout must skip reusable workflow callers", |
| 648 | ); |
| 649 | assert.deepEqual( |
| 650 | jobsWithoutTimeout("jobs:\n capped:\n runs-on: ubuntu-latest\n timeout-minutes: 15\n"), |
| 651 | [], |
| 652 | "jobsWithoutTimeout must accept a capped job", |
| 653 | ); |
| 654 | |
| 655 | for (const [name, source] of [ |
| 656 | ["release-candidate.yml", candidate], |
| 657 | ["release-artifacts.yml", artifacts], |
| 658 | ["release.yml", release], |
| 659 | ["release-republish.yml", republish], |
| 660 | ["ci.yml", ci], |
| 661 | ["nightly.yml", nightly], |
| 662 | ]) { |
| 663 | assert.deepEqual( |
| 664 | jobsWithoutTimeout(source), |
| 665 | [], |
| 666 | `${name}: every job must set timeout-minutes (#5496)`, |
| 667 | ); |
| 668 | } |
| 669 | |
| 670 | // The Windows artifact build historically runs 40-45 minutes, so its cap has to |
| 671 | // keep real margin — a tight bound here fails healthy releases. |
| 672 | const buildTimeout = artifacts.match(/^ build:\n(?:.*\n)*? timeout-minutes: (\d+)$/m); |
| 673 | assert.ok(buildTimeout, "release-artifacts build job must be capped"); |
| 674 | assert.ok( |
| 675 | Number(buildTimeout[1]) >= 60, |
| 676 | `artifact build cap ${buildTimeout[1]}m leaves no margin over a healthy 40-45m Windows build`, |
| 677 | ); |
| 678 | |
| 679 | function jobTimeout(source, job) { |
| 680 | const match = source.match( |
| 681 | new RegExp(`^ ${job}:\\n(?:.*\\n)*? timeout-minutes: (\\d+)$`, "m"), |
| 682 | ); |
| 683 | assert.ok(match, `${job} must declare timeout-minutes`); |
| 684 | return Number(match[1]); |
| 685 | } |
| 686 | |
| 687 | // Pin the measured release-lane budget: fast setup and packaging fail quickly, |
| 688 | // while cross-platform compilation keeps real margin over the 40-45m Windows |
| 689 | // build observed on the release train. |
| 690 | assert.equal(jobTimeout(candidate, "resolve"), 10); |
| 691 | assert.equal(jobTimeout(candidate, "web"), 15); |
| 692 | assert.equal(jobTimeout(artifacts, "pin"), 10); |
| 693 | assert.equal(jobTimeout(artifacts, "build"), 90); |
| 694 | for (const job of ["bundle", "windows-installer", "assemble", "smoke"]) { |
| 695 | assert.equal(jobTimeout(artifacts, job), 15, `${job} must keep the 15m packaging cap`); |
| 696 | } |
| 697 | assert.equal(jobTimeout(nightly, "build"), 90); |
| 698 | assert.equal(jobTimeout(release, "resolve"), 10); |
| 699 | // The v0.9.12 tag push finished every parity step and was then cancelled at |
| 700 | // 20 minutes inside rust-cache's post-run save; 45 keeps that margin. |
| 701 | assert.equal(jobTimeout(release, "parity"), 45); |
| 702 | |
| 703 | console.log( |
| 704 | "Workflow contracts OK: 6-target/12-asset single-runtime nightly and exact-head 7-target/34-asset release candidate.", |
| 705 | ); |
| 706 |