| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "fmt" |
| 6 | "os" |
| 7 | "os/exec" |
| 8 | "path/filepath" |
| 9 | "sort" |
| 10 | "strings" |
| 11 | "time" |
| 12 | "unicode/utf8" |
| 13 | |
| 14 | "reasonix/internal/ablation" |
| 15 | "reasonix/internal/shellparse" |
| 16 | ) |
| 17 | |
| 18 | type diffOpts struct { |
| 19 | bin, model, repo, base, testCmd string |
| 20 | ablate ablation.Set |
| 21 | maxSteps, timeoutSec, attempts int |
| 22 | } |
| 23 | |
| 24 | type testRef struct{ name, pkg string } |
| 25 | |
| 26 | // pinResult records whether one generated test fails when the PR's source is |
| 27 | // reverted (so it pins the change) and, if so, whether it failed by assertion |
| 28 | // (strong: it checks the new behavior) or only by compile error (weak: it just |
| 29 | // references a symbol the PR added). |
| 30 | type pinResult struct { |
| 31 | testRef |
| 32 | pins bool |
| 33 | byAssertion bool |
| 34 | } |
| 35 | |
| 36 | // runDiff asks the agent to write tests covering what the PR changed, grades |
| 37 | // them against the repo's own tests, and — because the agent is stochastic — |
| 38 | // retries up to o.attempts times until a run passes, keeping the best result. |
| 39 | func runDiff(o diffOpts) string { |
| 40 | srcFiles := changedGoFiles(o.repo, o.base, false) |
| 41 | if len(srcFiles) == 0 { |
| 42 | return "## 🤖 Reasonix e2e — diff test-gen\n\nNo Go source changes in this PR (excluding `_test.go`); nothing to generate tests for.\n" |
| 43 | } |
| 44 | pkgs := packagesOf(srcFiles) |
| 45 | prompt := buildDiffPrompt(srcFiles, pkgs, truncate(gitOut(o.repo, "diff", o.base+"...HEAD", "--"))) |
| 46 | |
| 47 | attempts := max(o.attempts, 1) |
| 48 | var best diffReport |
| 49 | made := 0 |
| 50 | for i := 1; i <= attempts; i++ { |
| 51 | if i > 1 { |
| 52 | resetTree(o.repo) |
| 53 | } |
| 54 | r := runOnce(o, srcFiles, pkgs, prompt) |
| 55 | made = i |
| 56 | if i == 1 || better(r, best) { |
| 57 | best = r |
| 58 | } |
| 59 | if best.passed { |
| 60 | break // stop at the first passing run; attempts is a retry budget |
| 61 | } |
| 62 | } |
| 63 | best.attempt, best.attempts = made, attempts |
| 64 | return renderDiff(best) |
| 65 | } |
| 66 | |
| 67 | // runOnce does one agent run + grade: generate tests, check they pass on HEAD, |
| 68 | // differential-check each against the reverted source, measure changed-line |
| 69 | // coverage, and confirm the agent didn't break the build anywhere. |
| 70 | func runOnce(o diffOpts, srcFiles, pkgs []string, prompt string) diffReport { |
| 71 | metricsPath := filepath.Join(o.repo, ".e2e-diff-metrics.json") |
| 72 | _ = os.Remove(metricsPath) |
| 73 | defer os.Remove(metricsPath) |
| 74 | |
| 75 | ctx, cancel := context.WithTimeout(context.Background(), time.Duration(o.timeoutSec)*time.Second) |
| 76 | defer cancel() |
| 77 | |
| 78 | args := []string{"run", "--metrics", metricsPath, "--max-steps", fmt.Sprint(o.maxSteps)} |
| 79 | if o.model != "" { |
| 80 | args = append(args, "--model", o.model) |
| 81 | } |
| 82 | if !o.ablate.Empty() { |
| 83 | args = append(args, "--ablate", o.ablate.String()) |
| 84 | } |
| 85 | args = append(args, prompt) |
| 86 | cmd := exec.CommandContext(ctx, o.bin, args...) |
| 87 | cmd.Dir = o.repo |
| 88 | cmd.Stdout = os.Stderr |
| 89 | cmd.Stderr = os.Stderr |
| 90 | cmd.WaitDelay = 10 * time.Second // bound the wait for a wedged child after ctx timeout |
| 91 | runErr := cmd.Run() |
| 92 | |
| 93 | // The agent's new files are untracked, so `git diff HEAD` would miss them; |
| 94 | // intent-to-add surfaces them as additions without committing. |
| 95 | _ = exec.Command("git", "-C", o.repo, "add", "-AN").Run() |
| 96 | |
| 97 | m, _ := readMetrics(metricsPath) |
| 98 | testDiff := gitOut(o.repo, "diff", "HEAD", "--", "*_test.go") |
| 99 | refs := parseNewTests(testDiff) |
| 100 | sourceTouched := len(changedGoFilesWorktree(o.repo, false)) |
| 101 | testsPass, testOut := runTests(o.repo, o.testCmd, pkgs) |
| 102 | |
| 103 | var pins []pinResult |
| 104 | var mut mutationResult |
| 105 | covered, coverTotal := 0, 0 |
| 106 | if len(refs) > 0 && testsPass { |
| 107 | covered, coverTotal = changedLineCoverage(o.repo, o.base, pkgs, srcFiles) |
| 108 | pins = differentialPerTest(o.repo, o.base, srcFiles, refs) |
| 109 | mut = runMutation(o.repo, o.base, srcFiles, refs) |
| 110 | } |
| 111 | buildOK, buildOut := goBuildAll(o.repo) |
| 112 | |
| 113 | passed := len(refs) > 0 && testsPass && buildOK && countPins(pins) > 0 |
| 114 | return diffReport{ |
| 115 | srcFiles: srcFiles, pkgs: pkgs, addedTestLines: countAdded(testDiff), |
| 116 | newTests: refs, sourceTouched: sourceTouched, testsPass: testsPass, |
| 117 | pins: pins, mut: mut, covered: covered, coverTotal: coverTotal, |
| 118 | buildOK: buildOK, buildOut: buildOut, failing: failingTestNames(testOut), |
| 119 | passed: passed, m: m, runErr: runErr, testOut: testOut, testDiff: testDiff, |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | // better reports whether candidate a is a stronger result than b: a pass beats a |
| 124 | // fail, then more assertion-pins, then more pins, then higher changed-line |
| 125 | // coverage. |
| 126 | func better(a, b diffReport) bool { |
| 127 | if a.passed != b.passed { |
| 128 | return a.passed |
| 129 | } |
| 130 | if x, y := countAssertionPins(a.pins), countAssertionPins(b.pins); x != y { |
| 131 | return x > y |
| 132 | } |
| 133 | if x, y := countPins(a.pins), countPins(b.pins); x != y { |
| 134 | return x > y |
| 135 | } |
| 136 | if a.mut.caught != b.mut.caught { |
| 137 | return a.mut.caught > b.mut.caught |
| 138 | } |
| 139 | return ratio(a.covered, a.coverTotal) > ratio(b.covered, b.coverTotal) |
| 140 | } |
| 141 | |
| 142 | func ratio(n, d int) float64 { |
| 143 | if d == 0 { |
| 144 | return 0 |
| 145 | } |
| 146 | return float64(n) / float64(d) |
| 147 | } |
| 148 | |
| 149 | // resetTree restores the PR-head tree between attempts, dropping the previous |
| 150 | // attempt's generated tests but keeping the provider config the workflow wrote. |
| 151 | func resetTree(repo string) { |
| 152 | _ = exec.Command("git", "-C", repo, "checkout", "--", ".").Run() |
| 153 | _ = exec.Command("git", "-C", repo, "clean", "-fd", "-e", "reasonix.toml").Run() |
| 154 | } |
| 155 | |
| 156 | func goBuildAll(repo string) (bool, string) { |
| 157 | cmd := exec.Command("go", "build", "./...") |
| 158 | cmd.Dir = repo |
| 159 | cmd.WaitDelay = 2 * time.Minute // bound the wait if `go build` hangs |
| 160 | out, err := cmd.CombinedOutput() |
| 161 | return err == nil, string(out) |
| 162 | } |
| 163 | |
| 164 | func buildDiffPrompt(srcFiles, pkgs []string, diffText string) string { |
| 165 | var b strings.Builder |
| 166 | b.WriteString("You are in a Go repository. This pull request changed these source files:\n") |
| 167 | for _, f := range srcFiles { |
| 168 | fmt.Fprintf(&b, " - %s\n", f) |
| 169 | } |
| 170 | b.WriteString("\nUnified diff of the change:\n```diff\n") |
| 171 | b.WriteString(diffText) |
| 172 | b.WriteString("\n```\n\n") |
| 173 | b.WriteString("Write focused Go unit tests that exercise the NEW or CHANGED behavior in those files. ") |
| 174 | b.WriteString("Add them to the appropriate *_test.go files in the same packages (") |
| 175 | b.WriteString(strings.Join(pkgs, ", ")) |
| 176 | b.WriteString("). Do NOT modify the non-test source files — only add or extend test files. ") |
| 177 | b.WriteString("Prefer small, focused edits and run `gofmt`/`go vet` on the test files as you go to avoid syntax errors. ") |
| 178 | b.WriteString("Then run the package tests and iterate until they pass. When finished, list the test functions you added.") |
| 179 | return b.String() |
| 180 | } |
| 181 | |
| 182 | type diffReport struct { |
| 183 | srcFiles, pkgs []string |
| 184 | addedTestLines int |
| 185 | newTests []testRef |
| 186 | sourceTouched int |
| 187 | testsPass bool |
| 188 | pins []pinResult |
| 189 | mut mutationResult |
| 190 | covered, coverTotal int |
| 191 | buildOK bool |
| 192 | buildOut string |
| 193 | failing []string |
| 194 | passed bool |
| 195 | attempt, attempts int |
| 196 | m runMetrics |
| 197 | runErr error |
| 198 | testOut string |
| 199 | testDiff string |
| 200 | } |
| 201 | |
| 202 | func renderDiff(r diffReport) string { |
| 203 | var b strings.Builder |
| 204 | result := "❌ fail" |
| 205 | if r.passed { |
| 206 | result = "✅ pass" |
| 207 | } |
| 208 | fmt.Fprint(&b, "## 🤖 Reasonix e2e — diff test-gen\n\n") |
| 209 | fmt.Fprintf(&b, "**Result:** %s · **%d** changed source file(s) across **%d** package(s)\n\n", result, len(r.srcFiles), len(r.pkgs)) |
| 210 | |
| 211 | pinned, byAssert := countPins(r.pins), countAssertionPins(r.pins) |
| 212 | fmt.Fprintf(&b, "| Metric | Value |\n|---|---|\n") |
| 213 | fmt.Fprintf(&b, "| New test functions added | %d |\n", len(r.newTests)) |
| 214 | fmt.Fprintf(&b, "| Test lines added | +%d |\n", r.addedTestLines) |
| 215 | fmt.Fprintf(&b, "| `go test` on affected pkgs | %s |\n", passFail(r.testsPass)) |
| 216 | fmt.Fprintf(&b, "| Differential (fail on pre-PR code) | %s |\n", differentialCell(r)) |
| 217 | if pinned > 0 { |
| 218 | fmt.Fprintf(&b, "| ↳ pin by assertion / by compile only | %d / %d |\n", byAssert, pinned-byAssert) |
| 219 | } |
| 220 | fmt.Fprintf(&b, "| Changed-line coverage | %s |\n", coverageCell(r)) |
| 221 | fmt.Fprintf(&b, "| Mutation (changed funcs caught) | %s |\n", mutationCell(r)) |
| 222 | fmt.Fprintf(&b, "| `go build ./...` (regression) | %s |\n", passFail(r.buildOK)) |
| 223 | fmt.Fprintf(&b, "| Non-test source touched by agent | %d file(s) |\n", r.sourceTouched) |
| 224 | fmt.Fprintf(&b, "| Cache hit | %s |\n", pct(r.m.CacheHitTokens, r.m.CacheHitTokens+r.m.CacheMissTokens)) |
| 225 | fmt.Fprintf(&b, "| Tokens (prompt / completion) | %s / %s |\n", comma(r.m.PromptTokens), comma(r.m.CompletionTokens)) |
| 226 | fmt.Fprintf(&b, "| Model calls | %d |\n", r.m.Steps) |
| 227 | fmt.Fprintf(&b, "| Cost | %s%.4f |\n", currencySym(r.m.Currency), r.m.Cost) |
| 228 | if r.m.CapabilityRoutes > 0 || r.m.CapabilitySkillInvocations > 0 || r.m.CapabilityMCPCall > 0 || r.m.ReadinessChecks > 0 { |
| 229 | fmt.Fprintf(&b, "| Capability routes (semantic) | %d (%d) |\n", r.m.CapabilityRoutes, r.m.CapabilitySemanticRoutes) |
| 230 | fmt.Fprintf(&b, "| Routed candidates (require / prefer / suggest / declined) | %d (%d / %d / %d / %d) |\n", r.m.CapabilityRoutedCandidates, r.m.CapabilityRoutedRequire, r.m.CapabilityRoutedPrefer, r.m.CapabilityRoutedSuggest, r.m.CapabilityDeclines) |
| 231 | fmt.Fprintf(&b, "| Skill invocations / MCP proxy calls | %d / %d |\n", r.m.CapabilitySkillInvocations, r.m.CapabilityMCPCall) |
| 232 | fmt.Fprintf(&b, "| Review blocks / readiness recoveries | %d / %d |\n", r.m.CapabilityReviewBlocks, r.m.ReadinessRecoveries) |
| 233 | if r.m.CapabilityRouterCost > 0 || r.m.CapabilityRouterLatencyMs > 0 { |
| 234 | fmt.Fprintf(&b, "| Capability-router cost / latency | %s%.4f / %dms |\n", currencySym(r.m.Currency), r.m.CapabilityRouterCost, r.m.CapabilityRouterLatencyMs) |
| 235 | } |
| 236 | } |
| 237 | if len(r.failing) > 0 { |
| 238 | fmt.Fprintf(&b, "| Failing tests | `%s` |\n", strings.Join(r.failing, "`, `")) |
| 239 | } |
| 240 | if r.attempts > 1 { |
| 241 | status := "none passed" |
| 242 | if r.passed { |
| 243 | status = "passed" |
| 244 | } |
| 245 | fmt.Fprintf(&b, "| Attempts | %d of up to %d (%s) |\n", r.attempt, r.attempts, status) |
| 246 | } |
| 247 | |
| 248 | fmt.Fprintf(&b, "\n**Packages:** %s\n", strings.Join(r.pkgs, ", ")) |
| 249 | if r.attempts <= 1 { |
| 250 | fmt.Fprintf(&b, "\n<sub>Single stochastic run — a green result is one sample, not a guarantee. Comment `/e2e diff x3` to retry up to 3×.</sub>\n") |
| 251 | } |
| 252 | if !r.buildOK && strings.TrimSpace(r.buildOut) != "" { |
| 253 | fmt.Fprintf(&b, "\n<details><summary>go build ./... output (tail)</summary>\n\n```\n%s\n```\n</details>\n", tail(r.buildOut, 40)) |
| 254 | } |
| 255 | if r.sourceTouched > 0 { |
| 256 | fmt.Fprintf(&b, "\n⚠️ The agent modified %d non-test source file(s); a green run may not reflect the PR's code. Review the diff.\n", r.sourceTouched) |
| 257 | } |
| 258 | |
| 259 | if len(r.pins) > 0 { |
| 260 | fmt.Fprintf(&b, "\n<details><summary>Per-test differential</summary>\n\n| Test | Package | Pins the change? |\n|---|---|---|\n") |
| 261 | for _, p := range r.pins { |
| 262 | fmt.Fprintf(&b, "| `%s` | %s | %s |\n", p.name, p.pkg, pinCell(p)) |
| 263 | } |
| 264 | fmt.Fprintf(&b, "\n</details>\n") |
| 265 | } |
| 266 | if strings.TrimSpace(r.testDiff) != "" { |
| 267 | fmt.Fprintf(&b, "\n<details><summary>Generated tests (review the assertions)</summary>\n\n```diff\n%s\n```\n</details>\n", truncateFor(r.testDiff, 20000)) |
| 268 | } |
| 269 | if !r.testsPass && strings.TrimSpace(r.testOut) != "" { |
| 270 | fmt.Fprintf(&b, "\n<details><summary>go test output (tail)</summary>\n\n```\n%s\n```\n</details>\n", tail(r.testOut, 60)) |
| 271 | } |
| 272 | if r.runErr != nil { |
| 273 | fmt.Fprintf(&b, "\n<sub>agent run note: %v</sub>\n", r.runErr) |
| 274 | } |
| 275 | fmt.Fprintf(&b, "\n<sub>Pass = the agent added ≥1 test, the affected packages are green, AND ≥1 new test fails when the PR's source is reverted. \"By assertion\" pins are strong (they check changed behavior); \"by compile only\" pins just need a PR-added symbol — and since Go compiles per package, one compile-coupled test marks every test in its package that way. Mutation is the behavioral signal for additive PRs: each changed function's return is replaced with zero values and the new tests are re-run; \"caught\" means a test asserts that output, \"survived\" means it doesn't. Read the generated tests above to judge the rest.</sub>\n") |
| 276 | return b.String() |
| 277 | } |
| 278 | |
| 279 | func differentialCell(r diffReport) string { |
| 280 | if !(len(r.newTests) > 0 && r.testsPass) { |
| 281 | return "n/a (tests not green)" |
| 282 | } |
| 283 | return fmt.Sprintf("%d/%d new tests", countPins(r.pins), len(r.pins)) |
| 284 | } |
| 285 | |
| 286 | func coverageCell(r diffReport) string { |
| 287 | if r.coverTotal == 0 { |
| 288 | return "n/a" |
| 289 | } |
| 290 | return fmt.Sprintf("%s (%d/%d changed lines)", pct(r.covered, r.coverTotal), r.covered, r.coverTotal) |
| 291 | } |
| 292 | |
| 293 | func mutationCell(r diffReport) string { |
| 294 | if r.mut.total == 0 { |
| 295 | return "n/a" |
| 296 | } |
| 297 | cell := fmt.Sprintf("%d/%d (%s)", r.mut.caught, r.mut.total, pct(r.mut.caught, r.mut.total)) |
| 298 | if len(r.mut.survivors) > 0 { |
| 299 | cell += fmt.Sprintf(" · survived: `%s`", strings.Join(r.mut.survivors, "`, `")) |
| 300 | } |
| 301 | return cell |
| 302 | } |
| 303 | |
| 304 | func pinCell(p pinResult) string { |
| 305 | switch { |
| 306 | case p.pins && p.byAssertion: |
| 307 | return "✅ by assertion" |
| 308 | case p.pins: |
| 309 | return "⚠️ by compile only" |
| 310 | default: |
| 311 | return "❌ no (passes on old code)" |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | // differentialPerTest reverts the PR's changed source to base (deleting files |
| 316 | // new in the PR), runs each generated test on its own against the old code, and |
| 317 | // restores the source. A test that fails on the old code pins the change. |
| 318 | func differentialPerTest(repo, base string, srcFiles []string, refs []testRef) []pinResult { |
| 319 | for _, f := range srcFiles { |
| 320 | if err := exec.Command("git", "-C", repo, "checkout", base, "--", f).Run(); err != nil { |
| 321 | _ = os.Remove(filepath.Join(repo, filepath.FromSlash(f))) |
| 322 | } |
| 323 | } |
| 324 | // Restore source even on panic; a tree left on `base` would mask the PR for later steps. |
| 325 | restored := false |
| 326 | defer func() { |
| 327 | if restored { |
| 328 | return |
| 329 | } |
| 330 | for _, f := range srcFiles { |
| 331 | _ = exec.Command("git", "-C", repo, "checkout", "HEAD", "--", f).Run() |
| 332 | } |
| 333 | }() |
| 334 | |
| 335 | out := make([]pinResult, 0, len(refs)) |
| 336 | for _, r := range refs { |
| 337 | cmd := exec.Command("go", "test", "-run", "^"+r.name+"$", r.pkg) |
| 338 | cmd.Dir = repo |
| 339 | cmd.WaitDelay = 2 * time.Minute // bound the wait for a hung test |
| 340 | raw, err := cmd.CombinedOutput() |
| 341 | out = append(out, pinResult{ |
| 342 | testRef: r, |
| 343 | pins: err != nil, |
| 344 | byAssertion: strings.Contains(string(raw), "--- FAIL: "+r.name), |
| 345 | }) |
| 346 | } |
| 347 | for _, f := range srcFiles { |
| 348 | _ = exec.Command("git", "-C", repo, "checkout", "HEAD", "--", f).Run() |
| 349 | } |
| 350 | restored = true |
| 351 | return out |
| 352 | } |
| 353 | |
| 354 | // changedLineCoverage runs the affected packages with a coverage profile and |
| 355 | // reports how many of the PR's changed source statement-lines the (new+existing) |
| 356 | // tests actually execute. covered/total are over changed lines that fall inside |
| 357 | // a coverage block; lines that aren't statements are ignored. |
| 358 | func changedLineCoverage(repo, base string, pkgs, srcFiles []string) (covered, total int) { |
| 359 | profile := filepath.Join(repo, ".e2e-cover.out") |
| 360 | defer os.Remove(profile) |
| 361 | args := append([]string{"test", "-covermode=set", "-coverprofile=" + profile, "-coverpkg=" + strings.Join(pkgs, ",")}, pkgs...) |
| 362 | cmd := exec.Command("go", args...) |
| 363 | cmd.Dir = repo |
| 364 | _ = cmd.Run() // a non-zero exit still writes the profile for the tests that ran |
| 365 | |
| 366 | blocks := parseCoverProfile(repo, profile) |
| 367 | for file, lines := range changedLineSet(repo, base, srcFiles) { |
| 368 | fileBlocks := blocks[file] |
| 369 | for ln := range lines { |
| 370 | for _, blk := range fileBlocks { |
| 371 | if ln >= blk.start && ln <= blk.end { |
| 372 | total++ |
| 373 | if blk.count > 0 { |
| 374 | covered++ |
| 375 | } |
| 376 | break |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | } |
| 381 | return covered, total |
| 382 | } |
| 383 | |
| 384 | type coverBlock struct { |
| 385 | start, end, count int |
| 386 | } |
| 387 | |
| 388 | // parseCoverProfile reads a Go coverage profile, keyed by repo-relative file path |
| 389 | // (the profile uses module-qualified paths; we match by repo-relative suffix). |
| 390 | func parseCoverProfile(repo, path string) map[string][]coverBlock { |
| 391 | data, err := os.ReadFile(path) |
| 392 | if err != nil { |
| 393 | return nil |
| 394 | } |
| 395 | out := map[string][]coverBlock{} |
| 396 | for ln := range strings.SplitSeq(string(data), "\n") { |
| 397 | if ln == "" || strings.HasPrefix(ln, "mode:") { |
| 398 | continue |
| 399 | } |
| 400 | colon := strings.LastIndexByte(ln, ':') |
| 401 | if colon < 0 { |
| 402 | continue |
| 403 | } |
| 404 | modPath, rest := ln[:colon], ln[colon+1:] |
| 405 | var sl, sc, el, ec, nstmt, count int |
| 406 | if _, err := fmt.Sscanf(rest, "%d.%d,%d.%d %d %d", &sl, &sc, &el, &ec, &nstmt, &count); err != nil { |
| 407 | continue |
| 408 | } |
| 409 | rel := repoRelFromModulePath(modPath) |
| 410 | out[rel] = append(out[rel], coverBlock{start: sl, end: el, count: count}) |
| 411 | } |
| 412 | return out |
| 413 | } |
| 414 | |
| 415 | // repoRelFromModulePath turns "reasonix/internal/agent/foo.go" into |
| 416 | // "internal/agent/foo.go" by dropping the first path element (the module root). |
| 417 | func repoRelFromModulePath(p string) string { |
| 418 | // Strip the full module prefix; a generic first-segment cut mis-strips a multi-segment module path. |
| 419 | prefix := "reasonix/" |
| 420 | if strings.HasPrefix(p, prefix) { |
| 421 | return p[len(prefix):] |
| 422 | } |
| 423 | if _, after, ok := strings.Cut(p, "/"); ok { |
| 424 | return after |
| 425 | } |
| 426 | return p |
| 427 | } |
| 428 | |
| 429 | // changedLineSet returns, per repo-relative source file, the set of new line |
| 430 | // numbers the PR added or changed (from a zero-context diff). |
| 431 | func changedLineSet(repo, base string, srcFiles []string) map[string]map[int]bool { |
| 432 | args := append([]string{"diff", "--unified=0", base + "...HEAD", "--"}, srcFiles...) |
| 433 | diff := gitOut(repo, args...) |
| 434 | out := map[string]map[int]bool{} |
| 435 | file := "" |
| 436 | newLine := 0 |
| 437 | for ln := range strings.SplitSeq(diff, "\n") { |
| 438 | // '-' (deletion) lines are intentionally unhandled: they don't advance the |
| 439 | // new-side line counter, so they fall through with no case. |
| 440 | switch { |
| 441 | case strings.HasPrefix(ln, "+++ b/"): |
| 442 | file = strings.TrimPrefix(ln, "+++ b/") |
| 443 | out[file] = map[int]bool{} |
| 444 | case strings.HasPrefix(ln, "@@"): |
| 445 | // @@ -a,b +c,d @@ — start collecting at new-side line c. |
| 446 | // Digit-only cut: malformed headers (e.g. `@@ +abc @@`) fail closed. |
| 447 | if _, after, ok := strings.Cut(ln, "+"); ok { |
| 448 | num := after |
| 449 | end := len(num) |
| 450 | for i := range len(num) { |
| 451 | if num[i] < '0' || num[i] > '9' { |
| 452 | end = i |
| 453 | break |
| 454 | } |
| 455 | } |
| 456 | _, _ = fmt.Sscanf(num[:end], "%d", &newLine) |
| 457 | } |
| 458 | case strings.HasPrefix(ln, "+") && !strings.HasPrefix(ln, "+++"): |
| 459 | if file != "" { |
| 460 | out[file][newLine] = true |
| 461 | } |
| 462 | newLine++ |
| 463 | } |
| 464 | } |
| 465 | return out |
| 466 | } |
| 467 | |
| 468 | func countPins(ps []pinResult) int { |
| 469 | n := 0 |
| 470 | for _, p := range ps { |
| 471 | if p.pins { |
| 472 | n++ |
| 473 | } |
| 474 | } |
| 475 | return n |
| 476 | } |
| 477 | |
| 478 | func countAssertionPins(ps []pinResult) int { |
| 479 | n := 0 |
| 480 | for _, p := range ps { |
| 481 | if p.pins && p.byAssertion { |
| 482 | n++ |
| 483 | } |
| 484 | } |
| 485 | return n |
| 486 | } |
| 487 | |
| 488 | // parseNewTests reads the working-tree *_test.go diff and returns the Test/Fuzz/ |
| 489 | // Benchmark functions the agent added, each tagged with its package directory. |
| 490 | func parseNewTests(diff string) []testRef { |
| 491 | var refs []testRef |
| 492 | pkg := "" |
| 493 | for ln := range strings.SplitSeq(diff, "\n") { |
| 494 | if after, ok := strings.CutPrefix(ln, "+++ b/"); ok { |
| 495 | pkg = "./" + filepath.ToSlash(filepath.Dir(after)) |
| 496 | continue |
| 497 | } |
| 498 | if !strings.HasPrefix(ln, "+") || strings.HasPrefix(ln, "+++") { |
| 499 | continue |
| 500 | } |
| 501 | body := strings.TrimSpace(ln[1:]) |
| 502 | if !strings.HasPrefix(body, "func ") { |
| 503 | continue |
| 504 | } |
| 505 | sig := strings.TrimPrefix(body, "func ") |
| 506 | // Method form `(r T) Name(...)` starts with '('; parse the receiver out before the name. |
| 507 | var name string |
| 508 | if sig[0] == '(' { |
| 509 | _, after, ok := strings.Cut(sig, ")") |
| 510 | if !ok { |
| 511 | continue |
| 512 | } |
| 513 | rest := strings.TrimSpace(after) |
| 514 | methodParen := strings.IndexByte(rest, '(') |
| 515 | if methodParen <= 0 { |
| 516 | continue |
| 517 | } |
| 518 | name = rest[:methodParen] |
| 519 | } else { |
| 520 | funcParen := strings.IndexByte(sig, '(') |
| 521 | if funcParen <= 0 { |
| 522 | continue |
| 523 | } |
| 524 | name = sig[:funcParen] |
| 525 | } |
| 526 | if strings.HasPrefix(name, "Test") || strings.HasPrefix(name, "Fuzz") || strings.HasPrefix(name, "Benchmark") { |
| 527 | refs = append(refs, testRef{name: name, pkg: pkg}) |
| 528 | } |
| 529 | } |
| 530 | return refs |
| 531 | } |
| 532 | |
| 533 | func countAdded(diff string) int { |
| 534 | n := 0 |
| 535 | for ln := range strings.SplitSeq(diff, "\n") { |
| 536 | if strings.HasPrefix(ln, "+") && !strings.HasPrefix(ln, "+++") { |
| 537 | n++ |
| 538 | } |
| 539 | } |
| 540 | return n |
| 541 | } |
| 542 | |
| 543 | // failingTestNames pulls the names out of `--- FAIL: TestX (…)` lines. |
| 544 | func failingTestNames(out string) []string { |
| 545 | var names []string |
| 546 | seen := map[string]bool{} |
| 547 | for ln := range strings.SplitSeq(out, "\n") { |
| 548 | ln = strings.TrimSpace(ln) |
| 549 | if !strings.HasPrefix(ln, "--- FAIL:") { |
| 550 | continue |
| 551 | } |
| 552 | rest := strings.Fields(strings.TrimSpace(strings.TrimPrefix(ln, "--- FAIL:"))) |
| 553 | if len(rest) > 0 && !seen[rest[0]] { |
| 554 | seen[rest[0]] = true |
| 555 | names = append(names, rest[0]) |
| 556 | } |
| 557 | } |
| 558 | return names |
| 559 | } |
| 560 | |
| 561 | func runTests(repo, testCmd string, pkgs []string) (bool, string) { |
| 562 | test, err := shellparse.ParseStaticCommand(testCmd, shellparse.StaticCommandPolicy{AllowEnvAssignments: true, AllowStderrToStdout: true}) |
| 563 | if err != nil { |
| 564 | return false, "invalid test command: " + err.Error() |
| 565 | } |
| 566 | fields := test.Argv |
| 567 | if len(fields) == 0 { |
| 568 | fields = []string{"go", "test"} |
| 569 | } |
| 570 | args := append(fields[1:], pkgs...) |
| 571 | cmd := exec.Command(fields[0], args...) |
| 572 | cmd.Dir = repo |
| 573 | if len(test.Env) > 0 { |
| 574 | cmd.Env = append(os.Environ(), test.Env...) |
| 575 | } |
| 576 | cmd.WaitDelay = 5 * time.Minute // bound the wait if `go test` hangs |
| 577 | out, err := cmd.CombinedOutput() |
| 578 | return err == nil, string(out) |
| 579 | } |
| 580 | |
| 581 | // changedGoFiles lists .go files changed by base...HEAD, excluding *_test.go |
| 582 | // when includeTests is false (we want the source under test). |
| 583 | func changedGoFiles(repo, base string, includeTests bool) []string { |
| 584 | return filterGo(gitOut(repo, "diff", "--name-only", base+"...HEAD", "--", "*.go"), includeTests) |
| 585 | } |
| 586 | |
| 587 | func changedGoFilesWorktree(repo string, includeTests bool) []string { |
| 588 | return filterGo(gitOut(repo, "diff", "--name-only", "HEAD", "--", "*.go"), includeTests) |
| 589 | } |
| 590 | |
| 591 | func filterGo(out string, includeTests bool) []string { |
| 592 | var keep []string |
| 593 | for f := range strings.FieldsSeq(strings.ReplaceAll(out, "\n", " ")) { |
| 594 | if strings.HasSuffix(f, "_test.go") && !includeTests { |
| 595 | continue |
| 596 | } |
| 597 | keep = append(keep, f) |
| 598 | } |
| 599 | sort.Strings(keep) |
| 600 | return keep |
| 601 | } |
| 602 | |
| 603 | func packagesOf(files []string) []string { |
| 604 | seen := map[string]bool{} |
| 605 | var pkgs []string |
| 606 | for _, f := range files { |
| 607 | dir := "./" + filepath.ToSlash(filepath.Dir(f)) |
| 608 | if !seen[dir] { |
| 609 | seen[dir] = true |
| 610 | pkgs = append(pkgs, dir) |
| 611 | } |
| 612 | } |
| 613 | sort.Strings(pkgs) |
| 614 | return pkgs |
| 615 | } |
| 616 | |
| 617 | func gitOut(repo string, args ...string) string { |
| 618 | cmd := exec.Command("git", append([]string{"-C", repo}, args...)...) |
| 619 | out, _ := cmd.Output() |
| 620 | return string(out) |
| 621 | } |
| 622 | |
| 623 | func truncate(s string) string { return truncateFor(s, 12000) } |
| 624 | |
| 625 | func truncateFor(s string, max int) string { |
| 626 | if max <= 0 || len(s) <= max { |
| 627 | return s |
| 628 | } |
| 629 | // Back the cut up to a rune boundary so we don't split a multi-byte UTF-8 rune. |
| 630 | cut := max |
| 631 | for cut > 0 && !utf8.RuneStart(s[cut]) { |
| 632 | cut-- |
| 633 | } |
| 634 | return s[:cut] + "\n…(truncated)…" |
| 635 | } |
| 636 | |
| 637 | func tail(s string, n int) string { |
| 638 | lines := strings.Split(strings.TrimRight(s, "\n"), "\n") |
| 639 | if len(lines) > n { |
| 640 | lines = lines[len(lines)-n:] |
| 641 | } |
| 642 | return strings.Join(lines, "\n") |
| 643 | } |
| 644 | |
| 645 | func passFail(ok bool) string { |
| 646 | if ok { |
| 647 | return "pass" |
| 648 | } |
| 649 | return "fail" |
| 650 | } |
| 651 |