| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "strings" |
| 6 | ) |
| 7 | |
| 8 | // Anchor arms decide what hypothesis the agent holds before it has looked at |
| 9 | // anything: none, the task's real cause, or a plausible cause that is wrong. |
| 10 | // Comparing the three prices how much a handed-down conclusion is worth, and |
| 11 | // how much of it survives contact with the evidence. |
| 12 | const ( |
| 13 | anchorBlind = "blind" |
| 14 | anchorCorrect = "correct" |
| 15 | anchorWrong = "wrong" |
| 16 | ) |
| 17 | |
| 18 | func normalizeAnchorArm(arm string) (string, error) { |
| 19 | switch strings.ToLower(strings.TrimSpace(arm)) { |
| 20 | case "", anchorBlind: |
| 21 | return anchorBlind, nil |
| 22 | case anchorCorrect: |
| 23 | return anchorCorrect, nil |
| 24 | case anchorWrong: |
| 25 | return anchorWrong, nil |
| 26 | default: |
| 27 | return "", fmt.Errorf("unknown anchor arm %q (want blind, correct or wrong)", arm) |
| 28 | } |
| 29 | } |
| 30 | |
| 31 | // seedFor returns the hypothesis this arm hands the agent, and whether the |
| 32 | // task can be scored in that arm at all. A task with no authored seed is never |
| 33 | // silently run blind: its control run would land in the seeded denominator and |
| 34 | // flatter whichever arm collected it. |
| 35 | func (t task) seedFor(anchor string) (string, bool) { |
| 36 | switch anchor { |
| 37 | case "", anchorBlind: |
| 38 | return "", true |
| 39 | case anchorCorrect: |
| 40 | seed := strings.TrimSpace(t.SeedCorrect) |
| 41 | return seed, seed != "" |
| 42 | case anchorWrong: |
| 43 | seed := strings.TrimSpace(t.SeedWrong) |
| 44 | return seed, seed != "" |
| 45 | } |
| 46 | return "", false |
| 47 | } |
| 48 | |
| 49 | // anchorPrompt puts this arm's hypothesis in front of the task, where the |
| 50 | // agent meets it before it has read anything. An anchor offered after |
| 51 | // exploration would be a different experiment. |
| 52 | func anchorPrompt(anchor string, t task) string { |
| 53 | seed, ok := t.seedFor(anchor) |
| 54 | if !ok || seed == "" { |
| 55 | return t.Prompt |
| 56 | } |
| 57 | return seed + "\n\n" + t.Prompt |
| 58 | } |
| 59 | |
| 60 | // anchorSkip returns the recorded skip for a task this arm cannot score, so a |
| 61 | // missing seed leaves a visible row instead of quietly shrinking the corpus. |
| 62 | func anchorSkip(cfg suiteConfig, t task) (result, bool) { |
| 63 | if _, ok := t.seedFor(cfg.anchor); ok { |
| 64 | return result{}, false |
| 65 | } |
| 66 | return result{ |
| 67 | task: t, Profile: benchmarkProfileStandard, Anchor: cfg.anchor, Skipped: true, |
| 68 | Note: "skipped: no seed_" + cfg.anchor + " authored for this task", |
| 69 | }, true |
| 70 | } |
| 71 | |
| 72 | // renderAnchor reports the arm and what it scored. Anchor resistance is only |
| 73 | // meaningful against a blind baseline from the same corpus, so this section |
| 74 | // publishes one arm's numbers and names the comparison rather than inventing a |
| 75 | // resistance figure from a single run. |
| 76 | func renderAnchor(results []result) string { |
| 77 | arm := "" |
| 78 | solved, total, skipped := 0, 0, 0 |
| 79 | for _, r := range results { |
| 80 | if strings.TrimSpace(r.Anchor) != "" { |
| 81 | arm = r.Anchor |
| 82 | } |
| 83 | if r.Skipped { |
| 84 | if strings.Contains(r.Note, "no seed_") { |
| 85 | skipped++ |
| 86 | } |
| 87 | continue |
| 88 | } |
| 89 | total++ |
| 90 | if r.Passed { |
| 91 | solved++ |
| 92 | } |
| 93 | } |
| 94 | if arm == "" || arm == anchorBlind { |
| 95 | // The control arm needs no section of its own: it is the ordinary run. |
| 96 | if skipped == 0 { |
| 97 | return "" |
| 98 | } |
| 99 | } |
| 100 | b := fmt.Sprintf("**Anchor**: arm **%s** · solved %d/%d (%s)\n", arm, solved, total, pct(solved, total)) |
| 101 | if skipped > 0 { |
| 102 | // Naming the drop matters: a seeded arm silently scoring fewer tasks |
| 103 | // than the blind baseline is not the same experiment. |
| 104 | b += fmt.Sprintf("- **%d task(s) skipped**: no seed authored, so they are outside this arm's denominator\n", skipped) |
| 105 | } |
| 106 | if arm == anchorWrong { |
| 107 | b += "- resistance = this solve rate over the blind arm's on the same tasks\n" |
| 108 | } |
| 109 | return b + "\n" |
| 110 | } |
| 111 |