| 1 | // Package ablation switches individual Reasonix subsystems off so a benchmark |
| 2 | // can attribute a change in solve rate to one of them. |
| 3 | package ablation |
| 4 | |
| 5 | import ( |
| 6 | "fmt" |
| 7 | "sort" |
| 8 | "strings" |
| 9 | ) |
| 10 | |
| 11 | type Module string |
| 12 | |
| 13 | const ( |
| 14 | Evidence Module = "evidence" |
| 15 | Planner Module = "planner" |
| 16 | Subagent Module = "subagent" |
| 17 | Retrieval Module = "retrieval" |
| 18 | Compaction Module = "compaction" |
| 19 | // FullFold off means a fold reads the previous projection instead of |
| 20 | // re-deriving its digest from the canonical transcript. |
| 21 | FullFold Module = "full-fold" |
| 22 | ) |
| 23 | |
| 24 | // Modules returns every switchable module in the order arm names use. |
| 25 | func Modules() []Module { |
| 26 | return []Module{Evidence, Planner, Subagent, Retrieval, Compaction, FullFold} |
| 27 | } |
| 28 | |
| 29 | // Set is the group of modules disabled for a run. The zero value is the |
| 30 | // control arm: everything on. |
| 31 | type Set struct { |
| 32 | off map[Module]bool |
| 33 | } |
| 34 | |
| 35 | // Parse reads a spec such as "evidence,planner". "" and "none" mean the control |
| 36 | // arm; "all" disables every module. |
| 37 | func Parse(spec string) (Set, error) { |
| 38 | spec = strings.TrimSpace(spec) |
| 39 | if spec == "" || strings.EqualFold(spec, "none") { |
| 40 | return Set{}, nil |
| 41 | } |
| 42 | if strings.EqualFold(spec, "all") { |
| 43 | return New(Modules()...), nil |
| 44 | } |
| 45 | known := map[Module]bool{} |
| 46 | for _, m := range Modules() { |
| 47 | known[m] = true |
| 48 | } |
| 49 | var mods []Module |
| 50 | for _, field := range strings.FieldsFunc(spec, func(r rune) bool { return r == ',' || r == ' ' }) { |
| 51 | m := Module(strings.ToLower(strings.TrimSpace(field))) |
| 52 | if !known[m] { |
| 53 | return Set{}, fmt.Errorf("unknown ablation module %q (want %s, or none/all)", field, joinModules(Modules(), ", ")) |
| 54 | } |
| 55 | mods = append(mods, m) |
| 56 | } |
| 57 | return New(mods...), nil |
| 58 | } |
| 59 | |
| 60 | // New returns a Set with the given modules disabled. |
| 61 | func New(mods ...Module) Set { |
| 62 | if len(mods) == 0 { |
| 63 | return Set{} |
| 64 | } |
| 65 | off := make(map[Module]bool, len(mods)) |
| 66 | for _, m := range mods { |
| 67 | off[m] = true |
| 68 | } |
| 69 | return Set{off: off} |
| 70 | } |
| 71 | |
| 72 | func (s Set) Off(m Module) bool { return s.off[m] } |
| 73 | |
| 74 | func (s Set) Empty() bool { return len(s.off) == 0 } |
| 75 | |
| 76 | // Arm is the published name of this configuration: "full" for the control arm, |
| 77 | // otherwise "no-evidence+no-planner". Stable across runs so results from |
| 78 | // different machines group by the same key. |
| 79 | func (s Set) Arm() string { |
| 80 | if s.Empty() { |
| 81 | return "full" |
| 82 | } |
| 83 | parts := make([]string, 0, len(s.off)) |
| 84 | for _, m := range s.disabled() { |
| 85 | parts = append(parts, "no-"+string(m)) |
| 86 | } |
| 87 | return strings.Join(parts, "+") |
| 88 | } |
| 89 | |
| 90 | // String round-trips back through Parse. |
| 91 | func (s Set) String() string { |
| 92 | if s.Empty() { |
| 93 | return "none" |
| 94 | } |
| 95 | return joinModules(s.disabled(), ",") |
| 96 | } |
| 97 | |
| 98 | func (s Set) disabled() []Module { |
| 99 | order := map[Module]int{} |
| 100 | for i, m := range Modules() { |
| 101 | order[m] = i |
| 102 | } |
| 103 | out := make([]Module, 0, len(s.off)) |
| 104 | for m := range s.off { |
| 105 | out = append(out, m) |
| 106 | } |
| 107 | sort.Slice(out, func(i, j int) bool { return order[out[i]] < order[out[j]] }) |
| 108 | return out |
| 109 | } |
| 110 | |
| 111 | func joinModules(mods []Module, sep string) string { |
| 112 | parts := make([]string, len(mods)) |
| 113 | for i, m := range mods { |
| 114 | parts[i] = string(m) |
| 115 | } |
| 116 | return strings.Join(parts, sep) |
| 117 | } |
| 118 |