| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "flag" |
| 5 | "fmt" |
| 6 | "os" |
| 7 | "strconv" |
| 8 | "strings" |
| 9 | ) |
| 10 | |
| 11 | // segment is one leg of a task: leg 1 starts the session, later legs resume it. |
| 12 | // Segmenting reaches the states that only appear in a long session — reload, |
| 13 | // prefix reconstruction, compaction across a boundary — without waiting hours |
| 14 | // for them to arrive on their own. |
| 15 | type segment struct { |
| 16 | index int |
| 17 | maxSteps int |
| 18 | prompt string |
| 19 | resume bool |
| 20 | } |
| 21 | |
| 22 | // continuationPrompt is deliberately empty of new instruction: a resumed leg |
| 23 | // must exercise the session's own memory of the task, not be handed the task |
| 24 | // again. A prompt that restates the work would hide exactly the degradation |
| 25 | // this measures. |
| 26 | const continuationPrompt = "Continue from where you left off." |
| 27 | |
| 28 | // planSegments splits a task into count legs, dividing its step budget and |
| 29 | // giving the remainder to the last leg so the total never exceeds max_steps. |
| 30 | // A steer entry replaces a leg's continuation prompt. |
| 31 | func planSegments(t task, count int, steers map[int]string) []segment { |
| 32 | if count < 2 { |
| 33 | return []segment{{index: 1, maxSteps: t.MaxSteps, prompt: t.Prompt}} |
| 34 | } |
| 35 | per := t.MaxSteps / count |
| 36 | out := make([]segment, 0, count) |
| 37 | for i := 1; i <= count; i++ { |
| 38 | steps := per |
| 39 | if i == count { |
| 40 | steps = t.MaxSteps - per*(count-1) |
| 41 | } |
| 42 | seg := segment{index: i, maxSteps: steps, prompt: continuationPrompt, resume: true} |
| 43 | if i == 1 { |
| 44 | seg.prompt, seg.resume = t.Prompt, false |
| 45 | } |
| 46 | if steer, ok := steers[i]; ok { |
| 47 | seg.prompt = steer |
| 48 | } |
| 49 | out = append(out, seg) |
| 50 | } |
| 51 | return out |
| 52 | } |
| 53 | |
| 54 | // segmentSettings validates the two LongRun pressure flags together: a steer |
| 55 | // aimed past the last leg would never be delivered, and silently dropping a |
| 56 | // user turn is exactly the kind of quiet no-op a benchmark must not have. |
| 57 | func segmentSettings(segments int, spec string) map[int]string { |
| 58 | steers, err := parseSteers(spec) |
| 59 | if err != nil { |
| 60 | fmt.Fprintln(os.Stderr, "steer:", err) |
| 61 | os.Exit(2) |
| 62 | } |
| 63 | for leg := range steers { |
| 64 | if leg > segments { |
| 65 | fmt.Fprintf(os.Stderr, "steer: leg %d needs -segments %d or more; the run has %d\n", leg, leg, segments) |
| 66 | os.Exit(2) |
| 67 | } |
| 68 | } |
| 69 | return steers |
| 70 | } |
| 71 | |
| 72 | // parseSteers reads "message@2,another@3": deliver that message as leg N's |
| 73 | // prompt. Steering is a user turn arriving mid-task, so it belongs to a leg |
| 74 | // boundary rather than to a wall-clock instant nothing can reproduce. |
| 75 | func parseSteers(spec string) (map[int]string, error) { |
| 76 | if strings.TrimSpace(spec) == "" { |
| 77 | return nil, nil |
| 78 | } |
| 79 | out := map[int]string{} |
| 80 | for field := range strings.SplitSeq(spec, ",") { |
| 81 | message, at, ok := strings.Cut(strings.TrimSpace(field), "@") |
| 82 | if !ok { |
| 83 | return nil, fmt.Errorf("steer %q: want <message>@<segment>", field) |
| 84 | } |
| 85 | leg, err := strconv.Atoi(strings.TrimSpace(at)) |
| 86 | if err != nil || leg < 2 { |
| 87 | return nil, fmt.Errorf("steer %q: the segment must be 2 or later — leg 1 carries the task itself", field) |
| 88 | } |
| 89 | if strings.TrimSpace(message) == "" { |
| 90 | return nil, fmt.Errorf("steer %q: the message is empty", field) |
| 91 | } |
| 92 | out[leg] = strings.TrimSpace(message) |
| 93 | } |
| 94 | return out, nil |
| 95 | } |
| 96 | |
| 97 | // pressureFlags groups the LongRun knobs: neutral metering, injected faults, |
| 98 | // and segmented resume with steering. They are registered and validated |
| 99 | // together because they only make sense together. |
| 100 | type pressureFlags struct { |
| 101 | meter, faults, steer *string |
| 102 | segments *int |
| 103 | } |
| 104 | |
| 105 | func registerPressureFlags() pressureFlags { |
| 106 | return pressureFlags{ |
| 107 | meter: flag.String("meter", "", "suite mode: route the benchmarked provider through the neutral measuring proxy, using this config.toml as the source (e.g. ~/.reasonix/config.toml). Spend is then counted at the request boundary instead of trusted from the harness"), |
| 108 | faults: flag.String("faults", "", "suite mode: inject provider failures through the meter — absolute indices (3:429) and/or a cadence that scales with the run (every:5:500). Requires -meter; the report gains a fault-recovery readout"), |
| 109 | steer: flag.String("steer", "", "suite mode: deliver a user turn at a leg boundary, e.g. \"also handle empty input@2\" (requires -segments >= that leg)"), |
| 110 | segments: flag.Int("segments", 1, "suite mode: split each task into N resumed legs (--continue between them), reaching reload and compaction pressure without waiting hours for it. The step budget is divided, never multiplied"), |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | func (p pressureFlags) settings() (config string, faults faultScript, segments int, steers map[int]string) { |
| 115 | config, faults = meterSettings(*p.meter, *p.faults) |
| 116 | return config, faults, *p.segments, segmentSettings(*p.segments, *p.steer) |
| 117 | } |
| 118 |