| 1 | package main |
| 2 | |
| 3 | import "testing" |
| 4 | |
| 5 | // The cost arm is deterministic, so it doubles as the regression guard for a |
| 6 | // growing session whose complete summary prefix still fits the model window. |
| 7 | func TestRepeatedFoldsStayBoundedAndKeepSucceeding(t *testing.T) { |
| 8 | const ( |
| 9 | window = 1_000_000 |
| 10 | gens = 6 |
| 11 | reserve = 8192 // summaryOutputReserve: the digest the call must still return |
| 12 | maxCalls = 1 // Harness-style folding issues one complete-prefix request |
| 13 | ) |
| 14 | res, err := runCost(gens, window, arms{}) |
| 15 | if err != nil { |
| 16 | t.Fatalf("runCost: %v", err) |
| 17 | } |
| 18 | if len(res) != gens { |
| 19 | t.Fatalf("generations = %d, want %d", len(res), gens) |
| 20 | } |
| 21 | for _, r := range res { |
| 22 | if r.Error != "" { |
| 23 | t.Errorf("gen %d failed to fold: %s", r.Gen+1, r.Error) |
| 24 | } |
| 25 | if r.LargestCall+reserve > window { |
| 26 | t.Errorf("gen %d largest summarizer call = %d tokens est.; with %d reserved for the digest that overflows the %d window", r.Gen+1, r.LargestCall, reserve, window) |
| 27 | } |
| 28 | if r.SummarizerCalls > maxCalls { |
| 29 | t.Errorf("gen %d cost %d summarizer calls, over the %d ceiling", r.Gen+1, r.SummarizerCalls, maxCalls) |
| 30 | } |
| 31 | if r.ProjectionTokens == 0 || r.ProjectionTokens >= r.CanonicalTokens { |
| 32 | t.Errorf("gen %d projection = %d tokens vs canonical %d; the fold saved nothing", r.Gen+1, r.ProjectionTokens, r.CanonicalTokens) |
| 33 | } |
| 34 | } |
| 35 | if last := res[len(res)-1]; last.CanonicalTokens <= res[0].CanonicalTokens { |
| 36 | t.Fatalf("canonical did not grow across generations: %d -> %d", res[0].CanonicalTokens, last.CanonicalTokens) |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | // Harness-style compaction never privately slices an oversized prefix or |
| 41 | // fabricates a digest. Admission failure is explicit and installs no summary. |
| 42 | func TestOversizedSummaryPrefixFailsWithoutPrivateShortening(t *testing.T) { |
| 43 | const window = 64_000 |
| 44 | res, err := runCost(1, window, arms{}) |
| 45 | if err != nil { |
| 46 | t.Fatalf("runCost: %v", err) |
| 47 | } |
| 48 | if len(res) != 1 { |
| 49 | t.Fatalf("generations = %d, want 1", len(res)) |
| 50 | } |
| 51 | r := res[0] |
| 52 | if r.Error == "" { |
| 53 | t.Fatal("oversized complete prefix unexpectedly succeeded") |
| 54 | } |
| 55 | if r.SummarizerCalls != 1 { |
| 56 | t.Fatalf("summarizer calls = %d, want one failed request", r.SummarizerCalls) |
| 57 | } |
| 58 | if r.LargestCall <= window { |
| 59 | t.Fatalf("largest call = %d, want an input over window %d", r.LargestCall, window) |
| 60 | } |
| 61 | if r.ProjectionTokens != 0 { |
| 62 | t.Fatalf("failed oversized fold installed a %d-token projection", r.ProjectionTokens) |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | // A probe must score the stale answer as lost even when the model hedges its |
| 67 | // way to mentioning the right one — "yes, but it has not been re-run since" |
| 68 | // is the exact shape a drifting digest produces. |
| 69 | func TestProbeScoringRejectsHedges(t *testing.T) { |
| 70 | var freshness probe |
| 71 | for _, p := range probeSuite() { |
| 72 | if p.class == "verification-freshness" { |
| 73 | freshness = p |
| 74 | } |
| 75 | } |
| 76 | if freshness.class == "" { |
| 77 | t.Fatal("verification-freshness probe missing from the suite") |
| 78 | } |
| 79 | for _, tc := range []struct { |
| 80 | answer string |
| 81 | want bool |
| 82 | }{ |
| 83 | {"No.", true}, |
| 84 | {"no — config/format.go changed after the last run", true}, |
| 85 | {"Yes", false}, |
| 86 | {"Yes, but it has not been re-run since the edit", false}, |
| 87 | {"I am not sure", false}, |
| 88 | } { |
| 89 | if got := freshness.score(tc.answer); got != tc.want { |
| 90 | t.Errorf("score(%q) = %v, want %v", tc.answer, got, tc.want) |
| 91 | } |
| 92 | } |
| 93 | } |
| 94 |