| 1 | package main |
| 2 | |
| 3 | import ( |
| 4 | "strings" |
| 5 | "testing" |
| 6 | ) |
| 7 | |
| 8 | func TestNormalizeAnchorArmRejectsUnknownArms(t *testing.T) { |
| 9 | for in, want := range map[string]string{"": anchorBlind, "BLIND": anchorBlind, "correct": anchorCorrect, " wrong ": anchorWrong} { |
| 10 | got, err := normalizeAnchorArm(in) |
| 11 | if err != nil || got != want { |
| 12 | t.Errorf("normalizeAnchorArm(%q) = %q, %v; want %q", in, got, err, want) |
| 13 | } |
| 14 | } |
| 15 | if _, err := normalizeAnchorArm("seeded"); err == nil { |
| 16 | t.Fatal("unknown anchor arm accepted") |
| 17 | } |
| 18 | } |
| 19 | |
| 20 | // A task with no authored seed must be unscorable in a seeded arm. Running it |
| 21 | // blind instead would put a control run in the seeded denominator, which is |
| 22 | // the one way an anchor-resistance figure can be quietly wrong. |
| 23 | func TestSeedForRefusesToRunAnUnseededTaskInASeededArm(t *testing.T) { |
| 24 | seeded := task{SeedCorrect: "It is the CRLF path.", SeedWrong: "It is the byte indexing."} |
| 25 | bare := task{} |
| 26 | |
| 27 | if seed, ok := seeded.seedFor(anchorWrong); !ok || seed != "It is the byte indexing." { |
| 28 | t.Errorf("seeded wrong arm = %q, %v", seed, ok) |
| 29 | } |
| 30 | if _, ok := bare.seedFor(anchorCorrect); ok { |
| 31 | t.Error("unseeded task accepted into the correct arm") |
| 32 | } |
| 33 | if _, ok := bare.seedFor(anchorWrong); ok { |
| 34 | t.Error("unseeded task accepted into the wrong arm") |
| 35 | } |
| 36 | // Blind is the control: every task belongs to it, seeded or not. |
| 37 | if seed, ok := bare.seedFor(anchorBlind); !ok || seed != "" { |
| 38 | t.Errorf("blind arm = %q, %v; want every task, unseeded", seed, ok) |
| 39 | } |
| 40 | if seed, ok := seeded.seedFor(anchorBlind); !ok || seed != "" { |
| 41 | t.Errorf("blind arm handed a seed: %q", seed) |
| 42 | } |
| 43 | } |
| 44 | |
| 45 | // The seed has to actually reach the prompt, and reach it in front: an |
| 46 | // experiment whose treatment never arrives would report the control's numbers |
| 47 | // under the treatment's name. |
| 48 | func TestAnchorPromptPutsTheSeedInFrontOfTheTask(t *testing.T) { |
| 49 | diagnose := task{ |
| 50 | Prompt: "Diagnose the failing test.", |
| 51 | SeedCorrect: "It is the CRLF path.", |
| 52 | SeedWrong: "It is the byte indexing.", |
| 53 | } |
| 54 | if got := anchorPrompt(anchorBlind, diagnose); got != diagnose.Prompt { |
| 55 | t.Errorf("blind arm altered the prompt: %q", got) |
| 56 | } |
| 57 | for arm, seed := range map[string]string{anchorCorrect: diagnose.SeedCorrect, anchorWrong: diagnose.SeedWrong} { |
| 58 | got := anchorPrompt(arm, diagnose) |
| 59 | if !strings.HasPrefix(got, seed) { |
| 60 | t.Errorf("%s arm did not lead with its seed: %q", arm, got) |
| 61 | } |
| 62 | if !strings.HasSuffix(got, diagnose.Prompt) { |
| 63 | t.Errorf("%s arm lost the task: %q", arm, got) |
| 64 | } |
| 65 | } |
| 66 | // An unseeded task is skipped, never quietly run as the control. |
| 67 | if got := anchorPrompt(anchorWrong, task{Prompt: "p"}); got != "p" { |
| 68 | t.Errorf("unseeded task was altered: %q", got) |
| 69 | } |
| 70 | } |
| 71 | |
| 72 | // The blind arm is the ordinary run and needs no section; a seeded arm must |
| 73 | // name itself, because its numbers are not comparable with the baseline's. |
| 74 | func TestRenderAnchorStaysSilentForTheControlArm(t *testing.T) { |
| 75 | got := renderAnchor([]result{{Passed: true, Anchor: anchorBlind}, {Anchor: anchorBlind}}) |
| 76 | if got != "" { |
| 77 | t.Fatalf("blind arm rendered a section: %q", got) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | // A seeded arm scoring fewer tasks than the baseline is a different corpus. |
| 82 | // The drop has to be stated, not left to be inferred from a smaller total. |
| 83 | func TestRenderAnchorNamesTheArmAndItsSkippedTasks(t *testing.T) { |
| 84 | got := renderAnchor([]result{ |
| 85 | {Passed: true, Anchor: anchorWrong}, |
| 86 | {Passed: false, Anchor: anchorWrong}, |
| 87 | {Anchor: anchorWrong, Skipped: true, Note: "skipped: no seed_wrong authored for this task"}, |
| 88 | }) |
| 89 | for _, want := range []string{"arm **wrong**", "solved 1/2", "**1 task(s) skipped**", "resistance ="} { |
| 90 | if !strings.Contains(got, want) { |
| 91 | t.Errorf("anchor section missing %q:\n%s", want, got) |
| 92 | } |
| 93 | } |
| 94 | } |
| 95 |