| 1 | package capdiag_test |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "net/http" |
| 6 | "net/http/httptest" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "strings" |
| 10 | "sync/atomic" |
| 11 | "testing" |
| 12 | |
| 13 | "reasonix/internal/capdiag" |
| 14 | "reasonix/internal/config" |
| 15 | "reasonix/internal/doctor" |
| 16 | ) |
| 17 | |
| 18 | func TestSkillDiagnosticsProbeHelper(t *testing.T) { |
| 19 | if marker := os.Getenv("REASONIX_SKILL_DIAGNOSTIC_PROBE"); marker != "" { |
| 20 | if err := os.WriteFile(marker, []byte("started"), 0600); err != nil { |
| 21 | t.Fatal(err) |
| 22 | } |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | func TestDoctorSkillReferenceParity(t *testing.T) { |
| 27 | for _, withMCP := range []bool{false, true} { |
| 28 | t.Run(fmt.Sprintf("mcp=%v", withMCP), func(t *testing.T) { |
| 29 | root, home := t.TempDir(), t.TempDir() |
| 30 | rh := filepath.Join(home, ".reasonix") |
| 31 | t.Setenv("HOME", home) |
| 32 | t.Setenv("USERPROFILE", home) |
| 33 | t.Setenv("REASONIX_HOME", rh) |
| 34 | t.Chdir(root) |
| 35 | custom, excluded := filepath.Join(root, "custom"), filepath.Join(root, "excluded") |
| 36 | var requests atomic.Int32 |
| 37 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 38 | requests.Add(1) |
| 39 | w.WriteHeader(http.StatusInternalServerError) |
| 40 | })) |
| 41 | defer server.Close() |
| 42 | cfgText := fmt.Sprintf(` |
| 43 | default_model = "diagnostic-test" |
| 44 | [[providers]] |
| 45 | name = "diagnostic-test" |
| 46 | kind = "openai" |
| 47 | model = "offline" |
| 48 | base_url = %q |
| 49 | api_key = "test-placeholder" |
| 50 | [skills] |
| 51 | paths = [%q, %q] |
| 52 | excluded_paths = [%q] |
| 53 | disabled_skills = ["disabled-example"] |
| 54 | `, server.URL, custom, excluded, excluded) |
| 55 | marker := filepath.Join(root, "mcp-started") |
| 56 | t.Setenv("REASONIX_SKILL_DIAGNOSTIC_PROBE", marker) |
| 57 | if withMCP { |
| 58 | exe, err := os.Executable() |
| 59 | if err != nil { |
| 60 | t.Fatal(err) |
| 61 | } |
| 62 | cfgText += fmt.Sprintf("\n[[plugins]]\nname = \"probe\"\ntype = \"stdio\"\ncommand = %q\nargs = [\"-test.run=^TestSkillDiagnosticsProbeHelper$\"]\nauto_start = true\n", exe) |
| 63 | } |
| 64 | write(t, filepath.Join(root, "reasonix.toml"), cfgText) |
| 65 | collect := func() ([]string, capdiag.Report) { |
| 66 | t.Helper() |
| 67 | cfg, err := config.LoadForRootReadOnly(root) |
| 68 | if err != nil { |
| 69 | t.Fatal(err) |
| 70 | } |
| 71 | ordinary := doctor.Collect(doctor.Options{Config: cfg}) |
| 72 | return ordinary.Warnings, capdiag.Collect(capdiag.Options{Root: root, HomeDir: home, ReasonixHomeDir: rh}) |
| 73 | } |
| 74 | warnings, report := collect() |
| 75 | for _, w := range warnings { |
| 76 | if strings.Contains(w, "allowed-tools") { |
| 77 | t.Fatal(w) |
| 78 | } |
| 79 | } |
| 80 | if report.Skills.Winners < 2 { |
| 81 | t.Fatal("built-in skills were not loaded") |
| 82 | } |
| 83 | for _, d := range report.Issues { |
| 84 | if strings.HasPrefix(d.Code, "skill.tool_reference_") { |
| 85 | t.Fatal(d) |
| 86 | } |
| 87 | } |
| 88 | writeSkill := func(base, name, ref string) { |
| 89 | t.Helper() |
| 90 | write(t, filepath.Join(base, name, "SKILL.md"), fmt.Sprintf("---\nname: %s\ndescription: Test skill\nallowed-tools: [%q]\n---\nTest.\n", name, ref)) |
| 91 | } |
| 92 | writeSkill(custom, "custom-example", "typo_read_file") |
| 93 | writeSkill(custom, "dynamic-example", "mcp__future__search") |
| 94 | writeSkill(custom, "disabled-example", "disabled_typo") |
| 95 | writeSkill(excluded, "excluded-example", "excluded_typo") |
| 96 | writeSkill(filepath.Join(rh, "skills"), "shadow-example", "shadowed_typo") |
| 97 | writeSkill(filepath.Join(root, ".reasonix", "skills"), "shadow-example", "use_capability") |
| 98 | warnings, report = collect() |
| 99 | joined := strings.Join(warnings, "\n") |
| 100 | matched := 0 |
| 101 | for _, d := range report.Issues { |
| 102 | if !strings.HasPrefix(d.Code, "skill.tool_reference_") { |
| 103 | continue |
| 104 | } |
| 105 | matched++ |
| 106 | if !strings.Contains(joined, d.Message) { |
| 107 | t.Fatalf("doctor missing capability finding: %+v\n%s", d, joined) |
| 108 | } |
| 109 | } |
| 110 | if matched != 2 { |
| 111 | t.Fatalf("got %d reference findings, want unknown and unverified: %+v", matched, report.Issues) |
| 112 | } |
| 113 | for _, bad := range []string{"disabled_typo", "excluded_typo", "shadowed_typo"} { |
| 114 | if strings.Contains(joined, bad) { |
| 115 | t.Fatalf("inactive skill warned: %s", joined) |
| 116 | } |
| 117 | } |
| 118 | if requests.Load() != 0 { |
| 119 | t.Fatal("diagnostics called the provider") |
| 120 | } |
| 121 | if _, err := os.Stat(marker); !os.IsNotExist(err) { |
| 122 | t.Fatalf("MCP process started: %v", err) |
| 123 | } |
| 124 | }) |
| 125 | } |
| 126 | } |
| 127 |