| 1 | // Run: tsx src/__tests__/provider-model-refresh.test.ts |
| 2 | |
| 3 | import { |
| 4 | apiKeyEnvFromProviderName, |
| 5 | createLatestRequestGate, |
| 6 | inferredVisionModels, |
| 7 | isLikelyChatModel, |
| 8 | isLikelyVisionModel, |
| 9 | mergedFetchedProviderModels, |
| 10 | mergeProviderModelContextWindows, |
| 11 | providerApiKeyEnvForSave, |
| 12 | providerDefaultModel, |
| 13 | providerIsConfigured, |
| 14 | providerModelCandidates, |
| 15 | providerModelContextWindowDrafts, |
| 16 | providerModelContextWindowIsSmall, |
| 17 | providerRequiresKey, |
| 18 | removeProviderAccessesForMock, |
| 19 | } from "../lib/providerModels"; |
| 20 | import type { ProviderView } from "../lib/types"; |
| 21 | |
| 22 | let passed = 0; |
| 23 | let failed = 0; |
| 24 | |
| 25 | function eq(a: unknown, b: unknown, label: string) { |
| 26 | if (JSON.stringify(a) === JSON.stringify(b)) { |
| 27 | process.stdout.write(` PASS ${label}\n`); |
| 28 | passed += 1; |
| 29 | } else { |
| 30 | process.stdout.write(` FAIL ${label}: expected ${JSON.stringify(b)}, got ${JSON.stringify(a)}\n`); |
| 31 | failed += 1; |
| 32 | } |
| 33 | } |
| 34 | |
| 35 | console.log("\nprovider model refresh"); |
| 36 | |
| 37 | const requestGate = createLatestRequestGate(); |
| 38 | const firstRefresh = requestGate.begin("builtin:deepseek"); |
| 39 | const secondRefresh = requestGate.begin("builtin:deepseek"); |
| 40 | eq( |
| 41 | [requestGate.isCurrent("builtin:deepseek", firstRefresh), requestGate.isCurrent("builtin:deepseek", secondRefresh)], |
| 42 | [false, true], |
| 43 | "newer grouped-provider refresh invalidates an older in-flight completion", |
| 44 | ); |
| 45 | requestGate.cancel("builtin:deepseek"); |
| 46 | eq( |
| 47 | requestGate.isCurrent("builtin:deepseek", secondRefresh), |
| 48 | false, |
| 49 | "provider edits invalidate an in-flight model refresh before it can publish stale state", |
| 50 | ); |
| 51 | |
| 52 | eq( |
| 53 | mergedFetchedProviderModels(["coding-pro"], ["coding-pro", "chat", "vision"]), |
| 54 | ["coding-pro", "chat", "vision"], |
| 55 | "appends discovered models without removing curated ones", |
| 56 | ); |
| 57 | |
| 58 | eq( |
| 59 | mergedFetchedProviderModels(["coding-pro"], ["coding-pro", "chat", "vision"], { preserveCurated: true }), |
| 60 | ["coding-pro"], |
| 61 | "background refresh preserves manually curated model list", |
| 62 | ); |
| 63 | |
| 64 | eq( |
| 65 | mergedFetchedProviderModels(["coding-pro"], ["chat", "vision"], { preserveCurated: true }), |
| 66 | ["coding-pro"], |
| 67 | "background refresh does not re-add deleted models", |
| 68 | ); |
| 69 | |
| 70 | eq( |
| 71 | mergedFetchedProviderModels(["mimo-v2.5-pro"], ["mimo-v2.5", "mimo-v2.5-pro"], { preserveCurated: true }), |
| 72 | ["mimo-v2.5-pro"], |
| 73 | "manual access refresh preserves selected MiMo model instead of importing provider catalog", |
| 74 | ); |
| 75 | |
| 76 | eq( |
| 77 | providerModelCandidates(["mimo-v2.5-pro"], ["mimo-v2.5", "mimo-v2.5-pro"]), |
| 78 | ["mimo-v2.5-pro", "mimo-v2.5"], |
| 79 | "manual access refresh can show provider catalog as unsaved candidates", |
| 80 | ); |
| 81 | |
| 82 | eq( |
| 83 | providerModelCandidates(["mimo-v2.5-pro"], ["mimo-v2.5-asr", "mimo-v2.5-tts", "mimo-v2.5", "mimo-v2.5-pro"]), |
| 84 | ["mimo-v2.5-pro", "mimo-v2.5"], |
| 85 | "manual access refresh filters non-chat candidates before saving", |
| 86 | ); |
| 87 | |
| 88 | eq( |
| 89 | [ |
| 90 | isLikelyChatModel("mimo-v2.5-pro"), |
| 91 | isLikelyChatModel("mimo-v2.5-asr"), |
| 92 | isLikelyChatModel("mimo-v2.5-tts"), |
| 93 | isLikelyChatModel("text-embedding-3-small"), |
| 94 | ], |
| 95 | [true, false, false, false], |
| 96 | "matches backend non-chat model heuristic", |
| 97 | ); |
| 98 | |
| 99 | eq( |
| 100 | [ |
| 101 | isLikelyVisionModel("gpt-4o"), |
| 102 | isLikelyVisionModel("gpt-4o-audio-preview"), |
| 103 | isLikelyVisionModel("gpt-4o-mini-audio-preview"), |
| 104 | isLikelyVisionModel("mimo-v2.5"), |
| 105 | isLikelyVisionModel("mimo-v2-omni"), |
| 106 | isLikelyVisionModel("qwen2.5-vl-72b-instruct"), |
| 107 | isLikelyVisionModel("mimo-v2.5-asr"), |
| 108 | ], |
| 109 | [true, false, false, true, true, true, false], |
| 110 | "detects likely image-capable chat model IDs", |
| 111 | ); |
| 112 | |
| 113 | eq( |
| 114 | inferredVisionModels([ |
| 115 | "mimo-v2.5-pro", |
| 116 | "mimo-v2.5", |
| 117 | "mimo-v2-omni", |
| 118 | "qwen-vl-plus", |
| 119 | "mimo-v2.5-asr", |
| 120 | "audio-omni-tts", |
| 121 | "gpt-4o-audio-preview", |
| 122 | "gpt-4o-mini-audio-preview", |
| 123 | ]), |
| 124 | ["mimo-v2.5", "mimo-v2-omni", "qwen-vl-plus"], |
| 125 | "infers image-capable models without importing audio models", |
| 126 | ); |
| 127 | |
| 128 | eq( |
| 129 | mergedFetchedProviderModels([], ["coding-pro", "chat"], { preserveCurated: true }), |
| 130 | ["coding-pro", "chat"], |
| 131 | "background refresh can populate an empty model list", |
| 132 | ); |
| 133 | |
| 134 | eq( |
| 135 | providerDefaultModel("coding-pro", ["coding-pro", "chat"]), |
| 136 | "coding-pro", |
| 137 | "preserves current default when it remains available", |
| 138 | ); |
| 139 | |
| 140 | eq( |
| 141 | providerDefaultModel("deleted", ["coding-pro", "chat"]), |
| 142 | "coding-pro", |
| 143 | "falls back to first saved model when default is unavailable", |
| 144 | ); |
| 145 | |
| 146 | const contextOverrides = [ |
| 147 | { |
| 148 | model: "short-model", |
| 149 | reasoningProtocol: "none", |
| 150 | supportedEfforts: [], |
| 151 | defaultEffort: "", |
| 152 | vision: null, |
| 153 | contextWindow: 32768, |
| 154 | maxOutputTokens: -1, |
| 155 | }, |
| 156 | { |
| 157 | model: "long-model", |
| 158 | reasoningProtocol: "", |
| 159 | supportedEfforts: [], |
| 160 | defaultEffort: "", |
| 161 | vision: null, |
| 162 | contextWindow: 1000000, |
| 163 | }, |
| 164 | ]; |
| 165 | |
| 166 | eq( |
| 167 | providerModelContextWindowDrafts(contextOverrides), |
| 168 | { "short-model": "32768", "long-model": "1000000" }, |
| 169 | "loads positive per-model context windows into editable drafts", |
| 170 | ); |
| 171 | |
| 172 | eq( |
| 173 | mergeProviderModelContextWindows(contextOverrides, ["short-model", "new-model"], { |
| 174 | "short-model": "65536", |
| 175 | "new-model": "131072", |
| 176 | }), |
| 177 | [ |
| 178 | { ...contextOverrides[0], contextWindow: 65536 }, |
| 179 | { |
| 180 | model: "new-model", |
| 181 | reasoningProtocol: "", |
| 182 | supportedEfforts: [], |
| 183 | defaultEffort: "", |
| 184 | vision: null, |
| 185 | contextWindow: 131072, |
| 186 | }, |
| 187 | ], |
| 188 | "merges model context edits while preserving other overrides and dropping removed models", |
| 189 | ); |
| 190 | |
| 191 | eq( |
| 192 | mergeProviderModelContextWindows([], ["invalid-model"], { "invalid-model": "Infinity" }), |
| 193 | [], |
| 194 | "drops non-finite model context values before crossing the desktop bridge", |
| 195 | ); |
| 196 | |
| 197 | eq( |
| 198 | mergeProviderModelContextWindows([ |
| 199 | { |
| 200 | model: "output-only", |
| 201 | reasoningProtocol: "", |
| 202 | supportedEfforts: [], |
| 203 | defaultEffort: "", |
| 204 | vision: null, |
| 205 | maxOutputTokens: 4096, |
| 206 | }, |
| 207 | ], ["output-only"], { "output-only": "" }), |
| 208 | [{ |
| 209 | model: "output-only", |
| 210 | reasoningProtocol: "", |
| 211 | supportedEfforts: [], |
| 212 | defaultEffort: "", |
| 213 | vision: null, |
| 214 | contextWindow: 0, |
| 215 | maxOutputTokens: 4096, |
| 216 | }], |
| 217 | "keeps output-only model overrides while editing context windows", |
| 218 | ); |
| 219 | |
| 220 | eq( |
| 221 | [ |
| 222 | providerModelContextWindowIsSmall("8192"), |
| 223 | providerModelContextWindowIsSmall("16384"), |
| 224 | providerModelContextWindowIsSmall(""), |
| 225 | providerModelContextWindowIsSmall("Infinity"), |
| 226 | ], |
| 227 | [true, false, false, false], |
| 228 | "warns only for positive context windows below 16384 tokens", |
| 229 | ); |
| 230 | |
| 231 | eq( |
| 232 | providerApiKeyEnvForSave("Local Gateway", "", ""), |
| 233 | "", |
| 234 | "keeps custom provider keyless when no key env or key value is supplied", |
| 235 | ); |
| 236 | |
| 237 | eq( |
| 238 | providerApiKeyEnvForSave("Local Gateway", "", "sk-test"), |
| 239 | "LOCAL_GATEWAY_API_KEY", |
| 240 | "creates a key env when saving an inline key for a new custom provider", |
| 241 | ); |
| 242 | |
| 243 | eq( |
| 244 | providerApiKeyEnvForSave("9router", "", "sk-test"), |
| 245 | "CUSTOM_9ROUTER_API_KEY", |
| 246 | "creates a valid key env for a digit-leading custom provider", |
| 247 | ); |
| 248 | |
| 249 | eq( |
| 250 | providerApiKeyEnvForSave("Local Gateway", "GATEWAY_KEY", ""), |
| 251 | "GATEWAY_KEY", |
| 252 | "preserves an explicitly configured key env", |
| 253 | ); |
| 254 | |
| 255 | eq( |
| 256 | [ |
| 257 | apiKeyEnvFromProviderName("9router"), |
| 258 | apiKeyEnvFromProviderName("商汤"), |
| 259 | apiKeyEnvFromProviderName("通义千问"), |
| 260 | ], |
| 261 | ["CUSTOM_9ROUTER_API_KEY", "CUSTOM_d39b9067_API_KEY", "CUSTOM_e995c4c9_API_KEY"], |
| 262 | "generates valid stable key envs for digit-leading and non-ASCII provider names", |
| 263 | ); |
| 264 | |
| 265 | eq( |
| 266 | providerApiKeyEnvForSave("商汤", "CUSTOM_API_KEY", "sk-test"), |
| 267 | "CUSTOM_API_KEY", |
| 268 | "preserves an explicitly configured legacy custom key env", |
| 269 | ); |
| 270 | |
| 271 | eq( |
| 272 | [ |
| 273 | providerRequiresKey({ apiKeyEnv: "" }), |
| 274 | providerIsConfigured({ apiKeyEnv: "", keySet: false }), |
| 275 | providerIsConfigured({ apiKeyEnv: "LOCAL_API_KEY", keySet: false, requiresKey: false }), |
| 276 | providerIsConfigured({ apiKeyEnv: "REMOTE_API_KEY", keySet: false, requiresKey: true }), |
| 277 | providerIsConfigured({ apiKeyEnv: "REMOTE_API_KEY", keySet: true, requiresKey: true }), |
| 278 | ], |
| 279 | [false, true, true, false, true], |
| 280 | "separates provider selectability from key presence for no-auth providers", |
| 281 | ); |
| 282 | |
| 283 | const mockProviders = [ |
| 284 | { name: "deepseek", builtIn: true, added: true }, |
| 285 | { name: "custom", builtIn: false, added: true }, |
| 286 | ] as ProviderView[]; |
| 287 | const removedMockProviders = removeProviderAccessesForMock(mockProviders, ["deepseek", "custom"]); |
| 288 | eq( |
| 289 | removedMockProviders.map(({ name, added }) => ({ name, added })), |
| 290 | [{ name: "deepseek", added: false }], |
| 291 | "mock access removal hides built-ins and deletes custom providers", |
| 292 | ); |
| 293 | eq( |
| 294 | mockProviders.map(({ name, added }) => ({ name, added })), |
| 295 | [{ name: "deepseek", added: true }, { name: "custom", added: true }], |
| 296 | "mock access removal does not mutate the previous settings snapshot", |
| 297 | ); |
| 298 | |
| 299 | console.log(`\n${passed} passed, ${failed} failed, ${passed + failed} total`); |
| 300 | if (failed > 0) process.exit(1); |
| 301 |