返回 CodeWhale
telemetry_contract.rs
根目录 / crates / tui / tests / telemetry_contract.rs
1 //! Process-level telemetry contract.
2 //!
3 //! Everything here drives the real `codewhale-tui` binary inside a sealed
4 //! `HOME`/`CODEWHALE_HOME`, with a loopback recorder standing in for the
5 //! telemetry endpoint. The unit tests in `codewhale-telemetry` prove the
6 //! predicate; these prove that the *emitting process* consults it — which is
7 //! the thing v1 of this design got wrong, because `resolve_runtime_options`
8 //! had no non-test caller and so neither `telemetry = false` in the config file
9 //! nor `CODEWHALE_TELEMETRY=0` was ever read by a process that would have sent.
10 //!
11 //! Two disciplines make the zero-request assertions non-vacuous:
12 //!
13 //! 1. `enabled_and_accepted_posts_exactly_one_batch` proves the client works.
14 //! Without it every "sends zero requests" test would also pass against a
15 //! client that never sends anything at all.
16 //! 2. Every off-test records the notice decision *and* points at a live
17 //! recorder, so it cannot pass through the consent gate by accident.
18 //!
19 //! The recorder is `http://127.0.0.1:<port>` — loopback, which is the one place
20 //! `validate_endpoint` permits plaintext, and a packet that never leaves the
21 //! machine.
22
23 #![cfg(unix)]
24
25 use std::io::Read;
26 use std::path::{Path, PathBuf};
27 use std::process::{Command, Output, Stdio};
28 use std::time::{Duration, Instant};
29
30 use codewhale_config::{SetupState, TELEMETRY_NOTICE_VERSION};
31 use serde_json::{Value, json};
32 use tempfile::TempDir;
33 use wait_timeout::ChildExt;
34 use wiremock::matchers::{method, path as path_matcher};
35 use wiremock::{Mock, MockServer, ResponseTemplate};
36
37 /// Where the recorder listens for batches.
38 const TELEMETRY_PATH: &str = "/v1/telemetry";
39 /// Where the mock model listens.
40 const MODEL_PATH: &str = "/v1/chat/completions";
41 const TEST_MODEL: &str = "telemetry-contract-model";
42
43 /// Sentinels planted through real inputs. None of these may appear in a batch.
44 ///
45 /// Deliberately low-entropy: `crates/tui/src/fleet/ledger.rs` notes that
46 /// realistic-looking tokens trip secret scanning at push time.
47 const SENTINEL_PROMPT: &str = "tc-prompt-sentinel-do-not-collect";
48 const SENTINEL_FILENAME: &str = "tc-workspace-sentinel-file.txt";
49 const SENTINEL_PROVIDER_TABLE: &str = "tc_custom_provider_sentinel";
50 const SENTINEL_MCP_SERVER: &str = "tc-mcp-server-sentinel";
51 const SENTINEL_API_KEY: &str = "tc-api-key-sentinel-not-a-real-key";
52 /// Planted by writing to `buffer.jsonl` directly, which is what any other
53 /// process running as this user can do.
54 const SENTINEL_INJECTED: &str = "tc-injected-sentinel-/Users/victim/secret-repo";
55 /// The key lives only in the child's environment, never in a file, so the
56 /// "absent from every written file" assertion means something.
57 const SENTINEL_API_KEY_ENV: &str = "TC_SENTINEL_API_KEY";
58
59 const EXEC_TIMEOUT: Duration = Duration::from_secs(90);
60
61 // ── Fixture ──────────────────────────────────────────────────────────────
62
63 struct Fixture {
64 _root: TempDir,
65 home: PathBuf,
66 codewhale_home: PathBuf,
67 workspace: PathBuf,
68 config_path: PathBuf,
69 endpoint: Option<String>,
70 }
71
72 impl Fixture {
73 fn new() -> Self {
74 let root = TempDir::new().expect("fixture root");
75 let home = root.path().join("home");
76 let codewhale_home = root.path().join("codewhale-home");
77 let workspace = root.path().join("workspace");
78 for dir in [&home, &codewhale_home, &workspace] {
79 std::fs::create_dir_all(dir).expect("create fixture dir");
80 }
81 let config_path = root.path().join("config.toml");
82 std::fs::write(&config_path, "").expect("write config");
83 Self {
84 _root: root,
85 home,
86 codewhale_home,
87 workspace,
88 config_path,
89 endpoint: None,
90 }
91 }
92
93 /// Point this fixture at a loopback recorder.
94 fn with_endpoint(mut self, base_url: &str) -> Self {
95 self.endpoint = Some(format!("{base_url}{TELEMETRY_PATH}"));
96 self
97 }
98
99 fn write_config(&self, body: &str) {
100 std::fs::write(&self.config_path, body).expect("write config");
101 }
102
103 /// Record the answer a user would have given on a TTY.
104 ///
105 /// Machine-scoped consent: the notice is only ever *rendered* on a
106 /// terminal, but the decision it records lives on this `CODEWHALE_HOME` and
107 /// authorizes later non-TTY runs against the same home.
108 fn record_notice(&self, opt_in: bool) {
109 let mut state = SetupState::default();
110 state.record_telemetry_notice(TELEMETRY_NOTICE_VERSION, opt_in);
111 state
112 .save_to(&self.codewhale_home.join("setup_state.json"))
113 .expect("write setup state");
114 }
115
116 fn setup_state_path(&self) -> PathBuf {
117 self.codewhale_home.join("setup_state.json")
118 }
119
120 fn telemetry_root(&self) -> PathBuf {
121 self.codewhale_home.join("telemetry")
122 }
123
124 fn command(&self) -> Command {
125 let mut command = Command::new(codewhale_tui_binary());
126 command
127 .current_dir(&self.workspace)
128 .env_clear()
129 .env("PATH", std::env::var_os("PATH").expect("PATH"))
130 .env("HOME", &self.home)
131 .env("USERPROFILE", &self.home)
132 .env("XDG_CONFIG_HOME", self.home.join(".config"))
133 .env("XDG_DATA_HOME", self.home.join(".local").join("share"))
134 .env("XDG_CACHE_HOME", self.home.join(".cache"))
135 .env("CODEWHALE_HOME", &self.codewhale_home)
136 .env("CODEWHALE_SECRET_BACKEND", "file")
137 .env("CODEWHALE_MEMORY", "false")
138 // A pinned mirror version keeps the release crate from issuing a
139 // metadata request, so the only egress a test can observe is the
140 // one this file is about.
141 .env(
142 "CODEWHALE_RELEASE_BASE_URL",
143 "https://example.invalid/releases",
144 )
145 .env("DEEPSEEK_TUI_VERSION", env!("CARGO_PKG_VERSION"))
146 .env("RUST_LOG", "warn")
147 .stdin(Stdio::null());
148 if let Some(endpoint) = &self.endpoint {
149 command.env("CODEWHALE_TELEMETRY_ENDPOINT", endpoint);
150 }
151 command
152 }
153
154 /// The cheapest subcommand that still traverses the whole telemetry
155 /// lifecycle: arm, `session_start`, dispatch, `session_end`, shutdown
156 /// flush. `completions` loads no config of its own and touches no network,
157 /// so anything the recorder sees came from telemetry.
158 fn run_completions(&self) -> Output {
159 let mut command = self.command();
160 command.args([
161 "--config",
162 self.config_path.to_str().expect("config path"),
163 "completions",
164 "bash",
165 ]);
166 let output = command.output().expect("run codewhale-tui completions");
167 assert!(
168 output.status.success(),
169 "completions failed\nstdout:\n{}\nstderr:\n{}",
170 String::from_utf8_lossy(&output.stdout),
171 String::from_utf8_lossy(&output.stderr)
172 );
173 output
174 }
175
176 /// Every regular file under the sealed roots, for leak scanning.
177 fn written_files(&self) -> Vec<PathBuf> {
178 let mut out = Vec::new();
179 for base in [&self.home, &self.codewhale_home, &self.workspace] {
180 collect_files(base, &mut out);
181 }
182 out.push(self.config_path.clone());
183 out
184 }
185 }
186
187 fn collect_files(dir: &Path, out: &mut Vec<PathBuf>) {
188 let Ok(entries) = std::fs::read_dir(dir) else {
189 return;
190 };
191 for entry in entries.flatten() {
192 let path = entry.path();
193 match entry.file_type() {
194 Ok(kind) if kind.is_dir() => collect_files(&path, out),
195 Ok(kind) if kind.is_file() => out.push(path),
196 _ => {}
197 }
198 }
199 }
200
201 fn codewhale_tui_binary() -> PathBuf {
202 if let Some(path) = option_env!("CARGO_BIN_EXE_codewhale-tui") {
203 return PathBuf::from(path);
204 }
205 if let Ok(path) = std::env::var("CARGO_BIN_EXE_codewhale-tui") {
206 return PathBuf::from(path);
207 }
208 let mut path = std::env::current_exe().expect("current test executable path");
209 path.pop();
210 if path.ends_with("deps") {
211 path.pop();
212 }
213 path.push(format!("codewhale-tui{}", std::env::consts::EXE_SUFFIX));
214 path
215 }
216
217 // ── Recorder ─────────────────────────────────────────────────────────────
218
219 /// A loopback endpoint that accepts every batch and keeps the body.
220 ///
221 /// The body is the point. The recorder `crates/tui/tests/diagnostic_read_only.rs`
222 /// copies deliberately drops it; a telemetry contract that cannot read what was
223 /// sent can only assert "something happened".
224 async fn start_recorder() -> MockServer {
225 let server = MockServer::start().await;
226 Mock::given(method("POST"))
227 .and(path_matcher(TELEMETRY_PATH))
228 .respond_with(ResponseTemplate::new(200))
229 .mount(&server)
230 .await;
231 server
232 }
233
234 /// Every request the recorder saw, batch bodies included.
235 async fn recorded_batches(server: &MockServer) -> Vec<Value> {
236 let requests = server
237 .received_requests()
238 .await
239 .expect("the recorder must retain its request log");
240 requests
241 .iter()
242 .filter(|request| request.url.path() == TELEMETRY_PATH)
243 .map(|request| {
244 serde_json::from_slice::<Value>(&request.body).unwrap_or_else(|error| {
245 panic!(
246 "a telemetry batch must be JSON: {error}\nbody: {}",
247 String::from_utf8_lossy(&request.body)
248 )
249 })
250 })
251 .collect()
252 }
253
254 /// How many chat completions the mock model served.
255 ///
256 /// The sentinel test's whole claim is that a prompt which *did* reach a model
257 /// did not reach a batch, so a run where the turn never happened would be
258 /// vacuous.
259 async fn model_request_count(server: &MockServer) -> usize {
260 server
261 .received_requests()
262 .await
263 .expect("the recorder must retain its request log")
264 .iter()
265 .filter(|request| request.url.path() == MODEL_PATH)
266 .count()
267 }
268
269 async fn assert_no_batches(server: &MockServer, why: &str) {
270 let batches = recorded_batches(server).await;
271 assert!(
272 batches.is_empty(),
273 "{why}: expected zero telemetry requests, recorded {}:\n{}",
274 batches.len(),
275 serde_json::to_string_pretty(&batches).unwrap_or_default()
276 );
277 }
278
279 // ── The client works ─────────────────────────────────────────────────────
280
281 /// Without this, every zero-request test below would also pass against a
282 /// client that never sends anything.
283 #[tokio::test(flavor = "current_thread")]
284 async fn enabled_and_accepted_posts_exactly_one_batch() {
285 let server = start_recorder().await;
286 let fixture = Fixture::new().with_endpoint(&server.uri());
287 fixture.write_config("telemetry = true\n");
288 fixture.record_notice(true);
289
290 fixture.run_completions();
291
292 let batches = recorded_batches(&server).await;
293 assert_eq!(batches.len(), 1, "one session is one batch");
294 let batch = &batches[0];
295 assert_eq!(batch["schema_version"], 1);
296 assert_eq!(batch["surface"], "cli");
297 assert!(
298 batch["install_id"]
299 .as_str()
300 .is_some_and(|id| id.len() == 36),
301 "install_id must be a UUID, got {:?}",
302 batch["install_id"]
303 );
304 let events = batch["events"].as_array().expect("events array");
305 assert!(
306 events.iter().any(|event| event["event"] == "session_start"),
307 "a batch must carry the session it describes: {batch}"
308 );
309 assert!(
310 events.iter().any(|event| event["event"] == "session_end"),
311 "the shutdown flush must carry session_end: {batch}"
312 );
313 }
314
315 // ── Off is real ──────────────────────────────────────────────────────────
316
317 /// The only test that proves the emitting process reads the config *file*.
318 ///
319 /// No environment variable is set here on purpose. `CODEWHALE_TELEMETRY=0` and
320 /// the config key travel different paths, and v1 of this design shipped a
321 /// kill switch that only the env half ever reached.
322 #[tokio::test(flavor = "current_thread")]
323 async fn config_file_only_opt_out_sends_zero_requests() {
324 let server = start_recorder().await;
325 let fixture = Fixture::new().with_endpoint(&server.uri());
326 fixture.write_config("telemetry = false\n");
327 fixture.record_notice(true);
328
329 let output = fixture.run_completions();
330 assert!(output.status.success());
331
332 assert_no_batches(&server, "`telemetry = false` in the config file").await;
333 }
334
335 #[tokio::test(flavor = "current_thread")]
336 async fn telemetry_disabled_by_env_sends_zero_requests() {
337 let server = start_recorder().await;
338 let fixture = Fixture::new().with_endpoint(&server.uri());
339 fixture.write_config("telemetry = true\n");
340 fixture.record_notice(true);
341
342 let mut command = fixture.command();
343 command
344 .env("CODEWHALE_TELEMETRY", "0")
345 .args([
346 "--config",
347 fixture.config_path.to_str().expect("config path"),
348 "completions",
349 "bash",
350 ])
351 .output()
352 .expect("run codewhale-tui completions");
353
354 assert_no_batches(&server, "`CODEWHALE_TELEMETRY=0`").await;
355 }
356
357 /// An unparseable env value fails **closed**, rather than falling through to
358 /// the config file's `true`.
359 #[tokio::test(flavor = "current_thread")]
360 async fn an_unparseable_telemetry_env_value_sends_zero_requests() {
361 let server = start_recorder().await;
362 let fixture = Fixture::new().with_endpoint(&server.uri());
363 fixture.write_config("telemetry = true\n");
364 fixture.record_notice(true);
365
366 fixture
367 .command()
368 .env("CODEWHALE_TELEMETRY", "maybe")
369 .args([
370 "--config",
371 fixture.config_path.to_str().expect("config path"),
372 "completions",
373 "bash",
374 ])
375 .output()
376 .expect("run codewhale-tui completions");
377
378 assert_no_batches(&server, "`CODEWHALE_TELEMETRY=maybe`").await;
379 }
380
381 /// The notice record is an independent AND condition. A pre-existing
382 /// `telemetry = true` is not consent: the key has been settable and inert for
383 /// a long time, so anyone who set it set a no-op.
384 #[tokio::test(flavor = "current_thread")]
385 async fn telemetry_enabled_without_notice_sends_zero_requests() {
386 let server = start_recorder().await;
387 let fixture = Fixture::new().with_endpoint(&server.uri());
388 fixture.write_config("telemetry = true\n");
389 // Deliberately no `record_notice`.
390
391 fixture.run_completions();
392
393 assert_no_batches(&server, "`telemetry = true` with no notice decision").await;
394 assert!(
395 !fixture.telemetry_root().exists(),
396 "a run that was never permitted to collect must not create {}",
397 fixture.telemetry_root().display()
398 );
399 }
400
401 /// A decision recorded against a *different* notice version is stale: the
402 /// content changed, so the answer is owed again and nothing is collected in
403 /// the meantime.
404 #[tokio::test(flavor = "current_thread")]
405 async fn a_stale_notice_version_sends_zero_requests() {
406 let server = start_recorder().await;
407 let fixture = Fixture::new().with_endpoint(&server.uri());
408 fixture.write_config("telemetry = true\n");
409 let mut state = SetupState::default();
410 state.record_telemetry_notice("0", true);
411 state
412 .save_to(&fixture.setup_state_path())
413 .expect("write setup state");
414
415 fixture.run_completions();
416
417 assert_no_batches(&server, "a decision recorded for an older notice version").await;
418 }
419
420 // ── Nothing survives a disable ───────────────────────────────────────────
421
422 /// A human's "off" wipes: tombstone first, data truncated, lock file left in
423 /// place, identity removed.
424 #[tokio::test(flavor = "current_thread")]
425 async fn disabling_after_buffering_wipes_and_sends_nothing() {
426 let server = start_recorder().await;
427 let fixture = Fixture::new().with_endpoint(&server.uri());
428 let root = fixture.telemetry_root();
429 seed_consenting_home(&root);
430 fixture.write_config("telemetry = false\n");
431 fixture.record_notice(true);
432
433 fixture.run_completions();
434
435 assert_no_batches(&server, "an explicit opt-out with a populated buffer").await;
436 assert!(
437 root.join("disabled").exists(),
438 "the tombstone is written first and never removed"
439 );
440 assert_eq!(
441 std::fs::read(root.join("buffer.jsonl")).expect("buffer survives as an empty file"),
442 Vec::<u8>::new(),
443 "buffered events must be truncated, not sent"
444 );
445 assert!(
446 root.join("buffer.jsonl.lock").exists(),
447 "the lock file is never unlinked: replacing it would leave appenders \
448 and compactors holding different inodes"
449 );
450 assert!(
451 !root.join("install_id.json").exists(),
452 "the install identity must not survive an opt-out"
453 );
454 }
455
456 /// "Telemetry resolved to false" is the *default* state of every installation.
457 /// A wipe keyed on it would delete a consenting user's identity and unflushed
458 /// buffer every time they ran one command on a fresh machine profile.
459 #[tokio::test(flavor = "current_thread")]
460 async fn forced_off_run_preserves_a_consenting_users_state() {
461 let server = start_recorder().await;
462 let fixture = Fixture::new().with_endpoint(&server.uri());
463 let root = fixture.telemetry_root();
464 seed_consenting_home(&root);
465 let before = snapshot(&root);
466 // No `telemetry` key at all, and no notice decision: forced off, not an
467 // answer.
468 fixture.write_config("");
469
470 fixture.run_completions();
471
472 assert_no_batches(&server, "a forced-off run").await;
473 assert_eq!(
474 snapshot(&root),
475 before,
476 "a forced-off run must leave a consenting user's telemetry state byte-identical"
477 );
478 }
479
480 /// The documented one-command kill switch stops collection and destroys
481 /// nothing.
482 ///
483 /// `CODEWHALE_TELEMETRY=0` used to resolve as an *answer*, so it took the
484 /// destructive opt-out branch: an agent harness that set it for one command
485 /// deleted the install id and truncated the dry-run records of the person who
486 /// owns the machine, and the "permanent" tombstone it left was cleared by the
487 /// user's very next ordinary run. Off for the run, and only for the run.
488 #[tokio::test(flavor = "current_thread")]
489 async fn a_run_scoped_kill_switch_preserves_a_consenting_users_state() {
490 let server = start_recorder().await;
491 let fixture = Fixture::new().with_endpoint(&server.uri());
492 let root = fixture.telemetry_root();
493 seed_consenting_home(&root);
494 let before = snapshot(&root);
495 fixture.write_config("telemetry = true\n");
496 fixture.record_notice(true);
497
498 for value in ["0", "off", "false"] {
499 fixture
500 .command()
501 .env("CODEWHALE_TELEMETRY", value)
502 .args([
503 "--config",
504 fixture.config_path.to_str().expect("config path"),
505 "completions",
506 "bash",
507 ])
508 .output()
509 .expect("run codewhale-tui completions");
510
511 assert_no_batches(&server, "a run-scoped kill switch").await;
512 assert!(
513 !root.join("disabled").exists(),
514 "`CODEWHALE_TELEMETRY={value}` tombstoned a machine nobody opted out"
515 );
516 assert_eq!(
517 snapshot(&root),
518 before,
519 "`CODEWHALE_TELEMETRY={value}` touched a consenting user's telemetry state"
520 );
521 }
522
523 // And the persistent switch still is the destructive one, on the same
524 // home, so the two are not merely both no-ops here.
525 fixture.write_config("telemetry = false\n");
526 fixture.run_completions();
527 assert!(
528 root.join("disabled").exists(),
529 "the config-file opt-out must still wipe and tombstone"
530 );
531 assert!(!root.join("install_id.json").exists());
532 }
533
534 /// A run that was never permitted to collect creates no directory at all —
535 /// which is also what makes the process panic hook, installed before the
536 /// command line is even parsed, write nothing for a disabled user.
537 #[tokio::test(flavor = "current_thread")]
538 async fn a_disabled_run_creates_no_telemetry_directory() {
539 let server = start_recorder().await;
540 let fixture = Fixture::new().with_endpoint(&server.uri());
541 fixture.write_config("telemetry = false\n");
542 fixture.record_notice(false);
543
544 fixture.run_completions();
545
546 assert_no_batches(&server, "a declined run").await;
547 assert!(
548 !fixture.telemetry_root().exists(),
549 "nothing may be created for a user who declined on a fresh home"
550 );
551 }
552
553 // ── The notice is never answered by silence ──────────────────────────────
554
555 /// Deferral is not a decision. `--skip-onboarding` records nothing, prints
556 /// nothing, and sends nothing — unlike the constitution checkpoint, which does
557 /// persist a `Deferred` completion.
558 #[tokio::test(flavor = "current_thread")]
559 async fn skip_onboarding_writes_no_telemetry_decision() {
560 let server = start_recorder().await;
561 let fixture = Fixture::new().with_endpoint(&server.uri());
562 fixture.write_config("telemetry = true\n");
563
564 let mut command = fixture.command();
565 let output = command
566 .args([
567 "--config",
568 fixture.config_path.to_str().expect("config path"),
569 "--skip-onboarding",
570 "completions",
571 "bash",
572 ])
573 .output()
574 .expect("run codewhale-tui completions");
575 assert!(output.status.success());
576
577 let stdout = String::from_utf8_lossy(&output.stdout);
578 let stderr = String::from_utf8_lossy(&output.stderr);
579 for stream in [&stdout, &stderr] {
580 assert!(
581 !stream.contains("keep telemetry off"),
582 "the notice must not be rendered on a path that cannot answer it"
583 );
584 }
585
586 if let Some(state) = SetupState::load_from(&fixture.setup_state_path()) {
587 assert_eq!(
588 state.telemetry_notice_decided_for, None,
589 "skip-onboarding must leave the telemetry decision unset"
590 );
591 }
592 assert_no_batches(&server, "`--skip-onboarding`").await;
593 }
594
595 // ── Payload red lines, through a real turn ───────────────────────────────
596
597 /// Five sentinel classes planted through real inputs — the prompt, a workspace
598 /// filename, a custom `[providers.<name>]` table key, an MCP server name, and
599 /// the API key — asserted absent from every recorded batch.
600 ///
601 /// The API key is held to the stricter standard the harness at
602 /// `crates/tui/tests/verifiers_harness_contract.rs` applies: absent from stdout,
603 /// stderr, and every file under the sealed roots as well. The other four
604 /// legitimately appear in files the product owns — a prompt is in the session
605 /// transcript, a provider table key is in the config the user wrote — so the
606 /// claim about them is precisely that they never reach a *batch*.
607 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
608 async fn batch_contains_no_planted_sentinel() {
609 let server = start_recorder().await;
610 mount_model(&server, Duration::ZERO).await;
611 let fixture = Fixture::new().with_endpoint(&server.uri());
612 plant_sentinels(&fixture, &server.uri());
613
614 let output = run_exec(&fixture, SENTINEL_PROMPT);
615
616 assert!(
617 model_request_count(&server).await > 0,
618 "the sentinel prompt must actually have reached a model, or this test \
619 proves nothing\nstdout:\n{}\nstderr:\n{}",
620 String::from_utf8_lossy(&output.stdout),
621 String::from_utf8_lossy(&output.stderr)
622 );
623 let batches = recorded_batches(&server).await;
624 assert!(
625 !batches.is_empty(),
626 "this test is only meaningful against a batch that was actually sent"
627 );
628 let serialized = serde_json::to_string(&batches).expect("serialize batches");
629 for sentinel in [
630 SENTINEL_PROMPT,
631 SENTINEL_FILENAME,
632 SENTINEL_PROVIDER_TABLE,
633 SENTINEL_MCP_SERVER,
634 SENTINEL_API_KEY,
635 ] {
636 assert!(
637 !serialized.contains(sentinel),
638 "sentinel `{sentinel}` reached a telemetry batch:\n{serialized}"
639 );
640 }
641
642 let stdout = String::from_utf8_lossy(&output.stdout);
643 let stderr = String::from_utf8_lossy(&output.stderr);
644 assert!(
645 !stdout.contains(SENTINEL_API_KEY),
646 "the API key leaked into stdout"
647 );
648 assert!(
649 !stderr.contains(SENTINEL_API_KEY),
650 "the API key leaked into stderr"
651 );
652 for file in fixture.written_files() {
653 let Ok(bytes) = std::fs::read(&file) else {
654 continue;
655 };
656 assert!(
657 !String::from_utf8_lossy(&bytes).contains(SENTINEL_API_KEY),
658 "the API key leaked into {}",
659 file.display()
660 );
661 }
662 }
663
664 /// The buffer file is an **untrusted input**, and this is the test that says so.
665 ///
666 /// Every bound in `codewhale-telemetry`'s schema is a property of how a payload
667 /// is *built*: closed enums, `u32`s, `ProviderKind::as_str()`,
668 /// `reduce_panic_site`. None of that survives the round trip, because `flush`
669 /// re-reads `buffer.jsonl` and deserializes it — and `$CODEWHALE_HOME` is a
670 /// predictable path that anything running as the user can append to. The
671 /// realistic writer is not an intruder: it is a `Bash` tool call this very
672 /// session made on the model's behalf, or an MCP server, or a hook. Without a
673 /// drain-path re-check, telemetry is a confused deputy that POSTs whatever that
674 /// writer chooses to the configured endpoint, under the user's install id, past
675 /// every egress control the user has on the provider route.
676 ///
677 /// The injection happens **after** the session has armed, on purpose: `init`
678 /// truncates the buffer, so a pre-arming plant proves nothing.
679 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
680 async fn a_hostile_buffer_line_never_reaches_a_batch() {
681 let server = start_recorder().await;
682 // A slow first token holds the session open long enough to append.
683 mount_model(&server, Duration::from_secs(5)).await;
684 let fixture = Fixture::new().with_endpoint(&server.uri());
685 plant_sentinels(&fixture, &server.uri());
686
687 let mut command = exec_command(&fixture, "hello");
688 let mut child = command.spawn().expect("spawn codewhale-tui exec");
689
690 let buffer = fixture.telemetry_root().join("buffer.jsonl");
691 wait_until(Duration::from_secs(30), || buffer.exists());
692 append_lines(
693 &buffer,
694 &[
695 // The path-bearing field, carrying a path it was never meant to.
696 json!({"event": "panic", "site": SENTINEL_INJECTED}).to_string(),
697 // The one event field read back from `state.json`.
698 json!({"event": "install_or_upgrade", "kind": "upgrade",
699 "previous_version": SENTINEL_INJECTED})
700 .to_string(),
701 // The provider set, whose whole design is that it cannot carry a
702 // customer's `[providers.<name>]` table key.
703 json!({"event": "session_end", "duration_bucket": "lt_1m",
704 "exit_class": "clean", "cold_start_bucket": null,
705 "providers": [SENTINEL_INJECTED],
706 "counters": {"turns": 0, "tool_calls": 0, "fleet_dispatch": 0,
707 "workflow_run": 0, "subagent_spawn": 0,
708 "mcp_server_connected": 0, "memory_search": 0,
709 "approval_modal_shown": 0, "approval_auto_allowed": 0,
710 "command_palette_open": 0},
711 "errors": {"auth_preflight_failed": 0, "provider_http_4xx": 0,
712 "provider_http_5xx": 0, "tool_denied_by_policy": 0,
713 "tool_timeout": 0, "network_error": 0},
714 "turn_wall": {"lt_5s": 0, "5_30s": 0, "30_120s": 0, "gte_120s": 0}})
715 .to_string(),
716 ],
717 );
718
719 let _ = child
720 .wait_timeout(EXEC_TIMEOUT)
721 .expect("wait for codewhale-tui exec")
722 .expect("codewhale-tui exec must exit");
723
724 let batches = recorded_batches(&server).await;
725 assert!(
726 !batches.is_empty(),
727 "this test is only meaningful against a batch that was actually sent"
728 );
729 let serialized = serde_json::to_string(&batches).expect("serialize batches");
730 assert!(
731 !serialized.contains(SENTINEL_INJECTED),
732 "a line appended to buffer.jsonl was POSTed verbatim:\n{serialized}"
733 );
734 }
735
736 /// Append raw lines to a sink, the way any other process on the machine would.
737 fn append_lines(path: &Path, lines: &[String]) {
738 use std::io::Write as _;
739 let mut file = std::fs::OpenOptions::new()
740 .append(true)
741 .open(path)
742 .expect("open the telemetry buffer");
743 for line in lines {
744 writeln!(file, "{line}").expect("append to the telemetry buffer");
745 }
746 }
747
748 /// The documented mid-session opt-out — `codewhale config set telemetry false`,
749 /// written by another process — must be observed by a session that is already
750 /// running. The flush re-resolves from disk before it sends anything.
751 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
752 async fn mid_session_opt_out_stops_the_shutdown_flush() {
753 let server = start_recorder().await;
754 // A slow first token gives the second writer a window while the session is
755 // armed and buffering.
756 mount_model(&server, Duration::from_secs(4)).await;
757 let fixture = Fixture::new().with_endpoint(&server.uri());
758 plant_sentinels(&fixture, &server.uri());
759
760 let mut command = exec_command(&fixture, "hello");
761 let mut child = command.spawn().expect("spawn codewhale-tui exec");
762
763 // Wait until the session has armed and started buffering, then take the
764 // documented way out from outside the process.
765 let buffer = fixture.telemetry_root().join("buffer.jsonl");
766 wait_until(Duration::from_secs(30), || buffer.exists());
767 fixture.write_config(&sentinel_config(&server.uri(), false));
768
769 let status = child
770 .wait_timeout(EXEC_TIMEOUT)
771 .expect("wait for codewhale-tui exec")
772 .expect("codewhale-tui exec must exit");
773 let _ = status;
774
775 assert_no_batches(&server, "an opt-out written mid-session").await;
776 let root = fixture.telemetry_root();
777 assert!(
778 root.join("disabled").exists(),
779 "the opt-out wipe must leave a tombstone the next run also honours"
780 );
781 }
782
783 /// Ctrl-C must not wait on a lock a second Codewhale process is holding.
784 ///
785 /// This is why appends never take the compaction lock: `flock` is per-fd within
786 /// a process, so a blocking acquisition on the signal path would hang exit for
787 /// as long as any other holder lives.
788 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
789 async fn ctrl_c_exits_while_a_second_process_holds_the_lock() {
790 let server = start_recorder().await;
791 mount_model(&server, Duration::from_secs(30)).await;
792 let fixture = Fixture::new().with_endpoint(&server.uri());
793 plant_sentinels(&fixture, &server.uri());
794
795 let mut command = exec_command(&fixture, "hello");
796 let mut child = command.spawn().expect("spawn codewhale-tui exec");
797
798 let root = fixture.telemetry_root();
799 let lock_path = root.join("buffer.jsonl.lock");
800 // Arming itself takes the lock, so the holder below must wait until the
801 // session is armed — otherwise this test would pin a deadlock it created.
802 wait_until(Duration::from_secs(30), || {
803 root.join("buffer.jsonl").exists()
804 });
805 let _holder = LockHolder::take(&lock_path);
806
807 // SIGINT to the child alone; the process group belongs to the test runner.
808 let pid = child.id() as libc::pid_t;
809 // SAFETY: `kill` with a pid this process spawned and has not reaped.
810 unsafe {
811 libc::kill(pid, libc::SIGINT);
812 }
813
814 let started = Instant::now();
815 let status = child
816 .wait_timeout(Duration::from_secs(10))
817 .expect("wait for codewhale-tui exec");
818 let status = status.unwrap_or_else(|| {
819 let _ = child.kill();
820 panic!(
821 "Ctrl-C blocked for {:?} while another process held the telemetry lock — \
822 the signal path must append without taking it",
823 started.elapsed()
824 )
825 });
826 assert_eq!(
827 status.code(),
828 Some(130),
829 "SIGINT must still exit 130 with the telemetry lock held elsewhere"
830 );
831 assert!(
832 started.elapsed() < Duration::from_secs(1),
833 "Ctrl-C took {:?} while the telemetry lock was held elsewhere",
834 started.elapsed()
835 );
836 }
837
838 /// Holds the telemetry compaction lock for the lifetime of the value.
839 struct LockHolder {
840 file: std::fs::File,
841 }
842
843 impl LockHolder {
844 fn take(path: &Path) -> Self {
845 let file = std::fs::OpenOptions::new()
846 .create(true)
847 .read(true)
848 .write(true)
849 .truncate(false)
850 .open(path)
851 .expect("open the telemetry lock");
852 let fd = std::os::unix::io::AsRawFd::as_raw_fd(&file);
853 // SAFETY: `fd` is owned by `file` and outlives the call.
854 let taken = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
855 assert_eq!(
856 taken, 0,
857 "the telemetry lock must be free before the test takes it"
858 );
859 Self { file }
860 }
861 }
862
863 impl Drop for LockHolder {
864 fn drop(&mut self) {
865 let fd = std::os::unix::io::AsRawFd::as_raw_fd(&self.file);
866 // SAFETY: same fd, still owned by `self`.
867 unsafe {
868 libc::flock(fd, libc::LOCK_UN);
869 }
870 }
871 }
872
873 // ── exec harness ─────────────────────────────────────────────────────────
874
875 fn sse_chunk(value: Value) -> String {
876 format!(
877 "data: {}\n\n",
878 serde_json::to_string(&value).expect("SSE JSON")
879 )
880 }
881
882 fn text_sse(text: &str) -> String {
883 [
884 sse_chunk(json!({
885 "id": "chatcmpl-tc",
886 "object": "chat.completion.chunk",
887 "model": TEST_MODEL,
888 "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": null}]
889 })),
890 sse_chunk(json!({
891 "id": "chatcmpl-tc",
892 "object": "chat.completion.chunk",
893 "model": TEST_MODEL,
894 "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}],
895 "usage": {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9}
896 })),
897 "data: [DONE]\n\n".to_string(),
898 ]
899 .join("")
900 }
901
902 async fn mount_model(server: &MockServer, delay: Duration) {
903 Mock::given(method("GET"))
904 .and(path_matcher("/v1/models"))
905 .respond_with(
906 ResponseTemplate::new(200)
907 .insert_header("content-type", "application/json")
908 .set_body_json(json!({
909 "object": "list",
910 "data": [{"id": TEST_MODEL, "object": "model"}]
911 })),
912 )
913 .mount(server)
914 .await;
915 Mock::given(method("POST"))
916 .and(path_matcher(MODEL_PATH))
917 .respond_with(
918 ResponseTemplate::new(200)
919 .insert_header("content-type", "text/event-stream")
920 .insert_header("cache-control", "no-cache")
921 .set_body_string(text_sse("acknowledged"))
922 .set_delay(delay),
923 )
924 .mount(server)
925 .await;
926 }
927
928 /// A config whose provider table key, MCP server name, and workspace file are
929 /// all sentinels, so a leak has somewhere to come from.
930 fn sentinel_config(base_url: &str, telemetry: bool) -> String {
931 format!(
932 "telemetry = {telemetry}\nprovider = \"{SENTINEL_PROVIDER_TABLE}\"\n\n\
933 [providers.{SENTINEL_PROVIDER_TABLE}]\n\
934 kind = \"openai-compatible\"\n\
935 base_url = \"{base_url}/v1\"\n\
936 model = \"{TEST_MODEL}\"\n\
937 api_key_env = \"{SENTINEL_API_KEY_ENV}\"\n"
938 )
939 }
940
941 fn plant_sentinels(fixture: &Fixture, base_url: &str) {
942 fixture.write_config(&sentinel_config(base_url, true));
943 fixture.record_notice(true);
944 std::fs::write(
945 fixture.workspace.join(SENTINEL_FILENAME),
946 "sentinel workspace file\n",
947 )
948 .expect("plant workspace file");
949 std::fs::write(
950 fixture.codewhale_home.join("mcp.json"),
951 json!({"mcpServers": {SENTINEL_MCP_SERVER: {"command": "/bin/true", "args": []}}})
952 .to_string(),
953 )
954 .expect("plant MCP config");
955 }
956
957 fn exec_command(fixture: &Fixture, prompt: &str) -> Command {
958 let mut command = fixture.command();
959 command
960 .env(
961 "CODEWHALE_MCP_CONFIG",
962 fixture.codewhale_home.join("mcp.json"),
963 )
964 .env(SENTINEL_API_KEY_ENV, SENTINEL_API_KEY)
965 .args([
966 "--config",
967 fixture.config_path.to_str().expect("config path"),
968 "--workspace",
969 fixture.workspace.to_str().expect("workspace path"),
970 "--no-project-config",
971 "--skip-onboarding",
972 "exec",
973 "--auto",
974 "--output-format",
975 "stream-json",
976 "--",
977 prompt,
978 ])
979 .stdout(Stdio::piped())
980 .stderr(Stdio::piped());
981 command
982 }
983
984 fn run_exec(fixture: &Fixture, prompt: &str) -> Output {
985 let mut command = exec_command(fixture, prompt);
986 let mut child = command.spawn().expect("spawn codewhale-tui exec");
987 let stdout = read_in_background(child.stdout.take().expect("stdout pipe"));
988 let stderr = read_in_background(child.stderr.take().expect("stderr pipe"));
989 let status = match child.wait_timeout(EXEC_TIMEOUT).expect("wait for exec") {
990 Some(status) => status,
991 None => {
992 let _ = child.kill();
993 panic!("codewhale-tui exec did not exit within {EXEC_TIMEOUT:?}");
994 }
995 };
996 Output {
997 status,
998 stdout: stdout.join().expect("stdout reader"),
999 stderr: stderr.join().expect("stderr reader"),
1000 }
1001 }
1002
1003 fn read_in_background(mut pipe: impl Read + Send + 'static) -> std::thread::JoinHandle<Vec<u8>> {
1004 std::thread::spawn(move || {
1005 let mut buffer = Vec::new();
1006 let _ = pipe.read_to_end(&mut buffer);
1007 buffer
1008 })
1009 }
1010
1011 fn wait_until(limit: Duration, mut ready: impl FnMut() -> bool) {
1012 let started = Instant::now();
1013 while started.elapsed() < limit {
1014 if ready() {
1015 return;
1016 }
1017 std::thread::sleep(Duration::from_millis(25));
1018 }
1019 panic!("condition was not reached within {limit:?}");
1020 }
1021
1022 // ── Seeded state ─────────────────────────────────────────────────────────
1023
1024 /// A home that already belongs to a consenting user: an identity, a populated
1025 /// buffer, a flush record, and the lock file.
1026 fn seed_consenting_home(root: &Path) {
1027 std::fs::create_dir_all(root).expect("create telemetry root");
1028 std::fs::write(
1029 root.join("install_id.json"),
1030 json!({
1031 "schema_version": 1,
1032 "install_id": "11111111-2222-3333-4444-555555555555",
1033 "rotated_at": "2026-01-01T00:00:00Z"
1034 })
1035 .to_string(),
1036 )
1037 .expect("seed install id");
1038 std::fs::write(
1039 root.join("state.json"),
1040 json!({"schema_version": 1, "last_version": "0.0.1"}).to_string(),
1041 )
1042 .expect("seed state");
1043 std::fs::write(
1044 root.join("buffer.jsonl"),
1045 format!(
1046 "{}\n",
1047 json!({"event": "session_start", "source": "unknown"})
1048 ),
1049 )
1050 .expect("seed buffer");
1051 std::fs::write(root.join("buffer.jsonl.lock"), b"").expect("seed lock file");
1052 }
1053
1054 fn snapshot(root: &Path) -> Vec<(String, Vec<u8>)> {
1055 let mut files = Vec::new();
1056 collect_files(root, &mut files);
1057 let mut out: Vec<(String, Vec<u8>)> = files
1058 .into_iter()
1059 .map(|path| {
1060 let name = path
1061 .strip_prefix(root)
1062 .unwrap_or(&path)
1063 .to_string_lossy()
1064 .into_owned();
1065 let bytes = std::fs::read(&path).unwrap_or_default();
1066 (name, bytes)
1067 })
1068 .collect();
1069 out.sort();
1070 out
1071 }
1072
1072 lines RUST