| 1 | use std::collections::VecDeque; |
| 2 | use std::io::{self, Write}; |
| 3 | use std::path::PathBuf; |
| 4 | use std::sync::{Arc, Mutex}; |
| 5 | |
| 6 | use encoding_rs::{CoderResult, Decoder, UTF_8}; |
| 7 | |
| 8 | const BOUNDED_OUTPUT_MAX_LINES: usize = 2_000; |
| 9 | const BOUNDED_OUTPUT_MAX_BYTES: usize = 50 * 1024; |
| 10 | const BOUNDED_OUTPUT_RETAIN_BYTES: usize = BOUNDED_OUTPUT_MAX_BYTES + 4; |
| 11 | |
| 12 | #[derive(Debug)] |
| 13 | pub(super) struct BoundedOutputSnapshot { |
| 14 | pub(super) content: String, |
| 15 | pub(super) total_bytes: usize, |
| 16 | pub(super) retained_bytes: usize, |
| 17 | pub(super) truncated: bool, |
| 18 | } |
| 19 | |
| 20 | /// One decoded, arrival-ordered stream: complete output goes to disk while |
| 21 | /// memory retains only enough tail bytes for the 2,000-line/50KiB result bound. |
| 22 | pub(super) struct BoundedOutputAccumulator { |
| 23 | tail: VecDeque<u8>, |
| 24 | tail_newlines: usize, |
| 25 | total_bytes: usize, |
| 26 | total_newlines: usize, |
| 27 | current_line_bytes: usize, |
| 28 | last_line_bytes: usize, |
| 29 | front_clipped: bool, |
| 30 | last_byte: Option<u8>, |
| 31 | decoder: Decoder, |
| 32 | stream_finished: bool, |
| 33 | stream_error: Option<String>, |
| 34 | temp: Option<tempfile::NamedTempFile>, |
| 35 | full_output_path: Option<PathBuf>, |
| 36 | /// Why the on-disk spill file could not be created (disk full, descriptor |
| 37 | /// exhaustion, unwritable temp dir). The stream still runs and the bounded |
| 38 | /// tail is still delivered; only "Full output: <path>" is unavailable. |
| 39 | spill_unavailable: Option<String>, |
| 40 | } |
| 41 | |
| 42 | impl BoundedOutputAccumulator { |
| 43 | /// Build an accumulator whose complete-output spill file lives in |
| 44 | /// `spill_dir` (`None` = process temp dir). Never fails: when the spill |
| 45 | /// file cannot be created (disk full, `EMFILE`, missing temp dir) the |
| 46 | /// command still runs and the bounded tail is still returned — the spill |
| 47 | /// is a convenience, not a precondition for executing `echo ok`. Tests |
| 48 | /// pass a nonexistent dir to fault-inject the failure. |
| 49 | pub(super) fn new_in(spill_dir: Option<&std::path::Path>) -> Self { |
| 50 | let mut builder = tempfile::Builder::new(); |
| 51 | builder.prefix("codewhale-bash-"); |
| 52 | let temp = match spill_dir { |
| 53 | Some(dir) => builder.tempfile_in(dir), |
| 54 | None => builder.tempfile(), |
| 55 | }; |
| 56 | let (temp, spill_unavailable) = match temp { |
| 57 | Ok(temp) => (Some(temp), None), |
| 58 | Err(error) => { |
| 59 | tracing::warn!( |
| 60 | error = %error, |
| 61 | "shell output spill file unavailable; continuing with the in-memory tail only" |
| 62 | ); |
| 63 | (None, Some(spill_unavailable_reason(&error))) |
| 64 | } |
| 65 | }; |
| 66 | Self { |
| 67 | tail: VecDeque::with_capacity(BOUNDED_OUTPUT_RETAIN_BYTES), |
| 68 | tail_newlines: 0, |
| 69 | total_bytes: 0, |
| 70 | total_newlines: 0, |
| 71 | current_line_bytes: 0, |
| 72 | last_line_bytes: 0, |
| 73 | front_clipped: false, |
| 74 | last_byte: None, |
| 75 | decoder: UTF_8.new_decoder_without_bom_handling(), |
| 76 | stream_finished: false, |
| 77 | stream_error: None, |
| 78 | temp, |
| 79 | full_output_path: None, |
| 80 | spill_unavailable, |
| 81 | } |
| 82 | } |
| 83 | |
| 84 | /// Why the complete output is not being persisted, if it is not. |
| 85 | #[cfg(test)] |
| 86 | pub(super) fn spill_unavailable(&self) -> Option<&str> { |
| 87 | self.spill_unavailable.as_deref() |
| 88 | } |
| 89 | |
| 90 | fn decode(&mut self, bytes: &[u8], last: bool) -> String { |
| 91 | let capacity = self |
| 92 | .decoder |
| 93 | .max_utf8_buffer_length(bytes.len()) |
| 94 | .unwrap_or(bytes.len().saturating_mul(3).saturating_add(3)); |
| 95 | let mut decoded = String::with_capacity(capacity); |
| 96 | let mut offset = 0; |
| 97 | loop { |
| 98 | let (result, read, _) = |
| 99 | self.decoder |
| 100 | .decode_to_string(&bytes[offset..], &mut decoded, last); |
| 101 | offset += read; |
| 102 | if result == CoderResult::InputEmpty { |
| 103 | return decoded; |
| 104 | } |
| 105 | decoded.reserve(capacity.max(4)); |
| 106 | } |
| 107 | } |
| 108 | |
| 109 | pub(super) fn append(&mut self, raw: &[u8]) -> io::Result<()> { |
| 110 | if self.stream_finished { |
| 111 | return Err(io::Error::other( |
| 112 | "shell output arrived after the stream closed", |
| 113 | )); |
| 114 | } |
| 115 | if let Some(temp) = self.temp.as_mut() { |
| 116 | temp.write_all(raw)?; |
| 117 | } |
| 118 | let decoded = self.decode(raw, false); |
| 119 | self.append_decoded(decoded.as_bytes()); |
| 120 | Ok(()) |
| 121 | } |
| 122 | |
| 123 | pub(super) fn finish(&mut self) -> io::Result<()> { |
| 124 | if !self.stream_finished { |
| 125 | let decoded = self.decode(&[], true); |
| 126 | self.append_decoded(decoded.as_bytes()); |
| 127 | if let Some(temp) = self.temp.as_mut() { |
| 128 | temp.flush()?; |
| 129 | } |
| 130 | self.stream_finished = true; |
| 131 | } |
| 132 | Ok(()) |
| 133 | } |
| 134 | |
| 135 | pub(super) fn record_error(&mut self, error: &io::Error) { |
| 136 | self.stream_error = Some(error.to_string()); |
| 137 | } |
| 138 | |
| 139 | fn append_decoded(&mut self, bytes: &[u8]) { |
| 140 | self.total_bytes = self.total_bytes.saturating_add(bytes.len()); |
| 141 | for &byte in bytes { |
| 142 | self.tail.push_back(byte); |
| 143 | if byte == b'\n' { |
| 144 | self.tail_newlines += 1; |
| 145 | self.total_newlines += 1; |
| 146 | self.last_line_bytes = self.current_line_bytes; |
| 147 | self.current_line_bytes = 0; |
| 148 | } else { |
| 149 | self.current_line_bytes += 1; |
| 150 | } |
| 151 | self.last_byte = Some(byte); |
| 152 | } |
| 153 | while self.tail.len() > BOUNDED_OUTPUT_RETAIN_BYTES { |
| 154 | self.pop_front(); |
| 155 | self.front_clipped = true; |
| 156 | } |
| 157 | while self.tail_lines() > BOUNDED_OUTPUT_MAX_LINES { |
| 158 | while let Some(byte) = self.tail.pop_front() { |
| 159 | if byte == b'\n' { |
| 160 | self.tail_newlines -= 1; |
| 161 | break; |
| 162 | } |
| 163 | } |
| 164 | self.front_clipped = false; |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | fn pop_front(&mut self) { |
| 169 | if self.tail.pop_front() == Some(b'\n') { |
| 170 | self.tail_newlines -= 1; |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | fn tail_lines(&self) -> usize { |
| 175 | self.tail_newlines + usize::from(self.tail.back().is_some_and(|byte| *byte != b'\n')) |
| 176 | } |
| 177 | |
| 178 | fn total_lines(&self) -> usize { |
| 179 | self.total_newlines + usize::from(self.last_byte.is_some_and(|byte| byte != b'\n')) |
| 180 | } |
| 181 | |
| 182 | fn selected(&self) -> (Vec<u8>, bool) { |
| 183 | let mut bytes = self.tail.iter().copied().collect::<Vec<_>>(); |
| 184 | let recent_line_bytes = if self.last_byte == Some(b'\n') { |
| 185 | self.last_line_bytes |
| 186 | } else { |
| 187 | self.current_line_bytes |
| 188 | }; |
| 189 | let partial_line = recent_line_bytes > BOUNDED_OUTPUT_MAX_BYTES; |
| 190 | if partial_line { |
| 191 | if bytes.last() == Some(&b'\n') { |
| 192 | bytes.pop(); |
| 193 | } |
| 194 | let floor = bytes.len().saturating_sub(BOUNDED_OUTPUT_MAX_BYTES); |
| 195 | let start = (floor..bytes.len()) |
| 196 | .find(|index| std::str::from_utf8(&bytes[*index..]).is_ok()) |
| 197 | .unwrap_or(bytes.len()); |
| 198 | bytes.drain(..start); |
| 199 | } else if self.front_clipped |
| 200 | && let Some(newline) = bytes.iter().position(|byte| *byte == b'\n') |
| 201 | { |
| 202 | bytes.drain(..=newline); |
| 203 | } |
| 204 | (bytes, partial_line) |
| 205 | } |
| 206 | |
| 207 | fn format_size(bytes: usize) -> String { |
| 208 | if bytes < 1024 { |
| 209 | format!("{bytes}B") |
| 210 | } else if bytes < 1024 * 1024 { |
| 211 | format!("{:.1}KB", bytes as f64 / 1024.0) |
| 212 | } else { |
| 213 | format!("{:.1}MB", bytes as f64 / (1024.0 * 1024.0)) |
| 214 | } |
| 215 | } |
| 216 | |
| 217 | pub(super) fn total_bytes(&self) -> usize { |
| 218 | self.total_bytes |
| 219 | } |
| 220 | |
| 221 | pub(super) fn snapshot(&mut self, finalize: bool) -> io::Result<BoundedOutputSnapshot> { |
| 222 | if let Some(error) = self.stream_error.as_ref() { |
| 223 | return Err(io::Error::other(error.clone())); |
| 224 | } |
| 225 | let (selected, partial_line) = self.selected(); |
| 226 | let retained_bytes = selected.len(); |
| 227 | let truncated = retained_bytes < self.total_bytes; |
| 228 | let total_lines = self.total_lines(); |
| 229 | let kept_lines = selected.iter().filter(|byte| **byte == b'\n').count() |
| 230 | + usize::from(selected.last().is_some_and(|byte| *byte != b'\n')); |
| 231 | let mut content = String::from_utf8(selected).expect("stream decoder emits valid UTF-8"); |
| 232 | |
| 233 | if finalize && self.stream_finished && self.full_output_path.is_none() { |
| 234 | if truncated { |
| 235 | if let Some(mut temp) = self.temp.take() { |
| 236 | temp.flush()?; |
| 237 | let (_, path) = temp.keep().map_err(|error| error.error)?; |
| 238 | self.full_output_path = Some(path); |
| 239 | } |
| 240 | } else { |
| 241 | self.temp.take(); |
| 242 | } |
| 243 | } |
| 244 | if truncated && finalize && self.full_output_path.is_none() { |
| 245 | let reason = self.spill_unavailable.as_deref().unwrap_or( |
| 246 | "the output stream did not close cleanly, so the spill file was not kept", |
| 247 | ); |
| 248 | content.push_str(&format!( |
| 249 | "\n\n[Showing the last {} of {} lines ({} limit). Full output was not persisted: {reason}]", |
| 250 | Self::format_size(retained_bytes), |
| 251 | total_lines, |
| 252 | Self::format_size(BOUNDED_OUTPUT_MAX_BYTES), |
| 253 | )); |
| 254 | } else if truncated |
| 255 | && finalize |
| 256 | && let Some(path) = self.full_output_path.as_ref() |
| 257 | { |
| 258 | if partial_line { |
| 259 | content.push_str(&format!( |
| 260 | "\n\n[Showing last {} of line {} (line is {}). Full output: {}]", |
| 261 | Self::format_size(retained_bytes), |
| 262 | total_lines, |
| 263 | Self::format_size(self.current_line_bytes), |
| 264 | path.display() |
| 265 | )); |
| 266 | } else { |
| 267 | let start = total_lines.saturating_sub(kept_lines) + 1; |
| 268 | let limit = if self.front_clipped { |
| 269 | format!(" ({} limit)", Self::format_size(BOUNDED_OUTPUT_MAX_BYTES)) |
| 270 | } else { |
| 271 | String::new() |
| 272 | }; |
| 273 | content.push_str(&format!( |
| 274 | "\n\n[Showing lines {start}-{total_lines} of {total_lines}{limit}. Full output: {}]", |
| 275 | path.display() |
| 276 | )); |
| 277 | } |
| 278 | } |
| 279 | Ok(BoundedOutputSnapshot { |
| 280 | content, |
| 281 | total_bytes: self.total_bytes, |
| 282 | retained_bytes, |
| 283 | truncated, |
| 284 | }) |
| 285 | } |
| 286 | |
| 287 | #[cfg(test)] |
| 288 | pub(super) fn retained_memory_bytes(&self) -> usize { |
| 289 | self.tail.len() |
| 290 | } |
| 291 | |
| 292 | #[cfg(test)] |
| 293 | pub(super) fn full_output_path(&self) -> Option<&std::path::Path> { |
| 294 | self.full_output_path.as_deref() |
| 295 | } |
| 296 | } |
| 297 | |
| 298 | /// Human-readable, actionable reason for a failed spill-file creation. |
| 299 | pub(super) fn spill_unavailable_reason(error: &io::Error) -> String { |
| 300 | match resource_exhaustion_hint(error) { |
| 301 | Some(hint) => format!("{error} ({hint})"), |
| 302 | None => error.to_string(), |
| 303 | } |
| 304 | } |
| 305 | |
| 306 | /// When an I/O error looks like host resource exhaustion, name the likely |
| 307 | /// cause and the remedy. Returns `None` for ordinary errors. |
| 308 | pub(super) fn resource_exhaustion_hint(error: &io::Error) -> Option<&'static str> { |
| 309 | use io::ErrorKind; |
| 310 | match error.kind() { |
| 311 | ErrorKind::StorageFull | ErrorKind::QuotaExceeded => { |
| 312 | return Some("the disk holding the temp dir is full; free space and retry"); |
| 313 | } |
| 314 | ErrorKind::OutOfMemory => { |
| 315 | return Some("the host is out of memory; close heavy processes and retry"); |
| 316 | } |
| 317 | _ => {} |
| 318 | } |
| 319 | let code = error.raw_os_error()?; |
| 320 | // ENOSPC / EDQUOT / EMFILE / ENFILE / ENOMEM / EAGAIN — the codes fork(2), |
| 321 | // pipe(2), and open(2) return when the machine is thrashing. |
| 322 | #[cfg(unix)] |
| 323 | { |
| 324 | if code == libc::ENOSPC || code == libc::EDQUOT { |
| 325 | return Some("the disk holding the temp dir is full; free space and retry"); |
| 326 | } |
| 327 | if code == libc::EMFILE || code == libc::ENFILE { |
| 328 | return Some( |
| 329 | "the process or host has run out of file descriptors; close background jobs or raise `ulimit -n` and retry", |
| 330 | ); |
| 331 | } |
| 332 | if code == libc::ENOMEM { |
| 333 | return Some("the host is out of memory; close heavy processes and retry"); |
| 334 | } |
| 335 | if code == libc::EAGAIN { |
| 336 | return Some( |
| 337 | "the host refused to create a process, thread, or pipe (resource limit reached); close heavy processes and retry", |
| 338 | ); |
| 339 | } |
| 340 | } |
| 341 | #[cfg(windows)] |
| 342 | { |
| 343 | // ERROR_DISK_FULL, ERROR_HANDLE_DISK_FULL, ERROR_NOT_ENOUGH_MEMORY, ERROR_TOO_MANY_OPEN_FILES |
| 344 | if code == 112 || code == 39 { |
| 345 | return Some("the disk holding the temp dir is full; free space and retry"); |
| 346 | } |
| 347 | if code == 8 { |
| 348 | return Some("the host is out of memory; close heavy processes and retry"); |
| 349 | } |
| 350 | if code == 4 { |
| 351 | return Some( |
| 352 | "the process has run out of file handles; close background jobs and retry", |
| 353 | ); |
| 354 | } |
| 355 | } |
| 356 | let _ = code; |
| 357 | None |
| 358 | } |
| 359 | |
| 360 | /// Hard in-flight ceiling for one raw shell stream held in memory (#5472). |
| 361 | /// Past this the oldest bytes are dropped — counted, never silently lost — so |
| 362 | /// one chatty command (`cargo build -v`, `git log -p`) cannot grow the process |
| 363 | /// by its entire output. Deliberately far above every consumer of these bytes: |
| 364 | /// the 30 KB tool-result truncation (`shell_output::MAX_OUTPUT_SIZE`), the |
| 365 | /// 1,200-char job-panel tail and the 1 KiB completion tail all fit with three |
| 366 | /// orders of magnitude to spare. The only surface a clip can reach is the |
| 367 | /// durable completion artifact, which records the omission explicitly. |
| 368 | pub(super) const RAW_STREAM_MAX_BYTES: usize = 16 * 1024 * 1024; |
| 369 | |
| 370 | /// Extra headroom before a front-drop, so the O(len) compaction runs once per |
| 371 | /// `cap / 4` bytes appended instead of once per chunk. |
| 372 | const RAW_STREAM_DROP_SLACK: usize = RAW_STREAM_MAX_BYTES / 4; |
| 373 | |
| 374 | /// Tail retained once a job's output has been *delivered* — the foreground |
| 375 | /// result is already the tool result, or the completion evidence is already |
| 376 | /// written to its session artifact. Everything past this is dead weight for |
| 377 | /// the up-to-1 h the finished record stays listed (#5472 finding 1). |
| 378 | pub(super) const RAW_STREAM_SETTLED_TAIL_BYTES: usize = 64 * 1024; |
| 379 | |
| 380 | /// One raw (undecoded) shell stream retained in memory for a live job. |
| 381 | /// |
| 382 | /// Bounded two independent ways, which is the whole point of the type: |
| 383 | /// `append` enforces `cap` while the command runs, and `release_to_tail` |
| 384 | /// collapses the buffer the moment its bytes have been delivered. Both record |
| 385 | /// how many leading bytes were discarded so `total_len` — and therefore every |
| 386 | /// `stdout_len` / `byte_length` the model and the artifact see — stays honest. |
| 387 | pub(super) struct RawOutputBuffer { |
| 388 | data: Vec<u8>, |
| 389 | dropped: usize, |
| 390 | cap: usize, |
| 391 | abandoned: bool, |
| 392 | } |
| 393 | |
| 394 | impl RawOutputBuffer { |
| 395 | pub(super) fn new() -> Self { |
| 396 | Self::with_cap(RAW_STREAM_MAX_BYTES) |
| 397 | } |
| 398 | |
| 399 | pub(super) fn with_cap(cap: usize) -> Self { |
| 400 | Self { |
| 401 | data: Vec::new(), |
| 402 | dropped: 0, |
| 403 | cap: cap.max(1), |
| 404 | abandoned: false, |
| 405 | } |
| 406 | } |
| 407 | |
| 408 | /// Append, returning `false` once nobody will ever read this stream again. |
| 409 | /// |
| 410 | /// The reader thread uses that as its exit condition, which is the only way |
| 411 | /// out when a descendant has escaped the process group and holds the pipe |
| 412 | /// write-end open: `read()` will never see EOF, so without this the thread |
| 413 | /// runs — and retains its buffer — for the life of the process (#5472 |
| 414 | /// finding 2). |
| 415 | pub(super) fn append(&mut self, bytes: &[u8]) -> bool { |
| 416 | if self.abandoned { |
| 417 | // Keep the total honest even though the bytes are discarded. |
| 418 | self.dropped = self.dropped.saturating_add(bytes.len()); |
| 419 | return false; |
| 420 | } |
| 421 | self.data.extend_from_slice(bytes); |
| 422 | if self.data.len() > self.cap.saturating_add(RAW_STREAM_DROP_SLACK.min(self.cap)) { |
| 423 | self.drop_front_to(self.cap); |
| 424 | } |
| 425 | true |
| 426 | } |
| 427 | |
| 428 | /// Give up on this stream: release everything held and stop accepting more. |
| 429 | /// |
| 430 | /// Called when the bounded reader join times out. The shell is already |
| 431 | /// terminal and its result already delivered, so nothing can consume these |
| 432 | /// bytes; holding them until the writer eventually closes is pure residency. |
| 433 | pub(super) fn abandon(&mut self) { |
| 434 | self.abandoned = true; |
| 435 | self.dropped = self.dropped.saturating_add(self.data.len()); |
| 436 | self.data = Vec::new(); |
| 437 | } |
| 438 | |
| 439 | /// Total bytes this stream has produced, including bytes no longer held. |
| 440 | pub(super) fn total_len(&self) -> usize { |
| 441 | self.dropped.saturating_add(self.data.len()) |
| 442 | } |
| 443 | |
| 444 | /// Leading bytes discarded by the in-flight cap or by `release_to_tail`. |
| 445 | pub(super) fn dropped(&self) -> usize { |
| 446 | self.dropped |
| 447 | } |
| 448 | |
| 449 | pub(super) fn retained(&self) -> &[u8] { |
| 450 | &self.data |
| 451 | } |
| 452 | |
| 453 | /// Collapse to at most `keep` trailing bytes and give the allocation back. |
| 454 | /// Called once a job is terminal *and* its output has been delivered. |
| 455 | pub(super) fn release_to_tail(&mut self, keep: usize) { |
| 456 | if self.data.len() <= keep { |
| 457 | return; |
| 458 | } |
| 459 | self.drop_front_to(keep); |
| 460 | self.data.shrink_to_fit(); |
| 461 | } |
| 462 | |
| 463 | fn drop_front_to(&mut self, keep: usize) { |
| 464 | let mut start = self.data.len().saturating_sub(keep); |
| 465 | // Snap forward off a UTF-8 continuation byte so the retained slice |
| 466 | // never begins mid-character (the leading-U+FFFD bug guarded against |
| 467 | // in `tail_from_buffer`). |
| 468 | while start < self.data.len() && (self.data[start] & 0xC0) == 0x80 { |
| 469 | start += 1; |
| 470 | } |
| 471 | self.data.drain(..start); |
| 472 | self.dropped = self.dropped.saturating_add(start); |
| 473 | } |
| 474 | } |
| 475 | |
| 476 | impl Default for RawOutputBuffer { |
| 477 | fn default() -> Self { |
| 478 | Self::new() |
| 479 | } |
| 480 | } |
| 481 | |
| 482 | pub(super) type SharedRawOutput = Arc<Mutex<RawOutputBuffer>>; |
| 483 | |
| 484 | pub(super) fn new_shared_raw_output() -> SharedRawOutput { |
| 485 | Arc::new(Mutex::new(RawOutputBuffer::new())) |
| 486 | } |
| 487 | |
| 488 | pub(super) fn take_delta_from_buffer( |
| 489 | buffer: &SharedRawOutput, |
| 490 | cursor: &mut usize, |
| 491 | ) -> (Vec<u8>, usize) { |
| 492 | let guard = buffer.lock().unwrap_or_else(|e| e.into_inner()); |
| 493 | let total = guard.total_len(); |
| 494 | // The cursor is an absolute offset into the stream. Bytes the bound already |
| 495 | // discarded can never be delivered as a delta, so skip forward over them |
| 496 | // rather than re-sending the retained tail as if it were new. |
| 497 | let start_abs = (*cursor).max(guard.dropped()).min(total); |
| 498 | let start = start_abs - guard.dropped(); |
| 499 | let retained = guard.retained(); |
| 500 | // Clone only the unread portion (the delta), not the entire accumulated buffer. |
| 501 | // Long-running processes can produce megabytes of output; cloning the full |
| 502 | // buffer on every poll held the ShellManager mutex for O(total_bytes) time. |
| 503 | let unread = &retained[start..]; |
| 504 | // A poll can land mid-character: the caller decodes this delta as UTF-8, so |
| 505 | // handing back a truncated multibyte sequence renders it as replacement |
| 506 | // glyphs and corrupts the next delta's leading byte too (the streaming-client |
| 507 | // bug from #1675, in the shell preview path). Leave an incomplete trailing |
| 508 | // sequence in the buffer for the next poll. Bytes that are genuinely invalid |
| 509 | // rather than merely unfinished still pass through, so binary output cannot |
| 510 | // stall the cursor, and the final result is read from the whole buffer. |
| 511 | let consumed = match std::str::from_utf8(unread) { |
| 512 | Ok(_) => unread.len(), |
| 513 | Err(error) if error.error_len().is_none() => error.valid_up_to(), |
| 514 | Err(_) => unread.len(), |
| 515 | }; |
| 516 | let delta = unread[..consumed].to_vec(); |
| 517 | *cursor = start_abs + consumed; |
| 518 | (delta, total) |
| 519 | } |
| 520 | |
| 521 | /// Read only the tail of a byte buffer and return (total_len, tail_string). |
| 522 | /// |
| 523 | /// Avoids cloning the full buffer when only a trailing excerpt is needed |
| 524 | /// (e.g. for the job-panel display). `max_tail_chars` is in Unicode scalar |
| 525 | /// values; we read at most `max_tail_chars * 4` bytes from the end to account |
| 526 | /// for multi-byte UTF-8 sequences. |
| 527 | pub(super) fn tail_from_buffer(buffer: &SharedRawOutput, max_tail_chars: usize) -> (usize, String) { |
| 528 | let guard = buffer.lock().unwrap_or_else(|e| e.into_inner()); |
| 529 | // The reported length is the stream's total, not what is still held: a |
| 530 | // released or clipped buffer must not make the model believe the command |
| 531 | // printed less than it did. |
| 532 | let total = guard.total_len(); |
| 533 | let retained = guard.retained(); |
| 534 | let retained_len = retained.len(); |
| 535 | // Over-estimate byte count (4 bytes per char worst case for UTF-8). |
| 536 | let mut tail_start = retained_len.saturating_sub(max_tail_chars.saturating_mul(4)); |
| 537 | // Snap forward to the next valid UTF-8 codepoint boundary so we don't |
| 538 | // pass a slice beginning with continuation bytes (0x80-0xBF) to |
| 539 | // from_utf8_lossy, which would emit a leading U+FFFD replacement char. |
| 540 | while tail_start < retained_len && (retained[tail_start] & 0xC0) == 0x80 { |
| 541 | tail_start += 1; |
| 542 | } |
| 543 | let tail_str = String::from_utf8_lossy(&retained[tail_start..]).into_owned(); |
| 544 | (total, tail_text(&tail_str, max_tail_chars)) |
| 545 | } |
| 546 | |
| 547 | pub(super) fn tail_text(text: &str, max_chars: usize) -> String { |
| 548 | if text.chars().count() <= max_chars { |
| 549 | return text.to_string(); |
| 550 | } |
| 551 | let tail = text |
| 552 | .chars() |
| 553 | .rev() |
| 554 | .take(max_chars) |
| 555 | .collect::<Vec<_>>() |
| 556 | .into_iter() |
| 557 | .rev() |
| 558 | .collect::<String>(); |
| 559 | format!("...{tail}") |
| 560 | } |
| 561 | |
| 562 | #[cfg(test)] |
| 563 | mod tests { |
| 564 | use super::{ |
| 565 | BOUNDED_OUTPUT_MAX_BYTES, BOUNDED_OUTPUT_MAX_LINES, BoundedOutputAccumulator, |
| 566 | RAW_STREAM_MAX_BYTES, RawOutputBuffer, SharedRawOutput, tail_from_buffer, |
| 567 | take_delta_from_buffer, |
| 568 | }; |
| 569 | use std::sync::{Arc, Mutex}; |
| 570 | |
| 571 | fn raw(bytes: &[u8]) -> SharedRawOutput { |
| 572 | let mut buffer = RawOutputBuffer::new(); |
| 573 | buffer.append(bytes); |
| 574 | Arc::new(Mutex::new(buffer)) |
| 575 | } |
| 576 | |
| 577 | fn append(buffer: &SharedRawOutput, bytes: &[u8]) { |
| 578 | buffer.lock().unwrap().append(bytes); |
| 579 | } |
| 580 | |
| 581 | #[test] |
| 582 | fn delta_holds_back_an_incomplete_trailing_utf8_sequence() { |
| 583 | // "宽" is three bytes; deliver two of them, then the rest. |
| 584 | let wide = "宽".as_bytes(); |
| 585 | let buffer = raw(b"ok "); |
| 586 | append(&buffer, &wide[..2]); |
| 587 | let mut cursor = 0usize; |
| 588 | |
| 589 | let (delta, total) = take_delta_from_buffer(&buffer, &mut cursor); |
| 590 | assert_eq!( |
| 591 | String::from_utf8(delta).expect("delta must be whole characters"), |
| 592 | "ok " |
| 593 | ); |
| 594 | assert_eq!(total, 5, "total still reports every buffered byte"); |
| 595 | assert_eq!(cursor, 3, "the split character stays unread"); |
| 596 | |
| 597 | append(&buffer, &wide[2..]); |
| 598 | let (delta, _) = take_delta_from_buffer(&buffer, &mut cursor); |
| 599 | assert_eq!( |
| 600 | String::from_utf8(delta).expect("delta must be whole characters"), |
| 601 | "宽" |
| 602 | ); |
| 603 | } |
| 604 | |
| 605 | #[test] |
| 606 | fn delta_does_not_stall_on_genuinely_invalid_bytes() { |
| 607 | // A lone 0xFF is never a valid start byte: passing it through keeps |
| 608 | // binary output flowing instead of parking the cursor forever. |
| 609 | let buffer = raw(&[b'a', 0xFF, b'b']); |
| 610 | let mut cursor = 0usize; |
| 611 | let (delta, total) = take_delta_from_buffer(&buffer, &mut cursor); |
| 612 | assert_eq!(delta, vec![b'a', 0xFF, b'b']); |
| 613 | assert_eq!(cursor, total); |
| 614 | } |
| 615 | |
| 616 | // === #5472: in-memory retention bounds for the raw `Bash` streams === |
| 617 | |
| 618 | #[test] |
| 619 | fn raw_buffer_caps_in_flight_bytes_and_keeps_the_total_honest() { |
| 620 | let mut buffer = RawOutputBuffer::with_cap(1_024); |
| 621 | // 4 MiB through a 1 KiB cap: the analogue of `cargo build -v` through |
| 622 | // the 16 MiB production ceiling. |
| 623 | for _ in 0..1_024 { |
| 624 | buffer.append(&[b'x'; 4_096]); |
| 625 | } |
| 626 | let produced = 1_024 * 4_096; |
| 627 | assert_eq!( |
| 628 | buffer.total_len(), |
| 629 | produced, |
| 630 | "the stream's length must survive the bound" |
| 631 | ); |
| 632 | assert_eq!(buffer.dropped(), produced - buffer.retained().len()); |
| 633 | assert!( |
| 634 | buffer.retained().len() <= 1_024 + 1_024 / 4, |
| 635 | "retained {} exceeded cap + slack", |
| 636 | buffer.retained().len() |
| 637 | ); |
| 638 | } |
| 639 | |
| 640 | #[test] |
| 641 | fn raw_buffer_release_collapses_to_a_tail_and_reports_the_omission() { |
| 642 | let mut buffer = RawOutputBuffer::new(); |
| 643 | buffer.append(&[b'y'; 200_000]); |
| 644 | assert_eq!(buffer.dropped(), 0, "200 KB is under the in-flight ceiling"); |
| 645 | |
| 646 | buffer.release_to_tail(1_000); |
| 647 | assert_eq!(buffer.retained().len(), 1_000); |
| 648 | assert_eq!(buffer.dropped(), 199_000); |
| 649 | assert_eq!( |
| 650 | buffer.total_len(), |
| 651 | 200_000, |
| 652 | "releasing memory must not rewrite how much the command printed" |
| 653 | ); |
| 654 | } |
| 655 | |
| 656 | #[test] |
| 657 | fn raw_buffer_never_retains_a_split_character() { |
| 658 | let mut buffer = RawOutputBuffer::with_cap(8); |
| 659 | // Each "宽" is 3 bytes, so a byte-exact tail would land mid-character. |
| 660 | for _ in 0..64 { |
| 661 | buffer.append("宽".as_bytes()); |
| 662 | } |
| 663 | assert!( |
| 664 | std::str::from_utf8(buffer.retained()).is_ok(), |
| 665 | "front-drop must snap off continuation bytes" |
| 666 | ); |
| 667 | |
| 668 | let mut released = RawOutputBuffer::new(); |
| 669 | for _ in 0..64 { |
| 670 | released.append("宽".as_bytes()); |
| 671 | } |
| 672 | released.release_to_tail(10); |
| 673 | assert!(std::str::from_utf8(released.retained()).is_ok()); |
| 674 | } |
| 675 | |
| 676 | #[test] |
| 677 | fn delta_skips_bytes_the_bound_already_discarded() { |
| 678 | // A consumer that stops reading while output keeps arriving must be |
| 679 | // moved forward, not handed the retained tail as if it were new bytes. |
| 680 | let buffer = Arc::new(Mutex::new(RawOutputBuffer::with_cap(16))); |
| 681 | append(&buffer, b"first-chunk-that-will-be-dropped-entirely"); |
| 682 | let mut cursor = 0usize; |
| 683 | let (delta, total) = take_delta_from_buffer(&buffer, &mut cursor); |
| 684 | let dropped = buffer.lock().unwrap().dropped(); |
| 685 | assert!(dropped > 0, "the cap must have clipped the front"); |
| 686 | assert_eq!(cursor, total, "cursor lands at the stream's true position"); |
| 687 | assert_eq!( |
| 688 | delta.len(), |
| 689 | total - dropped, |
| 690 | "only bytes still held can be delivered" |
| 691 | ); |
| 692 | |
| 693 | append(&buffer, b"tail"); |
| 694 | let (delta, _) = take_delta_from_buffer(&buffer, &mut cursor); |
| 695 | assert_eq!( |
| 696 | delta, |
| 697 | b"tail".to_vec(), |
| 698 | "subsequent deltas continue from the corrected cursor" |
| 699 | ); |
| 700 | } |
| 701 | |
| 702 | #[test] |
| 703 | fn tail_reports_the_stream_total_not_the_retained_length() { |
| 704 | let buffer = Arc::new(Mutex::new(RawOutputBuffer::new())); |
| 705 | append(&buffer, b"abcdefghij"); |
| 706 | buffer.lock().unwrap().release_to_tail(4); |
| 707 | let (total, tail) = tail_from_buffer(&buffer, 100); |
| 708 | assert_eq!(total, 10, "stdout_len must not shrink when memory is freed"); |
| 709 | assert_eq!(tail, "ghij"); |
| 710 | } |
| 711 | |
| 712 | #[test] |
| 713 | fn abandoning_a_stream_releases_it_and_stops_the_reader() { |
| 714 | let mut buffer = RawOutputBuffer::new(); |
| 715 | assert!(buffer.append(&[b'a'; 5_000]), "a live stream keeps reading"); |
| 716 | buffer.abandon(); |
| 717 | |
| 718 | assert_eq!(buffer.retained().len(), 0, "held bytes are released"); |
| 719 | assert_eq!( |
| 720 | buffer.total_len(), |
| 721 | 5_000, |
| 722 | "the stream's length survives the release" |
| 723 | ); |
| 724 | assert!( |
| 725 | !buffer.append(&[b'b'; 100]), |
| 726 | "an abandoned stream tells the reader thread to exit" |
| 727 | ); |
| 728 | assert_eq!(buffer.retained().len(), 0, "and retains nothing further"); |
| 729 | assert_eq!( |
| 730 | buffer.total_len(), |
| 731 | 5_100, |
| 732 | "bytes that arrive after the give-up are still counted, not hidden" |
| 733 | ); |
| 734 | } |
| 735 | |
| 736 | #[test] |
| 737 | fn raw_stream_ceiling_clears_every_downstream_bound() { |
| 738 | // The clip must be unreachable by the model-visible surfaces: the 30 KB |
| 739 | // result truncation, the 1,200-char job tail, the 1 KiB completion tail. |
| 740 | const { assert!(RAW_STREAM_MAX_BYTES > 30_000 * 100) }; |
| 741 | const { assert!(super::RAW_STREAM_SETTLED_TAIL_BYTES > 30_000) }; |
| 742 | } |
| 743 | |
| 744 | #[test] |
| 745 | fn bounded_output_keeps_last_two_thousand_complete_lines() { |
| 746 | let source = (0..=BOUNDED_OUTPUT_MAX_LINES) |
| 747 | .map(|index| format!("line-{index}")) |
| 748 | .collect::<Vec<_>>() |
| 749 | .join("\n"); |
| 750 | let mut output = BoundedOutputAccumulator::new_in(None); |
| 751 | output.append(source.as_bytes()).expect("append"); |
| 752 | output.finish().expect("finish"); |
| 753 | let snapshot = output.snapshot(true).expect("snapshot"); |
| 754 | assert!(snapshot.truncated); |
| 755 | assert!(snapshot.content.starts_with("line-1\n")); |
| 756 | assert!(snapshot.content.contains("Showing lines 2-2001 of 2001")); |
| 757 | } |
| 758 | |
| 759 | #[test] |
| 760 | fn bounded_output_streams_raw_full_output_and_bounds_decoded_tail() { |
| 761 | let raw = vec![0xFF; 2 * 1024 * 1024]; |
| 762 | let mut output = BoundedOutputAccumulator::new_in(None); |
| 763 | for chunk in raw.chunks(4_096) { |
| 764 | output.append(chunk).expect("append"); |
| 765 | assert!(output.retained_memory_bytes() <= BOUNDED_OUTPUT_MAX_BYTES + 4); |
| 766 | } |
| 767 | output.finish().expect("finish"); |
| 768 | let snapshot = output.snapshot(true).expect("snapshot"); |
| 769 | assert!(snapshot.truncated); |
| 770 | assert!(snapshot.retained_bytes <= BOUNDED_OUTPUT_MAX_BYTES); |
| 771 | assert!(snapshot.content.contains('\u{FFFD}')); |
| 772 | let path = output |
| 773 | .full_output_path() |
| 774 | .expect("full output") |
| 775 | .to_path_buf(); |
| 776 | assert_eq!(std::fs::read(&path).expect("read full output"), raw); |
| 777 | drop(output); |
| 778 | std::fs::remove_file(path).expect("remove full output"); |
| 779 | } |
| 780 | |
| 781 | #[test] |
| 782 | fn bounded_output_huge_terminal_line_matches_upstream_notice() { |
| 783 | let mut source = vec![b'x'; BOUNDED_OUTPUT_MAX_BYTES + 1_024]; |
| 784 | source.push(b'\n'); |
| 785 | let mut output = BoundedOutputAccumulator::new_in(None); |
| 786 | output.append(&source).expect("append"); |
| 787 | output.finish().expect("finish"); |
| 788 | let snapshot = output.snapshot(true).expect("snapshot"); |
| 789 | assert!(snapshot.content.contains("Showing last 50.0KB of line 1")); |
| 790 | assert!(snapshot.content.contains("line is 0B")); |
| 791 | let path = output |
| 792 | .full_output_path() |
| 793 | .expect("full output") |
| 794 | .to_path_buf(); |
| 795 | drop(output); |
| 796 | std::fs::remove_file(path).expect("remove full output"); |
| 797 | } |
| 798 | |
| 799 | #[test] |
| 800 | fn spill_failure_is_soft_and_names_the_reason() { |
| 801 | // A missing spill dir simulates a full or broken temp volume: the |
| 802 | // stream still runs, the tail is still delivered, and the notice says |
| 803 | // why "Full output: <path>" is absent instead of failing the command. |
| 804 | let missing = |
| 805 | std::env::temp_dir().join(format!("codewhale-missing-spill-{}", std::process::id())); |
| 806 | let mut output = BoundedOutputAccumulator::new_in(Some(&missing)); |
| 807 | let reason = output.spill_unavailable().expect("spill unavailable"); |
| 808 | assert!(!reason.is_empty(), "reason must name the io error"); |
| 809 | |
| 810 | output.append(b"ok\n").expect("append works without spill"); |
| 811 | output.finish().expect("finish works without spill"); |
| 812 | let short = output.snapshot(true).expect("snapshot"); |
| 813 | assert_eq!(short.content, "ok\n"); |
| 814 | assert!(!short.truncated); |
| 815 | assert!(output.full_output_path().is_none()); |
| 816 | |
| 817 | let source = (0..=BOUNDED_OUTPUT_MAX_LINES) |
| 818 | .map(|index| format!("line-{index}")) |
| 819 | .collect::<Vec<_>>() |
| 820 | .join("\n"); |
| 821 | let mut output = BoundedOutputAccumulator::new_in(Some(&missing)); |
| 822 | output.append(source.as_bytes()).expect("append"); |
| 823 | output.finish().expect("finish"); |
| 824 | let snapshot = output.snapshot(true).expect("snapshot"); |
| 825 | assert!(snapshot.truncated); |
| 826 | assert!(snapshot.content.starts_with("line-1\n")); |
| 827 | assert!( |
| 828 | snapshot.content.contains("Full output was not persisted:"), |
| 829 | "{}", |
| 830 | snapshot.content |
| 831 | ); |
| 832 | assert!(!snapshot.content.contains("Full output: ")); |
| 833 | assert!(output.full_output_path().is_none()); |
| 834 | } |
| 835 | |
| 836 | #[test] |
| 837 | fn resource_exhaustion_hint_names_disk_descriptors_and_memory() { |
| 838 | use std::io::{Error, ErrorKind}; |
| 839 | assert!( |
| 840 | super::resource_exhaustion_hint(&Error::from(ErrorKind::StorageFull)) |
| 841 | .expect("storage full") |
| 842 | .contains("disk") |
| 843 | ); |
| 844 | assert!( |
| 845 | super::resource_exhaustion_hint(&Error::from(ErrorKind::OutOfMemory)) |
| 846 | .expect("oom") |
| 847 | .contains("memory") |
| 848 | ); |
| 849 | #[cfg(unix)] |
| 850 | { |
| 851 | assert!( |
| 852 | super::resource_exhaustion_hint(&Error::from_raw_os_error(libc::ENOSPC)) |
| 853 | .expect("enospc") |
| 854 | .contains("disk") |
| 855 | ); |
| 856 | assert!( |
| 857 | super::resource_exhaustion_hint(&Error::from_raw_os_error(libc::EMFILE)) |
| 858 | .expect("emfile") |
| 859 | .contains("file descriptors") |
| 860 | ); |
| 861 | assert!( |
| 862 | super::resource_exhaustion_hint(&Error::from_raw_os_error(libc::EAGAIN)) |
| 863 | .expect("eagain") |
| 864 | .contains("retry") |
| 865 | ); |
| 866 | } |
| 867 | assert!(super::resource_exhaustion_hint(&Error::from(ErrorKind::NotFound)).is_none()); |
| 868 | assert!( |
| 869 | super::resource_exhaustion_hint(&Error::from(ErrorKind::PermissionDenied)).is_none() |
| 870 | ); |
| 871 | } |
| 872 | } |
| 873 |