A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
decode ! and gate pipe output across 8 KiB read boundaries (#3765)
kta414c1fc4088
diff --git a/.changes/incremental-utf8-pipe-decode.md b/.changes/incremental-utf8-pipe-decode.mdnew file mode 100644index 0000000000..e0b38bdade--- /dev/null+++ b/.changes/incremental-utf8-pipe-decode.md@@ -0,0 +1 @@+- `!` command output and autonomous gate output no longer turn a multi-byte character split across two pipe reads into replacement characters, and gate output past the 1 MiB cap no longer panics when the cap lands inside a character.diff --git a/crates/pa-core/src/autonomous/gates.rs b/crates/pa-core/src/autonomous/gates.rsindex c0e7a8052a..5adcd75b62 100644--- a/crates/pa-core/src/autonomous/gates.rs+++ b/crates/pa-core/src/autonomous/gates.rs@@ -14,7 +14,7 @@ use super::{/// Output cap for gate results surfaced in continuation text.const MAX_GATE_OUTPUT_CHARS: usize = 6000;/// Output cap for gate child processes (both streams combined per stream).-const MAX_CHILD_PROCESS_OUTPUT_CHARS: usize = 1024 * 1024;+const MAX_CHILD_PROCESS_OUTPUT_BYTES: usize = 1024 * 1024;/// Timeout for the git commands that capture a worktree snapshot.const SNAPSHOT_TIMEOUT_MS: u64 = 10_000;@@ -457,12 +457,13 @@ pub async fn run_gate_command(run_child_process(&shell.shell, &args, cwd, timeout_ms).await}-/// Read one output pipe to EOF, keeping at most `cap` characters; the-/// remainder is drained so a full pipe never blocks the child.+/// Read one output pipe to EOF, keeping at most `cap` bytes of text; the+/// remainder is drained so a full pipe never blocks the child. The bytes are+/// decoded once at the end, so a character split across two reads survives.async fn read_pipe_capped<R: tokio::io::AsyncRead + Unpin>(mut pipe: Option<R>) -> (String, bool) {use tokio::io::AsyncReadExt;- let cap = MAX_CHILD_PROCESS_OUTPUT_CHARS;- let mut kept = String::new();+ let cap = MAX_CHILD_PROCESS_OUTPUT_BYTES;+ let mut kept: Vec<u8> = Vec::new();let mut truncated = false;let mut buffer = [0u8; 8192];if let Some(pipe) = pipe.as_mut() {@@ -470,23 +471,23 @@ async fn read_pipe_capped<R: tokio::io::AsyncRead + Unpin>(mut pipe: Option<R>)match pipe.read(&mut buffer).await {Ok(0) | Err(_) => break,Ok(read) => {- let chunk = String::from_utf8_lossy(&buffer[..read]);- if kept.len() < cap {- let remaining = cap - kept.len();- if chunk.len() <= remaining {- kept.push_str(&chunk);- } else {- kept.push_str(&chunk[..remaining]);- truncated = true;- }- } else {+ let room = cap.saturating_sub(kept.len());+ if read > room {truncated = true;}+ kept.extend_from_slice(&buffer[..read.min(room)]);}}}}- (kept, truncated)+ if truncated {+ // The cap may have cut the last character short; drop its partial bytes.+ if let Some(last) = kept.utf8_chunks().last() {+ let partial = last.invalid().len();+ kept.truncate(kept.len() - partial);+ }+ }+ (String::from_utf8_lossy(&kept).into_owned(), truncated)}#[cfg(test)]@@ -711,4 +712,23 @@ mod tests {let digest = hash_untracked_path(&dir.path().join("untracked.txt"));assert!(digest.starts_with("file:"));}++ #[tokio::test]+ async fn a_character_split_across_pipe_reads_is_kept_whole() {+ let output = format!("{}\u{e9} done", "a".repeat(8191));+ let captured =+ read_pipe_capped(Some(std::io::Cursor::new(output.clone().into_bytes()))).await;+ assert_eq!(captured, (output, false));+ }++ #[tokio::test]+ async fn the_cap_never_cuts_inside_a_character() {+ let mut output = "a".repeat(MAX_CHILD_PROCESS_OUTPUT_BYTES - 1);+ output.push_str("\u{e9}\u{e9}");+ let captured = read_pipe_capped(Some(std::io::Cursor::new(output.into_bytes()))).await;+ assert_eq!(+ captured,+ ("a".repeat(MAX_CHILD_PROCESS_OUTPUT_BYTES - 1), true)+ );+ }}diff --git a/crates/pa-daemon/src/lib.rs b/crates/pa-daemon/src/lib.rsindex f3403019ff..89778350ca 100644--- a/crates/pa-daemon/src/lib.rs+++ b/crates/pa-daemon/src/lib.rs@@ -107,6 +107,7 @@ pub mod socket;pub(crate) mod state_getters;mod stop_cleanup;pub(crate) mod streaming;+pub(crate) mod streaming_utf8;pub mod supervisor;pub mod supervisor_link;pub(crate) mod supervisor_lost;diff --git a/crates/pa-daemon/src/streaming_utf8.rs b/crates/pa-daemon/src/streaming_utf8.rsnew file mode 100644index 0000000000..b1cf2997b7--- /dev/null+++ b/crates/pa-daemon/src/streaming_utf8.rs@@ -0,0 +1,101 @@+//! Streaming UTF-8 decoding for the `!` command runner's pipe reads, so a+//! character split across two reads decodes whole (TS `TextDecoder`+//! `{ stream: true }`).++/// Streaming UTF-8 decoder matching `TextDecoder` `{ stream: true }`: an+/// incomplete trailing sequence is buffered for the next chunk, and invalid+/// bytes become one U+FFFD per maximal subpart (the `String::from_utf8_lossy`+/// rule), so an overlong, surrogate or out-of-range sequence such as+/// `ED A0 80` yields one U+FFFD per byte. One decoder per byte stream: feed each+/// read to [`Self::decode_chunk`], then [`Self::flush`] once at end of stream.+#[derive(Default)]+pub(crate) struct StreamingUtf8 {+ buffer: Vec<u8>,+}++impl StreamingUtf8 {+ /// Decode the complete characters buffered so far plus `data`; a trailing+ /// incomplete sequence waits for the next chunk.+ pub(crate) fn decode_chunk(&mut self, data: &[u8]) -> String {+ self.buffer.extend_from_slice(data);+ let (text, consumed) = decode_lossy_prefix(&self.buffer);+ self.buffer.drain(..consumed);+ text+ }++ /// End of stream: decode whatever is left, a truncated trailing sequence+ /// as one U+FFFD.+ pub(crate) fn flush(&mut self) -> String {+ let buffer = std::mem::take(&mut self.buffer);+ let (mut text, consumed) = decode_lossy_prefix(&buffer);+ if consumed < buffer.len() {+ text.push('\u{FFFD}');+ }+ text+ }+}++/// Lossily decode `bytes` up to a trailing incomplete sequence, returning the+/// text and how many bytes it consumed; the unconsumed tail is a valid prefix+/// of a character that more bytes may complete.+fn decode_lossy_prefix(bytes: &[u8]) -> (String, usize) {+ let mut text = String::with_capacity(bytes.len());+ let mut rest = bytes;+ loop {+ match std::str::from_utf8(rest) {+ Ok(valid) => {+ text.push_str(valid);+ return (text, bytes.len());+ }+ Err(error) => {+ let (valid, after) = rest.split_at(error.valid_up_to());+ // `valid_up_to` marks the longest valid prefix, so this never fails.+ text.push_str(std::str::from_utf8(valid).unwrap_or_default());+ match error.error_len() {+ Some(invalid) => {+ text.push('\u{FFFD}');+ rest = &after[invalid..];+ }+ None => return (text, bytes.len() - after.len()),+ }+ }+ }+ }+}++#[cfg(test)]+mod tests {+ use super::StreamingUtf8;++ /// Feeding `bytes` one byte at a time and flushing decodes exactly what+ /// `String::from_utf8_lossy` (`TextDecoder`'s maximal-subpart rule) does for+ /// the whole input.+ fn assert_matches_lossy(bytes: &[u8]) {+ let expected = String::from_utf8_lossy(bytes).into_owned();+ let mut whole = StreamingUtf8::default();+ let mut whole_text = whole.decode_chunk(bytes);+ whole_text.push_str(&whole.flush());+ assert_eq!(whole_text, expected, "one chunk: {bytes:02X?}");+ let mut split = StreamingUtf8::default();+ let mut split_text: String = bytes+ .iter()+ .map(|byte| split.decode_chunk(std::slice::from_ref(byte)))+ .collect();+ split_text.push_str(&split.flush());+ assert_eq!(split_text, expected, "byte by byte: {bytes:02X?}");+ }++ #[test]+ fn split_chunks_and_a_truncated_tail_decode_like_one_input() {+ for bytes in [+ b"a\xED\xA0\x80\xE0\x80\xAFb".as_slice(),+ "\u{20ac}\u{1F600}".as_bytes(),+ b"x\xE2\x82".as_slice(),+ ] {+ assert_matches_lossy(bytes);+ }+ let mut decoder = StreamingUtf8::default();+ assert_eq!(decoder.decode_chunk(b"x\xE2\x82"), "x");+ assert_eq!(decoder.flush(), "\u{FFFD}");+ }+}diff --git a/crates/pa-daemon/src/user_bash.rs b/crates/pa-daemon/src/user_bash.rsindex ce61b32f28..c77b52d605 100644--- a/crates/pa-daemon/src/user_bash.rs+++ b/crates/pa-daemon/src/user_bash.rs@@ -532,8 +532,8 @@ impl OutputStream {self.clone()}- fn push(&mut self, data: &[u8]) {- let text = sanitize_output(&String::from_utf8_lossy(data));+ fn push(&mut self, data: &str) {+ let text = sanitize_output(data);if text.is_empty() {return;}@@ -628,16 +628,24 @@ async fn run_bash(run: RunBash<'_>) -> BashEnd {}));// stdout and stderr merge into one stream (TS wires both pipes to the// same `onData`); the shared accumulator keeps the arrival order.+ // A read can end inside a multi-byte character; each pipe's decoder holds its leading+ // bytes for the next read (TS `TextDecoder`, `stream: true`).let pump = |mut reader: Box<dyn AsyncRead + Unpin + Send>| {let stream = Arc::clone(&stream);async move {+ let mut decoder = crate::streaming_utf8::StreamingUtf8::default();let mut buffer = [0u8; 8192];loop {match reader.read(&mut buffer).await {Ok(0) | Err(_) => break,- Ok(read) => stream.lock().unwrap().push(&buffer[..read]),+ Ok(read) => {+ let text = decoder.decode_chunk(&buffer[..read]);+ stream.lock().unwrap().push(&text);+ }}}+ let rest = decoder.flush();+ stream.lock().unwrap().push(&rest);}};let stdout_pump = stdout.map(|pipe| pump(Box::new(pipe)));@@ -1035,6 +1043,29 @@ mod tests {);}+ /// Multi-byte characters that straddle a pipe read decode intact: a per-read decode replaced+ /// both halves with U+FFFD in the stream and the row the model reads.+ #[cfg(unix)]+ #[tokio::test]+ async fn characters_split_across_reads_decode_intact() {+ let cwd = tempfile::tempdir().expect("tempdir");+ let worker = created_worker(cwd.path()).await;+ // 1,500 seven-byte lines cross an 8 KiB read without reaching either output cap.+ let response = worker+ .dispatch(+ "execute_bash_and_wait",+ &json!({+ "activeSessionId": "bash-session",+ "command": "yes '\u{20ac}\u{20ac}' | head -n 1500",+ }),+ )+ .await;+ assert!(response.success, "failed: {response:?}");+ let data = response.data.expect("data");+ let output = data["output"].as_str().expect("output");+ assert_eq!(output, "\u{20ac}\u{20ac}\n".repeat(1500));+ }+/// A command whose shell dies by a signal reports 128 + the signal number, like a shell's/// `$?` (TS `normalizedExitCode`): with no exit code the model would read it as a success.#[cfg(unix)]