Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

decode ! and gate pipe output across 8 KiB read boundaries (#3765)

kta414c1fc4088
diff --git a/.changes/incremental-utf8-pipe-decode.md b/.changes/incremental-utf8-pipe-decode.md
new file mode 100644
index 0000000000..e0b38bdade
--- /dev/null
+++ b/.changes/incremental-utf8-pipe-decode.md
@@ -0,0 +1 @@
+- `!` command output and autonomous gate output no longer turn a multi-byte character split across two pipe reads into replacement characters, and gate output past the 1 MiB cap no longer panics when the cap lands inside a character.
diff --git a/crates/pa-core/src/autonomous/gates.rs b/crates/pa-core/src/autonomous/gates.rs
index c0e7a8052a..5adcd75b62 100644
--- a/crates/pa-core/src/autonomous/gates.rs
+++ b/crates/pa-core/src/autonomous/gates.rs
@@ -14,7 +14,7 @@ use super::{
/// Output cap for gate results surfaced in continuation text.
const MAX_GATE_OUTPUT_CHARS: usize = 6000;
/// Output cap for gate child processes (both streams combined per stream).
-const MAX_CHILD_PROCESS_OUTPUT_CHARS: usize = 1024 * 1024;
+const MAX_CHILD_PROCESS_OUTPUT_BYTES: usize = 1024 * 1024;
/// Timeout for the git commands that capture a worktree snapshot.
const SNAPSHOT_TIMEOUT_MS: u64 = 10_000;
@@ -457,12 +457,13 @@ pub async fn run_gate_command(
run_child_process(&shell.shell, &args, cwd, timeout_ms).await
}
-/// Read one output pipe to EOF, keeping at most `cap` characters; the
-/// remainder is drained so a full pipe never blocks the child.
+/// Read one output pipe to EOF, keeping at most `cap` bytes of text; the
+/// remainder is drained so a full pipe never blocks the child. The bytes are
+/// decoded once at the end, so a character split across two reads survives.
async fn read_pipe_capped<R: tokio::io::AsyncRead + Unpin>(mut pipe: Option<R>) -> (String, bool) {
use tokio::io::AsyncReadExt;
- let cap = MAX_CHILD_PROCESS_OUTPUT_CHARS;
- let mut kept = String::new();
+ let cap = MAX_CHILD_PROCESS_OUTPUT_BYTES;
+ let mut kept: Vec<u8> = Vec::new();
let mut truncated = false;
let mut buffer = [0u8; 8192];
if let Some(pipe) = pipe.as_mut() {
@@ -470,23 +471,23 @@ async fn read_pipe_capped<R: tokio::io::AsyncRead + Unpin>(mut pipe: Option<R>)
match pipe.read(&mut buffer).await {
Ok(0) | Err(_) => break,
Ok(read) => {
- let chunk = String::from_utf8_lossy(&buffer[..read]);
- if kept.len() < cap {
- let remaining = cap - kept.len();
- if chunk.len() <= remaining {
- kept.push_str(&chunk);
- } else {
- kept.push_str(&chunk[..remaining]);
- truncated = true;
- }
- } else {
+ let room = cap.saturating_sub(kept.len());
+ if read > room {
truncated = true;
}
+ kept.extend_from_slice(&buffer[..read.min(room)]);
}
}
}
}
- (kept, truncated)
+ if truncated {
+ // The cap may have cut the last character short; drop its partial bytes.
+ if let Some(last) = kept.utf8_chunks().last() {
+ let partial = last.invalid().len();
+ kept.truncate(kept.len() - partial);
+ }
+ }
+ (String::from_utf8_lossy(&kept).into_owned(), truncated)
}
#[cfg(test)]
@@ -711,4 +712,23 @@ mod tests {
let digest = hash_untracked_path(&dir.path().join("untracked.txt"));
assert!(digest.starts_with("file:"));
}
+
+ #[tokio::test]
+ async fn a_character_split_across_pipe_reads_is_kept_whole() {
+ let output = format!("{}\u{e9} done", "a".repeat(8191));
+ let captured =
+ read_pipe_capped(Some(std::io::Cursor::new(output.clone().into_bytes()))).await;
+ assert_eq!(captured, (output, false));
+ }
+
+ #[tokio::test]
+ async fn the_cap_never_cuts_inside_a_character() {
+ let mut output = "a".repeat(MAX_CHILD_PROCESS_OUTPUT_BYTES - 1);
+ output.push_str("\u{e9}\u{e9}");
+ let captured = read_pipe_capped(Some(std::io::Cursor::new(output.into_bytes()))).await;
+ assert_eq!(
+ captured,
+ ("a".repeat(MAX_CHILD_PROCESS_OUTPUT_BYTES - 1), true)
+ );
+ }
}
diff --git a/crates/pa-daemon/src/lib.rs b/crates/pa-daemon/src/lib.rs
index f3403019ff..89778350ca 100644
--- a/crates/pa-daemon/src/lib.rs
+++ b/crates/pa-daemon/src/lib.rs
@@ -107,6 +107,7 @@ pub mod socket;
pub(crate) mod state_getters;
mod stop_cleanup;
pub(crate) mod streaming;
+pub(crate) mod streaming_utf8;
pub mod supervisor;
pub mod supervisor_link;
pub(crate) mod supervisor_lost;
diff --git a/crates/pa-daemon/src/streaming_utf8.rs b/crates/pa-daemon/src/streaming_utf8.rs
new file mode 100644
index 0000000000..b1cf2997b7
--- /dev/null
+++ b/crates/pa-daemon/src/streaming_utf8.rs
@@ -0,0 +1,101 @@
+//! Streaming UTF-8 decoding for the `!` command runner's pipe reads, so a
+//! character split across two reads decodes whole (TS `TextDecoder`
+//! `{ stream: true }`).
+
+/// Streaming UTF-8 decoder matching `TextDecoder` `{ stream: true }`: an
+/// incomplete trailing sequence is buffered for the next chunk, and invalid
+/// bytes become one U+FFFD per maximal subpart (the `String::from_utf8_lossy`
+/// rule), so an overlong, surrogate or out-of-range sequence such as
+/// `ED A0 80` yields one U+FFFD per byte. One decoder per byte stream: feed each
+/// read to [`Self::decode_chunk`], then [`Self::flush`] once at end of stream.
+#[derive(Default)]
+pub(crate) struct StreamingUtf8 {
+ buffer: Vec<u8>,
+}
+
+impl StreamingUtf8 {
+ /// Decode the complete characters buffered so far plus `data`; a trailing
+ /// incomplete sequence waits for the next chunk.
+ pub(crate) fn decode_chunk(&mut self, data: &[u8]) -> String {
+ self.buffer.extend_from_slice(data);
+ let (text, consumed) = decode_lossy_prefix(&self.buffer);
+ self.buffer.drain(..consumed);
+ text
+ }
+
+ /// End of stream: decode whatever is left, a truncated trailing sequence
+ /// as one U+FFFD.
+ pub(crate) fn flush(&mut self) -> String {
+ let buffer = std::mem::take(&mut self.buffer);
+ let (mut text, consumed) = decode_lossy_prefix(&buffer);
+ if consumed < buffer.len() {
+ text.push('\u{FFFD}');
+ }
+ text
+ }
+}
+
+/// Lossily decode `bytes` up to a trailing incomplete sequence, returning the
+/// text and how many bytes it consumed; the unconsumed tail is a valid prefix
+/// of a character that more bytes may complete.
+fn decode_lossy_prefix(bytes: &[u8]) -> (String, usize) {
+ let mut text = String::with_capacity(bytes.len());
+ let mut rest = bytes;
+ loop {
+ match std::str::from_utf8(rest) {
+ Ok(valid) => {
+ text.push_str(valid);
+ return (text, bytes.len());
+ }
+ Err(error) => {
+ let (valid, after) = rest.split_at(error.valid_up_to());
+ // `valid_up_to` marks the longest valid prefix, so this never fails.
+ text.push_str(std::str::from_utf8(valid).unwrap_or_default());
+ match error.error_len() {
+ Some(invalid) => {
+ text.push('\u{FFFD}');
+ rest = &after[invalid..];
+ }
+ None => return (text, bytes.len() - after.len()),
+ }
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::StreamingUtf8;
+
+ /// Feeding `bytes` one byte at a time and flushing decodes exactly what
+ /// `String::from_utf8_lossy` (`TextDecoder`'s maximal-subpart rule) does for
+ /// the whole input.
+ fn assert_matches_lossy(bytes: &[u8]) {
+ let expected = String::from_utf8_lossy(bytes).into_owned();
+ let mut whole = StreamingUtf8::default();
+ let mut whole_text = whole.decode_chunk(bytes);
+ whole_text.push_str(&whole.flush());
+ assert_eq!(whole_text, expected, "one chunk: {bytes:02X?}");
+ let mut split = StreamingUtf8::default();
+ let mut split_text: String = bytes
+ .iter()
+ .map(|byte| split.decode_chunk(std::slice::from_ref(byte)))
+ .collect();
+ split_text.push_str(&split.flush());
+ assert_eq!(split_text, expected, "byte by byte: {bytes:02X?}");
+ }
+
+ #[test]
+ fn split_chunks_and_a_truncated_tail_decode_like_one_input() {
+ for bytes in [
+ b"a\xED\xA0\x80\xE0\x80\xAFb".as_slice(),
+ "\u{20ac}\u{1F600}".as_bytes(),
+ b"x\xE2\x82".as_slice(),
+ ] {
+ assert_matches_lossy(bytes);
+ }
+ let mut decoder = StreamingUtf8::default();
+ assert_eq!(decoder.decode_chunk(b"x\xE2\x82"), "x");
+ assert_eq!(decoder.flush(), "\u{FFFD}");
+ }
+}
diff --git a/crates/pa-daemon/src/user_bash.rs b/crates/pa-daemon/src/user_bash.rs
index ce61b32f28..c77b52d605 100644
--- a/crates/pa-daemon/src/user_bash.rs
+++ b/crates/pa-daemon/src/user_bash.rs
@@ -532,8 +532,8 @@ impl OutputStream {
self.clone()
}
- fn push(&mut self, data: &[u8]) {
- let text = sanitize_output(&String::from_utf8_lossy(data));
+ fn push(&mut self, data: &str) {
+ let text = sanitize_output(data);
if text.is_empty() {
return;
}
@@ -628,16 +628,24 @@ async fn run_bash(run: RunBash<'_>) -> BashEnd {
}));
// stdout and stderr merge into one stream (TS wires both pipes to the
// same `onData`); the shared accumulator keeps the arrival order.
+ // A read can end inside a multi-byte character; each pipe's decoder holds its leading
+ // bytes for the next read (TS `TextDecoder`, `stream: true`).
let pump = |mut reader: Box<dyn AsyncRead + Unpin + Send>| {
let stream = Arc::clone(&stream);
async move {
+ let mut decoder = crate::streaming_utf8::StreamingUtf8::default();
let mut buffer = [0u8; 8192];
loop {
match reader.read(&mut buffer).await {
Ok(0) | Err(_) => break,
- Ok(read) => stream.lock().unwrap().push(&buffer[..read]),
+ Ok(read) => {
+ let text = decoder.decode_chunk(&buffer[..read]);
+ stream.lock().unwrap().push(&text);
+ }
}
}
+ let rest = decoder.flush();
+ stream.lock().unwrap().push(&rest);
}
};
let stdout_pump = stdout.map(|pipe| pump(Box::new(pipe)));
@@ -1035,6 +1043,29 @@ mod tests {
);
}
+ /// Multi-byte characters that straddle a pipe read decode intact: a per-read decode replaced
+ /// both halves with U+FFFD in the stream and the row the model reads.
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn characters_split_across_reads_decode_intact() {
+ let cwd = tempfile::tempdir().expect("tempdir");
+ let worker = created_worker(cwd.path()).await;
+ // 1,500 seven-byte lines cross an 8 KiB read without reaching either output cap.
+ let response = worker
+ .dispatch(
+ "execute_bash_and_wait",
+ &json!({
+ "activeSessionId": "bash-session",
+ "command": "yes '\u{20ac}\u{20ac}' | head -n 1500",
+ }),
+ )
+ .await;
+ assert!(response.success, "failed: {response:?}");
+ let data = response.data.expect("data");
+ let output = data["output"].as_str().expect("output");
+ assert_eq!(output, "\u{20ac}\u{20ac}\n".repeat(1500));
+ }
+
/// A command whose shell dies by a signal reports 128 + the signal number, like a shell's
/// `$?` (TS `normalizedExitCode`): with no exit code the model would read it as a success.
#[cfg(unix)]