Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

Keep head and tail of python_repl cell output (#3526)

Sebastian Müller5241d87de55c
diff --git a/crates/pa-core/src/kernel/manager/events.rs b/crates/pa-core/src/kernel/manager/events.rs
index 79c915a6cf..0679131aa3 100644
--- a/crates/pa-core/src/kernel/manager/events.rs
+++ b/crates/pa-core/src/kernel/manager/events.rs
@@ -8,7 +8,6 @@ use super::{
BASH_ACTIVITY_DISPLAY_MIME, BASH_COMMAND_DISPLAY_MIME, DIFF_DISPLAY_MIME,
MAX_ATTACHMENT_DATA_CHARS, MAX_BACKGROUND_OUTPUT_CHARS,
};
-use std::fmt::Write as _;
impl Inner {
pub(crate) fn handle_event(self: &Arc<Self>, event: Event) {
@@ -158,30 +157,9 @@ impl Inner {
return;
};
let mut buffers = lock(&execution.buffers);
- let ExecBuffers {
- stdout,
- stdout_chars,
- stdout_truncated,
- stderr,
- stderr_chars,
- stderr_truncated,
- ..
- } = &mut *buffers;
match stream {
- StreamName::Stdout => append_truncated(
- stdout,
- stdout_truncated,
- stdout_chars,
- text,
- execution.max_chars,
- ),
- StreamName::Stderr => append_truncated(
- stderr,
- stderr_truncated,
- stderr_chars,
- text,
- execution.max_chars,
- ),
+ StreamName::Stdout => buffers.stdout.push(text, execution.max_chars),
+ StreamName::Stderr => buffers.stderr.push(text, execution.max_chars),
}
drop(buffers);
if let Some(on_stream) = &execution.opts.on_stream {
@@ -220,13 +198,14 @@ impl Inner {
Some(Err(_)) => {
buffers.attachment_oversized = true;
if !buffers.stderr.is_empty() {
- buffers.stderr.push('\n');
+ buffers.stderr.push("\n", execution.max_chars);
}
- let _ = write!(
- buffers.stderr,
- "attachment dropped: exceeds {MAX_ATTACHMENT_DATA_CHARS} base64 chars"
+ buffers.stderr.push(
+ &format!(
+ "attachment dropped: exceeds {MAX_ATTACHMENT_DATA_CHARS} base64 chars"
+ ),
+ execution.max_chars,
);
- buffers.stderr_chars = buffers.stderr.chars().count();
buffers.status = ExecuteStatus::Error;
}
Some(Ok(attachment)) => buffers.attachments.push(attachment),
diff --git a/crates/pa-core/src/kernel/manager/execution.rs b/crates/pa-core/src/kernel/manager/execution.rs
index 72268446d2..71fcd12457 100644
--- a/crates/pa-core/src/kernel/manager/execution.rs
+++ b/crates/pa-core/src/kernel/manager/execution.rs
@@ -166,26 +166,10 @@ impl Inner {
self.register_late_sent_agent_message_handler(&execution.request_id, callback);
}
- let mut stdout = std::mem::take(&mut buffers.stdout);
- buffers.stdout_chars = 0;
- let mut stderr = std::mem::take(&mut buffers.stderr);
- buffers.stderr_chars = 0;
+ let stdout = std::mem::take(&mut buffers.stdout).finish(execution.max_chars);
+ let stderr = std::mem::take(&mut buffers.stderr).finish(execution.max_chars);
let mut result = buffers.result.take();
let mut status = buffers.status;
- if buffers.stdout_truncated {
- let _ = write!(
- stdout,
- "\n[... output truncated at {} chars ...]",
- execution.max_chars
- );
- }
- if buffers.stderr_truncated {
- let _ = write!(
- stderr,
- "\n[... output truncated at {} chars ...]",
- execution.max_chars
- );
- }
if let Some(text) = &result {
if let Some((end, _)) = text.char_indices().nth(execution.max_chars) {
// Match the Unicode-scalar limit used for stdout and stderr.
diff --git a/crates/pa-core/src/kernel/manager/mod.rs b/crates/pa-core/src/kernel/manager/mod.rs
index 86316e7d9e..6c89a41209 100644
--- a/crates/pa-core/src/kernel/manager/mod.rs
+++ b/crates/pa-core/src/kernel/manager/mod.rs
@@ -83,15 +83,9 @@ struct ExitInfo {
/// Fields of a settled execution shared with the stdout reader task.
#[derive(Default)]
-// The mirrored TS API shape is deliberate (the booleans are the product's own surface).
-#[allow(clippy::struct_excessive_bools)]
struct ExecBuffers {
- stdout: String,
- stderr: String,
- stdout_chars: usize,
- stderr_chars: usize,
- stdout_truncated: bool,
- stderr_truncated: bool,
+ stdout: HeadTailBuffer,
+ stderr: HeadTailBuffer,
result: Option<String>,
diffs: Vec<KernelDiffDisplay>,
attachments: Vec<KernelAttachment>,
@@ -415,7 +409,7 @@ mod snapshot;
mod startup;
mod teardown;
-use requests::{append_truncated, describe_failure, Signal};
+use requests::{append_truncated, describe_failure, HeadTailBuffer, Signal};
// Public API
diff --git a/crates/pa-core/src/kernel/manager/requests.rs b/crates/pa-core/src/kernel/manager/requests.rs
index 487526d6bf..e00f594bcc 100644
--- a/crates/pa-core/src/kernel/manager/requests.rs
+++ b/crates/pa-core/src/kernel/manager/requests.rs
@@ -42,6 +42,122 @@ pub(crate) fn append_truncated(
}
}
+/// A bounded cell-stream buffer that keeps the head AND the tail of its text.
+/// Python prints its verdicts last — tracebacks, summaries, exit status — so a
+/// failing run loses exactly its outcome to a head-only cap.
+///
+/// The first three quarters of the stream's cap fill the head; later text
+/// rolls through a tail window of the last quarter, and the dropped middle is
+/// counted for the marker. Every cut goes through `char_indices`/char counts,
+/// so both cut points sit on char boundaries by construction.
+#[derive(Default)]
+pub(crate) struct HeadTailBuffer {
+ head: String,
+ head_chars: usize,
+ tail: String,
+ tail_chars: usize,
+ /// Chars dropped from the middle, named by [`Self::finish`].
+ omitted: usize,
+}
+
+impl HeadTailBuffer {
+ /// The stream's head and tail budgets within `max_chars`: the tail gets a
+ /// quarter, the head the rest. The one owner of the split.
+ fn split_cap(max_chars: usize) -> (usize, usize) {
+ let tail_cap = max_chars / 4;
+ (max_chars - tail_cap, tail_cap)
+ }
+
+ /// Whether any text was kept.
+ pub(crate) fn is_empty(&self) -> bool {
+ self.head.is_empty() && self.tail.is_empty()
+ }
+
+ /// Buffer stream text within `max_chars`: fill the head, then roll the
+ /// tail window at its cap, counting the chars that fall out of it.
+ pub(crate) fn push(&mut self, text: &str, max_chars: usize) {
+ let (head_cap, tail_cap) = Self::split_cap(max_chars);
+ if self.head_chars < head_cap {
+ let remaining = head_cap - self.head_chars;
+ let text_chars = text.chars().count();
+ if text_chars <= remaining {
+ self.head.push_str(text);
+ self.head_chars += text_chars;
+ return;
+ }
+ let end = text
+ .char_indices()
+ .nth(remaining)
+ .map_or(text.len(), |(i, _)| i);
+ self.head.push_str(&text[..end]);
+ self.head_chars = head_cap;
+ self.push_tail(&text[end..], tail_cap);
+ } else {
+ self.push_tail(text, tail_cap);
+ }
+ }
+
+ /// Roll text into the tail window. A frame at or over the window's size
+ /// displaces the old tail in place — buffering such a frame whole (then
+ /// draining it) would keep its allocation resident until settle. Smaller
+ /// frames accumulate, compacting the front once the window grows past
+ /// twice its cap (amortized linear, bounded memory).
+ fn push_tail(&mut self, text: &str, tail_cap: usize) {
+ let text_chars = text.chars().count();
+ if text_chars == 0 {
+ return;
+ }
+ if text_chars >= tail_cap {
+ self.omitted += self.tail_chars + text_chars - tail_cap;
+ // Keep the window's worth of the frame's END. The None case is
+ // tail_cap == 0: keep nothing.
+ let keep_from = text
+ .char_indices()
+ .nth(text_chars - tail_cap)
+ .map_or(text.len(), |(i, _)| i);
+ self.tail.clear();
+ self.tail.push_str(&text[keep_from..]);
+ self.tail_chars = tail_cap;
+ return;
+ }
+ self.tail.push_str(text);
+ self.tail_chars += text_chars;
+ if self.tail_chars > 2 * tail_cap {
+ self.drop_tail_front(self.tail_chars - tail_cap);
+ }
+ }
+
+ /// Drop the first `chars` chars off the tail, counting them omitted.
+ fn drop_tail_front(&mut self, chars: usize) {
+ let end = self
+ .tail
+ .char_indices()
+ .nth(chars)
+ .map_or(self.tail.len(), |(i, _)| i);
+ self.tail.drain(..end);
+ self.tail_chars -= chars;
+ self.omitted += chars;
+ }
+
+ /// Resolve the buffered text: trim the tail to its cap, then return head,
+ /// marker, and tail. Text that fit the cap passes through unchanged.
+ pub(crate) fn finish(mut self, max_chars: usize) -> String {
+ let (_, tail_cap) = Self::split_cap(max_chars);
+ if self.tail_chars > tail_cap {
+ self.drop_tail_front(self.tail_chars - tail_cap);
+ }
+ if self.omitted == 0 {
+ let mut text = self.head;
+ text.push_str(&self.tail);
+ return text;
+ }
+ format!(
+ "{}\n[... {} chars omitted ...]\n{}",
+ self.head, self.omitted, self.tail
+ )
+ }
+}
+
pub(crate) fn describe_failure(result: &ExecuteResult) -> String {
if let Some(error) = &result.error {
if error.evalue.is_empty() {
@@ -325,7 +441,7 @@ impl ReplKernelManager {
#[cfg(test)]
mod tests {
- use super::append_truncated;
+ use super::{append_truncated, HeadTailBuffer};
#[cfg(unix)]
#[tokio::test]
@@ -449,6 +565,85 @@ for line in sys.stdin:
manager.kill();
}
+ /// A cell's streamed output past the cap keeps the head and the tail: a
+ /// long run's printed verdict (tracebacks, summaries, exit status arrive
+ /// last) must survive the buffer, not just the first `max_chars` chars.
+ #[cfg(unix)]
+ #[tokio::test]
+ async fn long_cell_output_keeps_head_and_tail() {
+ use super::*;
+ use std::os::unix::fs::PermissionsExt;
+
+ // Protocol v3: ready, then one execute answered by stdout frames of
+ // filler past the cap (multi-byte text, so cut points can land inside
+ // a char run) with a printed traceback LAST, then done.
+ const TRACEBACK_RUNTIME: &str = r#"#!/usr/bin/env python3
+import json
+import sys
+
+print(json.dumps({"event": "ready", "protocol": 3, "python": "3.13.0"}), flush=True)
+for line in sys.stdin:
+ try:
+ req = json.loads(line)
+ except Exception:
+ continue
+ if req.get("type") == "execute":
+ rid = req.get("id")
+ filler = "é🍁xyz" * 20_000
+ for start in range(0, len(filler), 4_096):
+ chunk = filler[start:start + 4_096]
+ print(json.dumps({"event": "stdout", "id": rid, "text": chunk}), flush=True)
+ traceback = "Traceback (most recent call last):\n ...\nAssertionError: t1-traceback-sentinel\n"
+ print(json.dumps({"event": "stdout", "id": rid, "text": traceback}), flush=True)
+ print(json.dumps({"event": "done", "id": rid, "status": "ok"}), flush=True)
+"#;
+
+ let dir = tempfile::TempDir::new().expect("temp dir");
+ let python = dir.path().join("traceback-kernel");
+ std::fs::write(&python, TRACEBACK_RUNTIME).expect("write fake kernel");
+ std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o755))
+ .expect("chmod fake kernel");
+ let manager = ReplKernelManager::new(crate::kernel::shared::KernelManagerOptions {
+ python: Some(python),
+ ..Default::default()
+ });
+ manager
+ .start(KernelStartOptions::default())
+ .await
+ .expect("ready handshake");
+ let result = tokio::time::timeout(
+ Duration::from_secs(10),
+ manager.execute_bounded("1+1", ExecuteOptions::default(), None),
+ )
+ .await
+ .expect("the execute must settle")
+ .expect("execute returns a result");
+ assert_eq!(result.status, ExecuteStatus::Ok);
+ let stdout = result.stdout;
+ let filler = "é🍁xyz".repeat(20_000);
+ let traceback =
+ "Traceback (most recent call last):\n ...\nAssertionError: t1-traceback-sentinel\n";
+ assert!(
+ stdout.starts_with("é🍁xyz"),
+ "the head must keep the stream's first chars"
+ );
+ assert!(
+ stdout.ends_with(traceback),
+ "the tail must keep the verdict printed last"
+ );
+ let omitted = filler.chars().count() + traceback.chars().count() - DEFAULT_MAX_OUTPUT_CHARS;
+ let marker = format!("\n[... {omitted} chars omitted ...]\n");
+ let marker_at = stdout
+ .find(&marker)
+ .expect("the dropped middle must be named by a marker");
+ assert!(
+ marker_at + marker.len() <= stdout.len() - traceback.len(),
+ "the marker must sit between head and tail"
+ );
+ assert!(stdout.chars().count() <= DEFAULT_MAX_OUTPUT_CHARS + marker.chars().count());
+ manager.kill();
+ }
+
#[test]
fn capped_stream_matches_original_unicode_and_frame_semantics() {
let frames = ["", "a", "é", "🍁", "xy", "é🍁abc", "", "tail"];
@@ -502,4 +697,62 @@ for line in sys.stdin:
append_truncated(&mut empty, &mut never, &mut zero, "", 4);
assert!(!never);
}
+
+ #[test]
+ fn head_tail_buffer_matches_oracle() {
+ let long_z = "z".repeat(40);
+ let long_multi = "é🍁".repeat(30);
+ // Multi-byte frames long enough to roll the tail window several times.
+ let frames = [
+ "",
+ "a",
+ "é",
+ "🍁",
+ "xy",
+ "é🍁abc",
+ long_z.as_str(),
+ "",
+ "tail",
+ long_multi.as_str(),
+ ];
+ for cap in [0usize, 1, 2, 3, 4, 7, 8, 16] {
+ let mut buffer = HeadTailBuffer::default();
+ let mut all = String::new();
+ for text in frames {
+ buffer.push(text, cap);
+ all.push_str(text);
+ }
+ let expected = oracle(&all, cap);
+ assert_eq!(buffer.finish(cap), expected, "cap {cap}");
+ }
+ }
+
+ /// The oracle: verbatim when everything fits, else the first
+ /// `head_cap` chars, the named elision, and the last `tail_cap` chars.
+ /// The ratio itself comes from `split_cap`, its one owner.
+ fn oracle(all: &str, cap: usize) -> String {
+ let (head_cap, tail_cap) = HeadTailBuffer::split_cap(cap);
+ let chars: Vec<char> = all.chars().collect();
+ if chars.len() <= cap {
+ return all.to_string();
+ }
+ let head: String = chars[..head_cap].iter().collect();
+ let tail: String = chars[chars.len() - tail_cap..].iter().collect();
+ let omitted = chars.len() - head_cap - tail_cap;
+ format!("{head}\n[... {omitted} chars omitted ...]\n{tail}")
+ }
+
+ #[test]
+ fn one_oversized_frame_keeps_the_tail_window_small() {
+ // `print(big_string)` arrives as ONE stdout frame; buffering it whole
+ // would keep the frame's allocation resident until settle.
+ let frame = "é".repeat(1_000_000);
+ let mut buffer = HeadTailBuffer::default();
+ buffer.push(&frame, 16);
+ assert!(
+ buffer.tail.capacity() < 1_024,
+ "the tail window must not retain the frame's allocation"
+ );
+ assert_eq!(buffer.finish(16), oracle(&frame, 16));
+ }
}
diff --git a/crates/pa-core/src/kernel/shared.rs b/crates/pa-core/src/kernel/shared.rs
index 9a284546cb..302d007d3c 100644
--- a/crates/pa-core/src/kernel/shared.rs
+++ b/crates/pa-core/src/kernel/shared.rs
@@ -9,6 +9,8 @@ use serde_json::Value;
use crate::kernel::bootstrap::KernelPythonSkill;
+/// Per-cell stdout/stderr cap. `HeadTailBuffer` splits it between head and
+/// tail, so a long failing run keeps its printed verdict.
pub const DEFAULT_MAX_OUTPUT_CHARS: usize = 65_536;
pub const HOST_REQUEST_SHUTDOWN_TIMEOUT_MS: u64 = 5_000;
pub const KERNEL_SHUTDOWN_TIMEOUT_MS: u64 = 5_000;