A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
Keep head and tail of python_repl cell output (#3526)
Sebastian Müller5241d87de55c
diff --git a/crates/pa-core/src/kernel/manager/events.rs b/crates/pa-core/src/kernel/manager/events.rsindex 79c915a6cf..0679131aa3 100644--- a/crates/pa-core/src/kernel/manager/events.rs+++ b/crates/pa-core/src/kernel/manager/events.rs@@ -8,7 +8,6 @@ use super::{BASH_ACTIVITY_DISPLAY_MIME, BASH_COMMAND_DISPLAY_MIME, DIFF_DISPLAY_MIME,MAX_ATTACHMENT_DATA_CHARS, MAX_BACKGROUND_OUTPUT_CHARS,};-use std::fmt::Write as _;impl Inner {pub(crate) fn handle_event(self: &Arc<Self>, event: Event) {@@ -158,30 +157,9 @@ impl Inner {return;};let mut buffers = lock(&execution.buffers);- let ExecBuffers {- stdout,- stdout_chars,- stdout_truncated,- stderr,- stderr_chars,- stderr_truncated,- ..- } = &mut *buffers;match stream {- StreamName::Stdout => append_truncated(- stdout,- stdout_truncated,- stdout_chars,- text,- execution.max_chars,- ),- StreamName::Stderr => append_truncated(- stderr,- stderr_truncated,- stderr_chars,- text,- execution.max_chars,- ),+ StreamName::Stdout => buffers.stdout.push(text, execution.max_chars),+ StreamName::Stderr => buffers.stderr.push(text, execution.max_chars),}drop(buffers);if let Some(on_stream) = &execution.opts.on_stream {@@ -220,13 +198,14 @@ impl Inner {Some(Err(_)) => {buffers.attachment_oversized = true;if !buffers.stderr.is_empty() {- buffers.stderr.push('\n');+ buffers.stderr.push("\n", execution.max_chars);}- let _ = write!(- buffers.stderr,- "attachment dropped: exceeds {MAX_ATTACHMENT_DATA_CHARS} base64 chars"+ buffers.stderr.push(+ &format!(+ "attachment dropped: exceeds {MAX_ATTACHMENT_DATA_CHARS} base64 chars"+ ),+ execution.max_chars,);- buffers.stderr_chars = buffers.stderr.chars().count();buffers.status = ExecuteStatus::Error;}Some(Ok(attachment)) => buffers.attachments.push(attachment),diff --git a/crates/pa-core/src/kernel/manager/execution.rs b/crates/pa-core/src/kernel/manager/execution.rsindex 72268446d2..71fcd12457 100644--- a/crates/pa-core/src/kernel/manager/execution.rs+++ b/crates/pa-core/src/kernel/manager/execution.rs@@ -166,26 +166,10 @@ impl Inner {self.register_late_sent_agent_message_handler(&execution.request_id, callback);}- let mut stdout = std::mem::take(&mut buffers.stdout);- buffers.stdout_chars = 0;- let mut stderr = std::mem::take(&mut buffers.stderr);- buffers.stderr_chars = 0;+ let stdout = std::mem::take(&mut buffers.stdout).finish(execution.max_chars);+ let stderr = std::mem::take(&mut buffers.stderr).finish(execution.max_chars);let mut result = buffers.result.take();let mut status = buffers.status;- if buffers.stdout_truncated {- let _ = write!(- stdout,- "\n[... output truncated at {} chars ...]",- execution.max_chars- );- }- if buffers.stderr_truncated {- let _ = write!(- stderr,- "\n[... output truncated at {} chars ...]",- execution.max_chars- );- }if let Some(text) = &result {if let Some((end, _)) = text.char_indices().nth(execution.max_chars) {// Match the Unicode-scalar limit used for stdout and stderr.diff --git a/crates/pa-core/src/kernel/manager/mod.rs b/crates/pa-core/src/kernel/manager/mod.rsindex 86316e7d9e..6c89a41209 100644--- a/crates/pa-core/src/kernel/manager/mod.rs+++ b/crates/pa-core/src/kernel/manager/mod.rs@@ -83,15 +83,9 @@ struct ExitInfo {/// Fields of a settled execution shared with the stdout reader task.#[derive(Default)]-// The mirrored TS API shape is deliberate (the booleans are the product's own surface).-#[allow(clippy::struct_excessive_bools)]struct ExecBuffers {- stdout: String,- stderr: String,- stdout_chars: usize,- stderr_chars: usize,- stdout_truncated: bool,- stderr_truncated: bool,+ stdout: HeadTailBuffer,+ stderr: HeadTailBuffer,result: Option<String>,diffs: Vec<KernelDiffDisplay>,attachments: Vec<KernelAttachment>,@@ -415,7 +409,7 @@ mod snapshot;mod startup;mod teardown;-use requests::{append_truncated, describe_failure, Signal};+use requests::{append_truncated, describe_failure, HeadTailBuffer, Signal};// Public APIdiff --git a/crates/pa-core/src/kernel/manager/requests.rs b/crates/pa-core/src/kernel/manager/requests.rsindex 487526d6bf..e00f594bcc 100644--- a/crates/pa-core/src/kernel/manager/requests.rs+++ b/crates/pa-core/src/kernel/manager/requests.rs@@ -42,6 +42,122 @@ pub(crate) fn append_truncated(}}+/// A bounded cell-stream buffer that keeps the head AND the tail of its text.+/// Python prints its verdicts last — tracebacks, summaries, exit status — so a+/// failing run loses exactly its outcome to a head-only cap.+///+/// The first three quarters of the stream's cap fill the head; later text+/// rolls through a tail window of the last quarter, and the dropped middle is+/// counted for the marker. Every cut goes through `char_indices`/char counts,+/// so both cut points sit on char boundaries by construction.+#[derive(Default)]+pub(crate) struct HeadTailBuffer {+ head: String,+ head_chars: usize,+ tail: String,+ tail_chars: usize,+ /// Chars dropped from the middle, named by [`Self::finish`].+ omitted: usize,+}++impl HeadTailBuffer {+ /// The stream's head and tail budgets within `max_chars`: the tail gets a+ /// quarter, the head the rest. The one owner of the split.+ fn split_cap(max_chars: usize) -> (usize, usize) {+ let tail_cap = max_chars / 4;+ (max_chars - tail_cap, tail_cap)+ }++ /// Whether any text was kept.+ pub(crate) fn is_empty(&self) -> bool {+ self.head.is_empty() && self.tail.is_empty()+ }++ /// Buffer stream text within `max_chars`: fill the head, then roll the+ /// tail window at its cap, counting the chars that fall out of it.+ pub(crate) fn push(&mut self, text: &str, max_chars: usize) {+ let (head_cap, tail_cap) = Self::split_cap(max_chars);+ if self.head_chars < head_cap {+ let remaining = head_cap - self.head_chars;+ let text_chars = text.chars().count();+ if text_chars <= remaining {+ self.head.push_str(text);+ self.head_chars += text_chars;+ return;+ }+ let end = text+ .char_indices()+ .nth(remaining)+ .map_or(text.len(), |(i, _)| i);+ self.head.push_str(&text[..end]);+ self.head_chars = head_cap;+ self.push_tail(&text[end..], tail_cap);+ } else {+ self.push_tail(text, tail_cap);+ }+ }++ /// Roll text into the tail window. A frame at or over the window's size+ /// displaces the old tail in place — buffering such a frame whole (then+ /// draining it) would keep its allocation resident until settle. Smaller+ /// frames accumulate, compacting the front once the window grows past+ /// twice its cap (amortized linear, bounded memory).+ fn push_tail(&mut self, text: &str, tail_cap: usize) {+ let text_chars = text.chars().count();+ if text_chars == 0 {+ return;+ }+ if text_chars >= tail_cap {+ self.omitted += self.tail_chars + text_chars - tail_cap;+ // Keep the window's worth of the frame's END. The None case is+ // tail_cap == 0: keep nothing.+ let keep_from = text+ .char_indices()+ .nth(text_chars - tail_cap)+ .map_or(text.len(), |(i, _)| i);+ self.tail.clear();+ self.tail.push_str(&text[keep_from..]);+ self.tail_chars = tail_cap;+ return;+ }+ self.tail.push_str(text);+ self.tail_chars += text_chars;+ if self.tail_chars > 2 * tail_cap {+ self.drop_tail_front(self.tail_chars - tail_cap);+ }+ }++ /// Drop the first `chars` chars off the tail, counting them omitted.+ fn drop_tail_front(&mut self, chars: usize) {+ let end = self+ .tail+ .char_indices()+ .nth(chars)+ .map_or(self.tail.len(), |(i, _)| i);+ self.tail.drain(..end);+ self.tail_chars -= chars;+ self.omitted += chars;+ }++ /// Resolve the buffered text: trim the tail to its cap, then return head,+ /// marker, and tail. Text that fit the cap passes through unchanged.+ pub(crate) fn finish(mut self, max_chars: usize) -> String {+ let (_, tail_cap) = Self::split_cap(max_chars);+ if self.tail_chars > tail_cap {+ self.drop_tail_front(self.tail_chars - tail_cap);+ }+ if self.omitted == 0 {+ let mut text = self.head;+ text.push_str(&self.tail);+ return text;+ }+ format!(+ "{}\n[... {} chars omitted ...]\n{}",+ self.head, self.omitted, self.tail+ )+ }+}+pub(crate) fn describe_failure(result: &ExecuteResult) -> String {if let Some(error) = &result.error {if error.evalue.is_empty() {@@ -325,7 +441,7 @@ impl ReplKernelManager {#[cfg(test)]mod tests {- use super::append_truncated;+ use super::{append_truncated, HeadTailBuffer};#[cfg(unix)]#[tokio::test]@@ -449,6 +565,85 @@ for line in sys.stdin:manager.kill();}+ /// A cell's streamed output past the cap keeps the head and the tail: a+ /// long run's printed verdict (tracebacks, summaries, exit status arrive+ /// last) must survive the buffer, not just the first `max_chars` chars.+ #[cfg(unix)]+ #[tokio::test]+ async fn long_cell_output_keeps_head_and_tail() {+ use super::*;+ use std::os::unix::fs::PermissionsExt;++ // Protocol v3: ready, then one execute answered by stdout frames of+ // filler past the cap (multi-byte text, so cut points can land inside+ // a char run) with a printed traceback LAST, then done.+ const TRACEBACK_RUNTIME: &str = r#"#!/usr/bin/env python3+import json+import sys++print(json.dumps({"event": "ready", "protocol": 3, "python": "3.13.0"}), flush=True)+for line in sys.stdin:+ try:+ req = json.loads(line)+ except Exception:+ continue+ if req.get("type") == "execute":+ rid = req.get("id")+ filler = "é🍁xyz" * 20_000+ for start in range(0, len(filler), 4_096):+ chunk = filler[start:start + 4_096]+ print(json.dumps({"event": "stdout", "id": rid, "text": chunk}), flush=True)+ traceback = "Traceback (most recent call last):\n ...\nAssertionError: t1-traceback-sentinel\n"+ print(json.dumps({"event": "stdout", "id": rid, "text": traceback}), flush=True)+ print(json.dumps({"event": "done", "id": rid, "status": "ok"}), flush=True)+"#;++ let dir = tempfile::TempDir::new().expect("temp dir");+ let python = dir.path().join("traceback-kernel");+ std::fs::write(&python, TRACEBACK_RUNTIME).expect("write fake kernel");+ std::fs::set_permissions(&python, std::fs::Permissions::from_mode(0o755))+ .expect("chmod fake kernel");+ let manager = ReplKernelManager::new(crate::kernel::shared::KernelManagerOptions {+ python: Some(python),+ ..Default::default()+ });+ manager+ .start(KernelStartOptions::default())+ .await+ .expect("ready handshake");+ let result = tokio::time::timeout(+ Duration::from_secs(10),+ manager.execute_bounded("1+1", ExecuteOptions::default(), None),+ )+ .await+ .expect("the execute must settle")+ .expect("execute returns a result");+ assert_eq!(result.status, ExecuteStatus::Ok);+ let stdout = result.stdout;+ let filler = "é🍁xyz".repeat(20_000);+ let traceback =+ "Traceback (most recent call last):\n ...\nAssertionError: t1-traceback-sentinel\n";+ assert!(+ stdout.starts_with("é🍁xyz"),+ "the head must keep the stream's first chars"+ );+ assert!(+ stdout.ends_with(traceback),+ "the tail must keep the verdict printed last"+ );+ let omitted = filler.chars().count() + traceback.chars().count() - DEFAULT_MAX_OUTPUT_CHARS;+ let marker = format!("\n[... {omitted} chars omitted ...]\n");+ let marker_at = stdout+ .find(&marker)+ .expect("the dropped middle must be named by a marker");+ assert!(+ marker_at + marker.len() <= stdout.len() - traceback.len(),+ "the marker must sit between head and tail"+ );+ assert!(stdout.chars().count() <= DEFAULT_MAX_OUTPUT_CHARS + marker.chars().count());+ manager.kill();+ }+#[test]fn capped_stream_matches_original_unicode_and_frame_semantics() {let frames = ["", "a", "é", "🍁", "xy", "é🍁abc", "", "tail"];@@ -502,4 +697,62 @@ for line in sys.stdin:append_truncated(&mut empty, &mut never, &mut zero, "", 4);assert!(!never);}++ #[test]+ fn head_tail_buffer_matches_oracle() {+ let long_z = "z".repeat(40);+ let long_multi = "é🍁".repeat(30);+ // Multi-byte frames long enough to roll the tail window several times.+ let frames = [+ "",+ "a",+ "é",+ "🍁",+ "xy",+ "é🍁abc",+ long_z.as_str(),+ "",+ "tail",+ long_multi.as_str(),+ ];+ for cap in [0usize, 1, 2, 3, 4, 7, 8, 16] {+ let mut buffer = HeadTailBuffer::default();+ let mut all = String::new();+ for text in frames {+ buffer.push(text, cap);+ all.push_str(text);+ }+ let expected = oracle(&all, cap);+ assert_eq!(buffer.finish(cap), expected, "cap {cap}");+ }+ }++ /// The oracle: verbatim when everything fits, else the first+ /// `head_cap` chars, the named elision, and the last `tail_cap` chars.+ /// The ratio itself comes from `split_cap`, its one owner.+ fn oracle(all: &str, cap: usize) -> String {+ let (head_cap, tail_cap) = HeadTailBuffer::split_cap(cap);+ let chars: Vec<char> = all.chars().collect();+ if chars.len() <= cap {+ return all.to_string();+ }+ let head: String = chars[..head_cap].iter().collect();+ let tail: String = chars[chars.len() - tail_cap..].iter().collect();+ let omitted = chars.len() - head_cap - tail_cap;+ format!("{head}\n[... {omitted} chars omitted ...]\n{tail}")+ }++ #[test]+ fn one_oversized_frame_keeps_the_tail_window_small() {+ // `print(big_string)` arrives as ONE stdout frame; buffering it whole+ // would keep the frame's allocation resident until settle.+ let frame = "é".repeat(1_000_000);+ let mut buffer = HeadTailBuffer::default();+ buffer.push(&frame, 16);+ assert!(+ buffer.tail.capacity() < 1_024,+ "the tail window must not retain the frame's allocation"+ );+ assert_eq!(buffer.finish(16), oracle(&frame, 16));+ }}diff --git a/crates/pa-core/src/kernel/shared.rs b/crates/pa-core/src/kernel/shared.rsindex 9a284546cb..302d007d3c 100644--- a/crates/pa-core/src/kernel/shared.rs+++ b/crates/pa-core/src/kernel/shared.rs@@ -9,6 +9,8 @@ use serde_json::Value;use crate::kernel::bootstrap::KernelPythonSkill;+/// Per-cell stdout/stderr cap. `HeadTailBuffer` splits it between head and+/// tail, so a long failing run keeps its printed verdict.pub const DEFAULT_MAX_OUTPUT_CHARS: usize = 65_536;pub const HOST_REQUEST_SHUTDOWN_TIMEOUT_MS: u64 = 5_000;pub const KERNEL_SHUTDOWN_TIMEOUT_MS: u64 = 5_000;