A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
//! The kernel's persistent `bash()` shells (prime-agent-runtime `rlm/bash.py`,
//! `bash(..., persistent=True)`): every session's shells live in one home,
//! `<agent dir>/persistent-shells/`, as a `<id>.json` record assigned to the
//! session that started it, plus the shell's `<id>.out` and `<id>.rc`. The
//! daemon reads the records to list each saved session's live shells and every
//! shell machine-wide, reads a shell's output, and kills one on request.
//! Nothing in the session lifecycle kills them: a deleted session's shells keep
//! running as orphans, still assigned to it.
use std::collections::HashMap;
use std::io::{Read, Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant, UNIX_EPOCH};
use pa_core::platform::process::{process_group_exists, signal_process_group, Signal};
use serde::Deserialize;
use serde_json::{json, Value};
use crate::protocol::{response_failure, response_line, response_success};
const HOME_DIR: &str = "persistent-shells";
/// How long a shell may exit on its own after SIGTERM before its group gets SIGKILL.
const TERM_GRACE: Duration = Duration::from_secs(2);
/// The kernel activity rows cap the command at the same length.
const COMMAND_CAP: usize = 512;
/// The output read serves its lines from at most this many trailing bytes.
const TAIL_BYTES: u64 = 64 * 1024;
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
struct Shell {
id: String,
pid: u32,
start_id: String,
command: String,
started_at: String,
session_id: String,
}
fn shell_file(agent_dir: &Path, id: &str, kind: &str) -> PathBuf {
agent_dir.join(HOME_DIR).join(format!("{id}.{kind}"))
}
fn registered_shells(agent_dir: &Path) -> Vec<Shell> {
let Ok(entries) = std::fs::read_dir(agent_dir.join(HOME_DIR)) else {
return Vec::new();
};
let mut shells: Vec<Shell> = entries
.flatten()
.map(|entry| entry.path())
.filter(|path| path.extension().is_some_and(|ext| ext == "json"))
.filter_map(|path| std::fs::read(path).ok())
.filter_map(|raw| serde_json::from_slice(&raw).ok())
.collect();
shells.sort_by(|a, b| a.started_at.cmp(&b.started_at));
shells
}
fn find_shell(agent_dir: &Path, id: &str) -> Option<Shell> {
registered_shells(agent_dir)
.into_iter()
.find(|shell| shell.id == id)
}
/// Whether the shell's pid still runs under the recorded start identity.
fn runs_as_registered(shell: &Shell) -> bool {
matches!(crate::lease::is_process_alive(shell.pid), Ok(true))
&& crate::lease::get_process_start_id(shell.pid).as_deref() == Some(shell.start_id.as_str())
}
/// An extant group with no verified leader is not safely attributable to this record.
fn group_unverified(shell: &Shell) -> bool {
!runs_as_registered(shell) && process_group_exists(shell.pid as i32).unwrap_or(true)
}
/// One shell as a row with the kernel bash activity row field names.
fn activity_row(shell: &Shell, status: &str) -> Value {
json!({
"id": shell.id,
"command": shell.command.chars().take(COMMAND_CAP).collect::<String>(),
"pid": shell.pid,
"startedAt": shell.started_at,
"status": status,
"persistent": true,
})
}
/// The live persistent shells by assigned session id: a recycled pid or an
/// unreadable identity never counts as live.
pub(crate) fn live_shell_rows_by_session(agent_dir: &Path) -> HashMap<String, Vec<Value>> {
let mut rows: HashMap<String, Vec<Value>> = HashMap::new();
for shell in registered_shells(agent_dir)
.into_iter()
.filter(runs_as_registered)
{
let row = activity_row(&shell, "running");
rows.entry(shell.session_id).or_default().push(row);
}
rows
}
/// Every registered shell with its assigned `sessionId`: running while its
/// leader runs as registered, unverified when a group remains without that
/// proof, otherwise finished with its recorded exit code (if any). Its
/// `durationMs` runs from `startedAt` to now, or to the exit code's write
/// once finished.
pub(crate) fn shell_rows(agent_dir: &Path) -> Vec<Value> {
registered_shells(agent_dir)
.iter()
.map(|shell| {
let running = runs_as_registered(shell);
let status = if running {
"running"
} else if group_unverified(shell) {
"unverified"
} else {
"finished"
};
let mut row = activity_row(shell, status);
row["sessionId"] = json!(shell.session_id);
let rc = shell_file(agent_dir, &shell.id, "rc");
let ended_ms = if status == "finished" {
row["exitCode"] = json!(std::fs::read_to_string(&rc)
.ok()
.and_then(|rc| rc.trim().parse::<i64>().ok()));
std::fs::metadata(&rc)
.and_then(|metadata| metadata.modified())
.ok()
.and_then(|modified| modified.duration_since(UNIX_EPOCH).ok())
.and_then(|since| u64::try_from(since.as_millis()).ok())
} else {
Some(crate::util::now_ms())
};
row["durationMs"] = json!(crate::util::iso_to_unix_ms(&shell.started_at)
.zip(ended_ms)
.map(|(started, ended)| ended.saturating_sub(started)));
row
})
.collect()
}
/// The newest `lines` lines of the shell's output (read from its last
/// [`TAIL_BYTES`]); `None` for an unknown shell. The kernel creates the `.out`
/// before it writes the record, so a failed read is an error, never an empty tail.
pub(crate) fn output_tail(
agent_dir: &Path,
id: &str,
lines: usize,
) -> Result<Option<String>, String> {
let Some(shell) = find_shell(agent_dir, id) else {
return Ok(None);
};
let bytes = read_tail(&shell_file(agent_dir, &shell.id, "out"))
.map_err(|error| format!("Could not read persistent shell output: {error}"))?;
let text = String::from_utf8_lossy(&bytes);
let all: Vec<&str> = text.lines().collect();
Ok(Some(all[all.len().saturating_sub(lines)..].join("\n")))
}
fn read_tail(path: &Path) -> std::io::Result<Vec<u8>> {
let mut file = std::fs::File::open(path)?;
let size = file.metadata()?.len();
file.seek(SeekFrom::Start(size.saturating_sub(TAIL_BYTES)))?;
let mut bytes = Vec::new();
file.take(TAIL_BYTES).read_to_end(&mut bytes)?;
Ok(bytes)
}
/// Kill a shell only while the recorded leader identity still proves ownership.
/// A durable leaderless group may belong to a reused PGID: refuse and retain it.
pub(crate) fn kill_shell(agent_dir: &Path, id: &str) -> Result<bool, String> {
let Some(shell) = find_shell(agent_dir, id) else {
return Ok(false);
};
if !runs_as_registered(&shell) {
return if group_unverified(&shell) {
Err("Cannot verify persistent shell group ownership; no signal sent".to_string())
} else {
Ok(true)
};
}
if !signal_process_group(shell.pid as i32, Signal::Term) {
return Err("Could not signal persistent shell group".to_string());
}
// The grace belongs to the whole group: a leader that exits on TERM must not cut short
// its members' own TERM handling.
let deadline = Instant::now() + TERM_GRACE;
while Instant::now() < deadline {
if !process_group_exists(shell.pid as i32).unwrap_or(true) {
return Ok(true);
}
std::thread::sleep(Duration::from_millis(50));
}
kill_group(&shell)?;
Ok(true)
}
/// The KILL after a verified TERM. A leader that exited during the grace still names the group
/// verified before the TERM: a PGID is not reused while its group has members. A leader pid
/// that runs again under another start identity was recycled, so its group is refused.
fn kill_group(shell: &Shell) -> Result<(), String> {
let leader_exited = matches!(crate::lease::is_process_alive(shell.pid), Ok(false));
if !runs_as_registered(shell) && !leader_exited {
return if group_unverified(shell) {
Err(
"Cannot verify persistent shell group ownership; no further signal sent"
.to_string(),
)
} else {
Ok(())
};
}
if signal_process_group(shell.pid as i32, Signal::Kill)
|| !process_group_exists(shell.pid as i32).unwrap_or(true)
{
Ok(())
} else {
Err("Could not signal persistent shell group".to_string())
}
}
/// One persistent-shell supervisor arm, run off the runtime (file reads and
/// the kill's grace block): `None` answers an unknown shell.
pub(crate) async fn respond(
command_id: &str,
type_name: &str,
run: impl FnOnce() -> Result<Option<Value>, String> + Send + 'static,
) -> Value {
let response = match tokio::task::spawn_blocking(run).await {
Ok(Ok(Some(data))) => response_success(Some(command_id), type_name, Some(data)),
Ok(Ok(None)) => response_failure(
Some(command_id),
type_name,
"Unknown persistent shell",
None,
),
Ok(Err(error)) => response_failure(Some(command_id), type_name, &error, None),
Err(error) => response_failure(Some(command_id), type_name, &error.to_string(), None),
};
response_line(&response)
}
#[cfg(all(test, unix))]
mod test_support {
use std::path::Path;
use std::process::{Child, Command};
/// A setsid'd `sleep` registered in `agent_dir`'s persistent-shell home,
/// assigned to `session_id`, under `start_id` (`None`: its real start identity).
pub(super) fn registered_sleeper(
agent_dir: &Path,
session_id: &str,
start_id: Option<&str>,
) -> Child {
let mut command = Command::new("sleep");
command.arg("60");
pa_core::platform::process::set_new_session(&mut command);
let child = command.spawn().expect("spawn sleeper");
let start_id = start_id.map_or_else(
|| crate::lease::get_process_start_id(child.id()).expect("start id"),
str::to_string,
);
register(agent_dir, child.id(), &start_id, session_id);
child
}
/// Write one record for shell `pid` under `start_id`, assigned to
/// `session_id`; returns its id.
pub(super) fn register(agent_dir: &Path, pid: u32, start_id: &str, session_id: &str) -> String {
let home = agent_dir.join(super::HOME_DIR);
std::fs::create_dir_all(&home).expect("persistent-shell home");
let id = format!("{pid:032}");
let record = serde_json::json!({
"id": id,
"pid": pid,
"startId": start_id,
"command": "sleep 60",
"startedAt": "2026-01-01T00:00:00+00:00",
"sessionId": session_id,
});
std::fs::write(home.join(format!("{id}.json")), record.to_string()).expect("write record");
id
}
}
#[cfg(all(test, unix))]
mod tests {
use std::os::unix::process::{CommandExt, ExitStatusExt};
use std::process::{Child, Command};
use nix::sys::signal::{kill, Signal as NixSignal};
use nix::unistd::Pid;
use super::test_support::{register, registered_sleeper};
use super::*;
/// Send `child` our own SIGTERM and return the signal it died by: an
/// earlier SIGKILL is already pending and wins.
fn fatal_signal_after_own_term(mut child: Child) -> Option<i32> {
let _ = kill(Pid::from_raw(child.id() as i32), NixSignal::SIGTERM);
child.wait().unwrap().signal()
}
/// A registered `sleep` leading its own process group, plus one member in
/// that group.
fn registered_group(agent_dir: &std::path::Path) -> (Child, Child, String) {
let mut leader_command = Command::new("sleep");
leader_command.arg("60");
pa_core::platform::process::set_new_process_group(&mut leader_command);
let leader = leader_command.spawn().unwrap();
let start_id = crate::lease::get_process_start_id(leader.id()).unwrap();
let id = register(agent_dir, leader.id(), &start_id, "s1");
let member = Command::new("sleep")
.arg("60")
.process_group(leader.id() as i32)
.spawn()
.unwrap();
(leader, member, id)
}
#[test]
fn recycled_pid_is_neither_reported_nor_killed() {
let root = tempfile::tempdir().unwrap();
let stranger = registered_sleeper(root.path(), "s1", Some("proc:not-this-process"));
let id = format!("{:032}", stranger.id());
assert!(live_shell_rows_by_session(root.path()).is_empty());
assert_eq!(shell_rows(root.path())[0]["status"], "unverified");
assert!(kill_shell(root.path(), &id).is_err());
assert_eq!(
fatal_signal_after_own_term(stranger),
Some(NixSignal::SIGTERM as i32),
"never signalled"
);
}
#[test]
fn kill_stops_the_whole_group_and_leaves_the_shell_listed_as_finished() {
let root = tempfile::tempdir().unwrap();
let (mut leader, member, id) = registered_group(root.path());
std::fs::write(shell_file(root.path(), &id, "out"), "line\n").unwrap();
let _outcome = kill_shell(root.path(), &id);
assert!(!leader.wait().unwrap().success(), "the leader was killed");
let member_status = member.wait_with_output().unwrap().status;
assert_eq!(
member_status.signal(),
Some(NixSignal::SIGTERM as i32),
"the group's TERM reached the member"
);
assert_eq!(shell_rows(root.path())[0]["status"], "finished");
assert_eq!(
output_tail(root.path(), &id, 1),
Ok(Some("line".to_string()))
);
assert_eq!(kill_shell(root.path(), "unknown"), Ok(false));
}
#[test]
fn kill_escalates_to_a_member_that_outlives_its_leaders_term() {
use std::io::BufRead;
let root = tempfile::tempdir().unwrap();
let (mut leader, member, id) = registered_group(root.path());
let mut stubborn = Command::new("sh")
.args(["-c", "trap '' TERM; echo ready; exec sleep 60"])
.process_group(leader.id() as i32)
.stdout(std::process::Stdio::piped())
.spawn()
.unwrap();
let mut ready = String::new();
std::io::BufReader::new(stubborn.stdout.take().unwrap())
.read_line(&mut ready)
.unwrap();
assert_eq!(kill_shell(root.path(), &id), Ok(true));
leader.wait().unwrap();
member.wait_with_output().unwrap();
assert_eq!(
stubborn.wait().unwrap().signal(),
Some(NixSignal::SIGKILL as i32),
"the member that ignored TERM got the group's KILL"
);
}
#[test]
fn a_leader_exiting_on_term_leaves_its_members_the_full_grace() {
use std::io::BufRead;
let root = tempfile::tempdir().unwrap();
let marker = root.path().join("cleaned-up");
let (leader, member, id) = registered_group(root.path());
let mut cleanup = Command::new("sh")
.args([
"-c",
"trap 'sleep 1; touch \"$0\"; exit 0' TERM; echo ready; while :; do sleep 0.05; done",
])
.arg(&marker)
.process_group(leader.id() as i32)
.stdout(std::process::Stdio::piped())
.spawn()
.unwrap();
let mut ready = String::new();
std::io::BufReader::new(cleanup.stdout.take().unwrap())
.read_line(&mut ready)
.unwrap();
// Reaped as the kernel reaps them: only live members keep the group.
let reapers: Vec<_> = [leader, member, cleanup]
.into_iter()
.map(|mut child| std::thread::spawn(move || child.wait().unwrap()))
.collect();
assert_eq!(kill_shell(root.path(), &id), Ok(true));
let statuses: Vec<_> = reapers.into_iter().map(|r| r.join().unwrap()).collect();
assert!(
marker.exists() && statuses[2].success(),
"the member finished its TERM cleanup after the leader exited: {statuses:?}"
);
}
#[test]
fn exited_leaders_group_is_unverified_and_never_signalled() {
let root = tempfile::tempdir().unwrap();
let (mut leader, member, id) = registered_group(root.path());
leader.kill().unwrap();
leader.wait().unwrap();
// The same observable state also describes an unrelated group after
// durable PID/PGID reuse. No old leader start-id can distinguish them.
let record = shell_file(root.path(), &id, "json");
let mut stale: Value = serde_json::from_slice(&std::fs::read(&record).unwrap()).unwrap();
stale["startId"] = json!("proc:old-unrelated-leader");
std::fs::write(&record, stale.to_string()).unwrap();
assert_eq!(shell_rows(root.path())[0]["status"], "unverified");
assert!(kill_shell(root.path(), &id).is_err());
assert!(record.exists());
assert_eq!(
fatal_signal_after_own_term(member),
Some(NixSignal::SIGTERM as i32),
"no earlier KILL reached the unverified group"
);
}
#[test]
fn pid_recycled_during_the_grace_is_not_killed() {
let root = tempfile::tempdir().unwrap();
let mut stranger = registered_sleeper(root.path(), "s1", Some("proc:not-this-process"));
let shells = registered_shells(root.path());
assert!(kill_group(&shells[0]).is_err());
assert!(signal_process_group(stranger.id() as i32, Signal::Term));
let status = stranger.wait().unwrap();
assert_eq!(
status.signal(),
Some(NixSignal::SIGTERM as i32),
"no SIGKILL came first"
);
}
#[test]
fn shell_rows_list_every_shell_with_its_session_and_the_output_tail_reads_the_newest_lines() {
let root = tempfile::tempdir().unwrap();
let mut sleeper = registered_sleeper(root.path(), "s1", None);
let mut done = Command::new("true").spawn().unwrap();
done.wait().unwrap();
let finished = register(root.path(), done.id(), "proc:0", "gone");
std::fs::write(shell_file(root.path(), &finished, "rc"), "3\n").unwrap();
std::fs::write(
shell_file(root.path(), &finished, "out"),
"one\ntwo\nthree\n",
)
.unwrap();
let rows = shell_rows(root.path());
let row = |id: &str| rows.iter().find(|row| row["id"] == id).unwrap().clone();
let running = row(&format!("{:032}", sleeper.id()));
assert_eq!(
(
&running["status"],
&running["sessionId"],
running.get("exitCode")
),
(&json!("running"), &json!("s1"), None)
);
let done_row = row(&finished);
assert_eq!(
(
&done_row["status"],
&done_row["sessionId"],
&done_row["exitCode"]
),
(&json!("finished"), &json!("gone"), &json!(3))
);
assert!(running["durationMs"].is_u64() && done_row["durationMs"].is_u64());
assert_eq!(
output_tail(root.path(), &finished, 2),
Ok(Some("two\nthree".to_string()))
);
assert_eq!(output_tail(root.path(), "unknown", 2), Ok(None));
sleeper.kill().unwrap();
sleeper.wait().unwrap();
}
#[test]
fn an_unreadable_output_file_is_an_error_not_an_empty_tail() {
let root = tempfile::tempdir().unwrap();
let mut sleeper = registered_sleeper(root.path(), "s1", None);
let id = format!("{:032}", sleeper.id());
std::fs::create_dir(shell_file(root.path(), &id, "out")).unwrap();
let tail = output_tail(root.path(), &id, 5);
sleeper.kill().unwrap();
sleeper.wait().unwrap();
assert!(tail.is_err(), "{tail:?}");
}
}