Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust
17.2 KB
Raw Blame
//! Persistent `bash()` shells across the daemon: a shell started with
//! `bash(..., persistent=True)` survives the daemon's shutdown (it is neither
//! journaled for the orphan reaper nor killed by the kernel's exit hook), shows
//! on the INACTIVE session's saved-catalog row, is re-attached by the kernel of
//! the reopened session, and keeps running after the session's delete, its
//! record still assigned to the deleted session. The machine-wide
//! `list_persistent_shells` lists every shell with its assignment, and
//! `kill_persistent_shell` ends an orphan. The live-kernel test skips (with a
//! note) without a kernel install.
#![cfg(target_os = "linux")]
// Narrowing casts sit at OS boundaries (pid) where the kernel bounds the values.
#![allow(clippy::cast_possible_wrap, clippy::cast_possible_truncation)]
// Style gate only, not correctness: the live test is one long scenario.
#![allow(clippy::too_many_lines)]

use std::io::{BufRead, BufReader, Write};
use std::os::unix::net::UnixStream;
use std::os::unix::process::CommandExt;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Stdio};
use std::time::{Duration, Instant};

use nix::sys::signal::{killpg, Signal};
use nix::unistd::Pid;
use serde_json::{json, Value};

fn kernel_python() -> Option<PathBuf> {
    if let Some(explicit) = std::env::var_os("PA_E2E_KERNEL_PYTHON") {
        return Some(PathBuf::from(explicit));
    }
    let candidate = PathBuf::from(std::env::var("HOME").unwrap_or_default())
        .join(".prime/agent/kernel-venv/bin/python");
    if candidate.exists() {
        return Some(candidate);
    }
    eprintln!(
        "kernel python {} not found; skipping live persistent-shell e2e",
        candidate.display()
    );
    None
}

struct Daemon {
    child: Child,
}

impl Drop for Daemon {
    fn drop(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

/// Persistent shells outlive their session: never leak one.
struct GroupGuard(i32);

impl Drop for GroupGuard {
    fn drop(&mut self) {
        let _ = killpg(Pid::from_raw(self.0), Signal::SIGKILL);
    }
}

/// Whether any non-zombie process is in group `pgid` (an unreaped zombie still
/// answers `kill(-pgid, 0)`).
fn group_alive(pgid: i32) -> bool {
    std::fs::read_dir("/proc")
        .expect("read /proc")
        .flatten()
        .filter_map(|entry| std::fs::read_to_string(entry.path().join("stat")).ok())
        .any(|stat| {
            let fields: Vec<&str> = stat[stat.rfind(')').map_or(0, |at| at + 2)..]
                .split(' ')
                .collect();
            fields.first() != Some(&"Z") && fields.get(2) == Some(&pgid.to_string().as_str())
        })
}

// The returned Daemon's Drop kills and reaps the child.
#[allow(clippy::zombie_processes)]
fn spawn_supervisor(socket: &Path, agent_dir: &Path, kernel_python: Option<&Path>) -> Daemon {
    std::fs::create_dir_all(agent_dir).expect("agent dir");
    let mut command = Command::new(env!("CARGO_BIN_EXE_pa-daemon"));
    command
        .arg("supervisor")
        .arg("--socket")
        .arg(socket)
        .arg("--agent-dir")
        .arg(agent_dir)
        .env("PRIME_AGENT_CODING_AGENT_DIR", agent_dir)
        .env_remove("RLM_SESSION_DIR")
        .env_remove("RLM_HARNESS_STATE_DIR")
        .env(
            pa_daemon::worker::WORKER_SUPERVISOR_LOST_EXIT_MS_ENV,
            "15000",
        )
        .stdout(Stdio::null())
        .stderr(Stdio::null());
    if let Some(python) = kernel_python {
        command.env("PRIME_AGENT_KERNEL_PYTHON", python);
    }
    let child = command.spawn().expect("spawn pa-daemon supervisor");
    let deadline = Instant::now() + Duration::from_secs(15);
    while UnixStream::connect(socket).is_err() {
        assert!(Instant::now() < deadline, "supervisor socket never came up");
        std::thread::sleep(Duration::from_millis(20));
    }
    Daemon { child }
}

struct Client {
    reader: BufReader<UnixStream>,
    writer: UnixStream,
}

impl Client {
    fn connect(socket: &Path) -> Self {
        let stream = UnixStream::connect(socket).expect("connect supervisor");
        let writer = stream.try_clone().expect("clone socket");
        stream
            .set_read_timeout(Some(Duration::from_millis(100)))
            .expect("set timeout");
        let mut client = Client {
            reader: BufReader::new(stream),
            writer,
        };
        let _hello = client.read_line();
        client
    }

    fn request(&mut self, id: &str, command: &Value) -> Value {
        let mut line = json!({
            "type": "command",
            "id": id,
            "protocol": { "name": "prime-agent.daemon", "version": 7 },
            "command": command,
        })
        .to_string();
        line.push('\n');
        self.writer.write_all(line.as_bytes()).expect("send");
        let deadline = Instant::now() + Duration::from_mins(4);
        loop {
            assert!(Instant::now() < deadline, "no response for id {id}");
            let reply = self.read_line();
            if reply.get("id").and_then(Value::as_str) == Some(id)
                && reply.get("type").and_then(Value::as_str) == Some("response")
            {
                assert_eq!(reply["success"], true, "{id} failed: {reply}");
                return reply;
            }
        }
    }

    fn read_line(&mut self) -> Value {
        let mut line = String::new();
        loop {
            match self.reader.read_line(&mut line) {
                Ok(0) => panic!("supervisor closed the connection"),
                Ok(_) if line.trim().is_empty() => line.clear(),
                Ok(_) => return serde_json::from_str(line.trim()).expect("parse line"),
                Err(_) => {}
            }
        }
    }
}

fn saved_row(client: &mut Client, id: &str, sessions_dir: &Path, session_id: &str) -> Value {
    let listed = client.request(
        id,
        &json!({ "type": "list_saved_sessions", "cwd": "/",
                 "sessionDir": sessions_dir.to_string_lossy() }),
    );
    listed["data"]["sessions"]
        .as_array()
        .expect("sessions")
        .iter()
        .find(|row| row["id"] == session_id)
        .cloned()
        .expect("the session has a catalog row")
}

#[test]
fn persistent_python_unit_regressions() {
    let Some(python) = kernel_python() else {
        assert!(
            std::env::var_os("CI").is_none(),
            "CI must bootstrap the kernel interpreter before persistent-shell tests"
        );
        return;
    };
    let runtime = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../prime-agent-runtime");
    let output = Command::new(python)
        .current_dir(&runtime)
        .env("PYTHONPATH", runtime.join("src"))
        .args(["-m", "unittest", "test.test_bash.PersistentBashTest"])
        .output()
        .expect("run persistent-shell Python regressions");
    assert!(
        output.status.success(),
        "persistent-shell Python regressions failed:\n{}\n{}",
        String::from_utf8_lossy(&output.stdout),
        String::from_utf8_lossy(&output.stderr)
    );
}

#[test]
fn saved_row_lists_its_live_shells_and_the_machine_wide_list_every_shell() {
    let dir = tempfile::TempDir::new().expect("temp dir");
    let agent_dir = dir.path().join("agent");
    let sessions_dir = agent_dir.join("sessions");
    std::fs::create_dir_all(&sessions_dir).expect("sessions dir");
    let session_id = "11111111-2222-3333-4444-555555555555";
    std::fs::write(
        sessions_dir.join(format!("{session_id}.jsonl")),
        format!(
            "{}\n",
            json!({ "type": "session", "version": 3, "id": session_id,
                    "timestamp": "2026-01-01T00:00:00.000Z", "cwd": "/tmp" })
        ),
    )
    .expect("write session");
    let mut sleeper = Command::new("sleep")
        .arg("60")
        .process_group(0)
        .spawn()
        .expect("spawn sleeper");
    let _guard = GroupGuard(sleeper.id() as i32);
    let mut gone = Command::new("true").spawn().expect("spawn true");
    gone.wait().expect("reap true");
    let sleeper_start_id = pa_daemon::lease::get_process_start_id(sleeper.id()).expect("start id");
    let home = agent_dir.join("persistent-shells");
    std::fs::create_dir_all(&home).expect("persistent-shell home");
    for (id, pid, start_id, session) in [
        ("a", sleeper.id(), sleeper_start_id.as_str(), session_id),
        ("b", gone.id(), "proc:0", session_id),
        (
            "c",
            sleeper.id(),
            sleeper_start_id.as_str(),
            "another-session",
        ),
    ] {
        let id = id.repeat(32);
        std::fs::write(
            home.join(format!("{id}.json")),
            json!({ "id": id, "pid": pid, "startId": start_id, "command": "sleep 60",
                    "startedAt": "2026-01-01T00:00:00+00:00", "sessionId": session })
            .to_string(),
        )
        .expect("write record");
    }

    let socket = dir.path().join("supervisor.sock");
    let _daemon = spawn_supervisor(&socket, &agent_dir, None);
    let mut client = Client::connect(&socket);
    let row = saved_row(&mut client, "l1", &sessions_dir, session_id);

    assert_eq!(
        row["persistentShells"],
        json!([{ "id": "a".repeat(32), "command": "sleep 60", "pid": sleeper.id(),
                 "startedAt": "2026-01-01T00:00:00+00:00", "status": "running",
                 "persistent": true }])
    );
    let listed = client.request("s1", &json!({ "type": "list_persistent_shells" }));
    let mut shells: Vec<String> = listed["data"]["activities"]
        .as_array()
        .expect("activities")
        .iter()
        .map(|row| format!("{} {} {}", row["id"], row["status"], row["sessionId"]))
        .collect();
    shells.sort();
    assert_eq!(
        shells,
        [
            format!("\"{}\" \"running\" \"{session_id}\"", "a".repeat(32)),
            format!("\"{}\" \"finished\" \"{session_id}\"", "b".repeat(32)),
            format!("\"{}\" \"running\" \"another-session\"", "c".repeat(32)),
        ],
        "the machine-wide list holds every shell with its assignment"
    );
    sleeper.kill().expect("kill sleeper");
    sleeper.wait().expect("reap sleeper");
}

#[test]
fn persistent_shell_survives_daemon_shutdown_is_reattached_and_outlives_session_delete() {
    let Some(kernel_python) = kernel_python() else {
        return;
    };
    let dir = tempfile::TempDir::new().expect("temp dir");
    let agent_dir = dir.path().join("agent");
    let sessions_dir = dir.path().join("sessions");
    std::fs::create_dir_all(&sessions_dir).expect("sessions dir");
    let receipt = dir.path().join("pid.txt");
    let script = write_faux_script(
        &dir.path().join("faux.json"),
        &format!(
            "h = bash('echo hi; sleep 600', persistent=True)\nopen({:?}, 'w').write(str(h.pid))\n",
            receipt.to_string_lossy()
        ),
    );

    let baseline = kernel_pids();
    let socket = dir.path().join("supervisor.sock");
    let mut daemon = spawn_supervisor(&socket, &agent_dir, Some(&kernel_python));
    let mut client = Client::connect(&socket);
    let created = client.request(
        "c1",
        &json!({ "type": "create", "config": {
            "cwd": dir.path().to_string_lossy(),
            "sessionDir": sessions_dir.to_string_lossy(),
            "script": script.to_string_lossy(),
        } }),
    );
    let session_id = created["data"]["sessionId"]
        .as_str()
        .expect("session id")
        .to_string();
    client.request(
        "p1",
        &json!({ "type": "prompt", "activeSessionId": session_id, "message": "start it" }),
    );
    client.request(
        "i1",
        &json!({ "type": "wait_for_idle", "activeSessionId": session_id }),
    );
    let pid: i32 = std::fs::read_to_string(&receipt)
        .expect("the cell wrote the shell pid")
        .parse()
        .expect("pid");
    let _guard = GroupGuard(pid);

    // Daemon stop: the workers dispose their kernels (exit hook + orphan reaper).
    client.request("sd", &json!({ "type": "shutdown" }));
    let deadline = Instant::now() + Duration::from_secs(90);
    while daemon.child.try_wait().expect("poll supervisor").is_none() {
        assert!(Instant::now() < deadline, "the supervisor never exited");
        std::thread::sleep(Duration::from_millis(100));
    }
    let deadline = Instant::now() + Duration::from_secs(90);
    while kernel_pids().iter().any(|pid| !baseline.contains(pid)) {
        assert!(
            Instant::now() < deadline,
            "the session kernel outlived shutdown"
        );
        std::thread::sleep(Duration::from_millis(100));
    }
    assert!(
        group_alive(pid),
        "the persistent shell died with the daemon"
    );

    // A fresh daemon lists the inactive session's shell.
    let socket = dir.path().join("supervisor-2.sock");
    let _daemon = spawn_supervisor(&socket, &agent_dir, Some(&kernel_python));
    let mut client = Client::connect(&socket);
    let row = saved_row(&mut client, "l1", &sessions_dir, &session_id);
    assert_eq!(row["persistentShells"][0]["pid"], pid);

    // Reopening the session boots a new kernel, which re-attaches the shell.
    let reattached = dir.path().join("reattached.json");
    let script = write_faux_script(
        &dir.path().join("faux-reopen.json"),
        &format!(
            "import json\nfrom rlm import persistent_shells\njson.dump([[h.pid, h.running, h.output()] for h in persistent_shells()], open({:?}, 'w'))\n",
            reattached.to_string_lossy()
        ),
    );
    let session_file = sessions_dir.join(format!("{session_id}.jsonl"));
    let reopened = client.request(
        "c2",
        &json!({ "type": "create", "sessionPath": session_file.to_string_lossy(), "config": {
            "cwd": dir.path().to_string_lossy(),
            "sessionDir": sessions_dir.to_string_lossy(),
            "script": script.to_string_lossy(),
        } }),
    );
    let active_id = reopened["data"]["activeSessionId"]
        .as_str()
        .or_else(|| reopened["data"]["id"].as_str())
        .expect("active session id")
        .to_string();
    client.request(
        "p2",
        &json!({ "type": "prompt", "activeSessionId": active_id, "message": "find it" }),
    );
    client.request(
        "i2",
        &json!({ "type": "wait_for_idle", "activeSessionId": active_id }),
    );
    let handles: Value = serde_json::from_str(
        &std::fs::read_to_string(&reattached).expect("the reopened kernel wrote its handles"),
    )
    .expect("parse handles");
    assert_eq!(handles, json!([[pid, true, "hi\n"]]));

    // Close the session, then delete it: the shell runs on as an orphan.
    client.request(
        "k1",
        &json!({ "type": "kill", "activeSessionId": active_id }),
    );
    client.request(
        "d1",
        &json!({ "type": "delete_saved_session", "sessionPath": session_file.to_string_lossy() }),
    );
    assert!(!session_file.exists());
    assert!(
        group_alive(pid),
        "the session delete killed its persistent shell"
    );
    let records: Vec<Value> = std::fs::read_dir(agent_dir.join("persistent-shells"))
        .expect("persistent-shell home")
        .flatten()
        .filter(|entry| entry.path().extension().is_some_and(|ext| ext == "json"))
        .map(|entry| {
            serde_json::from_slice(&std::fs::read(entry.path()).expect("read record"))
                .expect("parse record")
        })
        .collect();
    assert_eq!(records.len(), 1);
    assert_eq!(records[0]["pid"], pid);
    assert_eq!(records[0]["sessionId"], session_id.as_str());

    // The machine-wide list still names the deleted session; its kill ends the orphan.
    let listed = client.request("s1", &json!({ "type": "list_persistent_shells" }));
    let shell = &listed["data"]["activities"][0];
    assert_eq!(
        (&shell["pid"], &shell["sessionId"], &shell["status"]),
        (&json!(pid), &json!(session_id), &json!("running"))
    );
    client.request(
        "x1",
        &json!({ "type": "kill_persistent_shell", "shellId": shell["id"] }),
    );
    let deadline = Instant::now() + Duration::from_secs(10);
    while group_alive(pid) {
        assert!(
            Instant::now() < deadline,
            "the kill left the orphan's group alive"
        );
        std::thread::sleep(Duration::from_millis(50));
    }
    let listed = client.request("s2", &json!({ "type": "list_persistent_shells" }));
    assert_eq!(
        listed["data"]["activities"][0]["status"], "finished",
        "the killed orphan stays listed as finished"
    );
}

fn write_faux_script(path: &Path, cell: &str) -> PathBuf {
    std::fs::write(
        path,
        json!({ "engine": "faux", "responses": [
            { "content": [{ "type": "toolCall", "name": "python_repl",
                            "arguments": { "code": cell } }] },
            { "text": "done" },
        ] })
        .to_string(),
    )
    .expect("write faux script");
    path.to_path_buf()
}

/// Every live `python -m rlm.repl` kernel pid on the box.
fn kernel_pids() -> Vec<u32> {
    std::fs::read_dir("/proc")
        .expect("read /proc")
        .flatten()
        .filter_map(|entry| {
            let pid = entry.file_name().to_string_lossy().parse::<u32>().ok()?;
            let cmdline = std::fs::read_to_string(entry.path().join("cmdline")).ok()?;
            let args: Vec<&str> = cmdline.split('\0').collect();
            args.windows(2)
                .any(|window| window == ["-m", "rlm.repl"])
                .then_some(pid)
        })
        .collect()
}