Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

Scope startup RLM ledger reseeding to the opened parent (#3414)

Sebastian Müller73acc534673c
diff --git a/.changes/rlm-reseed-scoped-to-opened-parent.md b/.changes/rlm-reseed-scoped-to-opened-parent.md
new file mode 100644
index 0000000000..cc1958df03
--- /dev/null
+++ b/.changes/rlm-reseed-scoped-to-opened-parent.md
@@ -0,0 +1 @@
+- Opening a session no longer scans the whole RLM spawn ledger and the whole session-artifacts tree: only the opened parent's own subagents are re-listed, and a create that makes a brand-new session skips the scan, so session opens stay fast as the ledger grows.
diff --git a/crates/pa-daemon/src/registry.rs b/crates/pa-daemon/src/registry.rs
index c67c15dbad..9bee2c82d0 100644
--- a/crates/pa-daemon/src/registry.rs
+++ b/crates/pa-daemon/src/registry.rs
@@ -354,7 +354,8 @@ pub(crate) struct WorkerRegistration {
}
/// Accepted registration state per worker: the identity plus the
-/// registration count (epoch 1 = boot, epoch > 1 = re-registration after a supervisor restart).
+/// registration count (epoch 1 = boot; every later epoch is an in-place
+/// re-registration, e.g. `notify_session_created` after a create).
#[derive(Debug, Clone, PartialEq, Eq)]
pub(crate) struct RegistrationRecord {
pub(crate) registration: WorkerRegistration,
diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rs
index 466b318ba7..dfd2e74f16 100644
--- a/crates/pa-daemon/src/rlm_children/registry.rs
+++ b/crates/pa-daemon/src/rlm_children/registry.rs
@@ -8,7 +8,6 @@ use super::{
bail, json, rlm_child_label, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand,
DeletedChild, Map, Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS,
};
-use crate::lease::canonical_session_path;
/// The already-gone marker inside a close failure: a close walking a
/// child that died earlier must not fail (a missing child session is a completed no-op).
@@ -311,16 +310,14 @@ fn ledger_child_records(
};
let ledger =
crate::rlm_ledger::RlmSpawnLedger::new(agent_dir, Path::new(&sessions_dir), |_| {});
- let edges = ledger.live_edges().unwrap_or_else(|error| {
- eprintln!("pa-daemon: RLM ledger reseed skipped: {error:#}");
- Vec::new()
- });
- let parent_file = canonical_session_path(parent_file);
+ let edges = ledger
+ .live_edges_of_parent(parent_file)
+ .unwrap_or_else(|error| {
+ eprintln!("pa-daemon: RLM ledger reseed skipped: {error:#}");
+ Vec::new()
+ });
let mut records = Vec::new();
for edge in edges {
- if canonical_session_path(Path::new(&edge.parent)) != parent_file {
- continue;
- }
let child = PathBuf::from(&edge.child);
let session_id = child
.file_stem()
diff --git a/crates/pa-daemon/src/rlm_ledger.rs b/crates/pa-daemon/src/rlm_ledger.rs
index 3bf550d7b2..a58bd778e0 100644
--- a/crates/pa-daemon/src/rlm_ledger.rs
+++ b/crates/pa-daemon/src/rlm_ledger.rs
@@ -129,6 +129,106 @@ pub fn rlm_ledger_path(agent_dir: &Path, sessions_dir: &Path) -> PathBuf {
agent_dir.join(RLM_LEDGER_DIR).join(format!("{hash}.jsonl"))
}
+/// The liveness pass shared by the live-edge reads: deleted edges never
+/// resolve; the survivors' recorded endpoints do - a dead endpoint drops
+/// the edge, a moved path resolves through its durable session id - and
+/// the resolved paths replace the recorded ones.
+fn resolve_live_edges(state: &ReplayState, resolver: &mut LivePathResolver) -> Vec<RlmLedgerEdge> {
+ let mut edges = Vec::with_capacity(state.edges.len());
+ for edge in &state.edges {
+ if edge.deleted.is_some() {
+ continue;
+ }
+ let (Some(child), Some(parent)) = (
+ resolver.resolve(&edge.child),
+ resolver.resolve(&edge.parent),
+ ) else {
+ continue;
+ };
+ edges.push(RlmLedgerEdge {
+ parent: parent.to_string_lossy().to_string(),
+ child: child.to_string_lossy().to_string(),
+ ..edge.clone()
+ });
+ }
+ edges
+}
+
+/// Fold parsed records into the replayed edge set, keeping only the
+/// records whose child id `keep` selects.
+fn fold_records(records: Vec<LedgerRecord>, keep: impl Fn(&str) -> bool) -> ReplayState {
+ let mut state = ReplayState::default();
+ for record in records {
+ match record {
+ LedgerRecord::Spawn {
+ child_id,
+ parent,
+ child,
+ depth,
+ name,
+ } if keep(&child_id) => {
+ let key = edge_key(&child_id, &child);
+ if let Some(at) = state.index.get(&key).copied() {
+ state.edges[at] = RlmLedgerEdge {
+ child_id,
+ parent,
+ child,
+ depth,
+ name,
+ deleted: None,
+ deleted_usage: None,
+ };
+ } else {
+ state.index.insert(key.clone(), state.edges.len());
+ state.edges.push(RlmLedgerEdge {
+ child_id,
+ parent,
+ child,
+ depth,
+ name,
+ deleted: None,
+ deleted_usage: None,
+ });
+ }
+ }
+ LedgerRecord::Rename {
+ child_id,
+ child,
+ name,
+ } if keep(&child_id) => {
+ let key = edge_key(&child_id, &child);
+ if let Some(&at) = state.index.get(&key) {
+ state.edges[at].name = name;
+ } else if let Some(at) = sole_edge_by_child_id(&state, &child_id) {
+ state.edges[at].name = name;
+ }
+ }
+ LedgerRecord::Delete {
+ child_id,
+ child,
+ reason,
+ usage,
+ } if keep(&child_id) => {
+ let key = edge_key(&child_id, &child);
+ let at = match state.index.get(&key).copied() {
+ Some(at) => Some(at),
+ None => sole_edge_by_child_id(&state, &child_id),
+ };
+ if let Some(at) = at {
+ state.edges[at].deleted = Some(reason);
+ // The snapshot is sticky: a re-tombstone without a usage
+ // block never clears it; a fresh capture replaces it.
+ if let Some(usage) = usage {
+ state.edges[at].deleted_usage = Some(usage);
+ }
+ }
+ }
+ _ => {}
+ }
+ }
+ state
+}
+
impl RlmSpawnLedger {
/// Ledger over one sessions dir, with a caller-supplied log sink for
/// degraded reads and seed skips.
@@ -546,23 +646,28 @@ impl RlmSpawnLedger {
let state = self.replay_cached()?;
let mut resolver =
LivePathResolver::new(self.agent_dir.clone(), self.canonical_sessions_dir.clone());
- let mut edges = Vec::with_capacity(state.edges.len());
- for edge in &state.edges {
- if edge.deleted.is_some() {
- continue;
- }
- let (Some(child), Some(parent)) = (
- resolver.resolve(&edge.child),
- resolver.resolve(&edge.parent),
- ) else {
- continue;
- };
- edges.push(RlmLedgerEdge {
- parent: parent.to_string_lossy().to_string(),
- child: child.to_string_lossy().to_string(),
- ..edge.clone()
- });
- }
+ Ok(resolve_live_edges(&state, &mut resolver))
+ }
+
+ /// The opened parent file's live edges: only the records of children
+ /// spawned by that parent file (matched by its durable session id,
+ /// the file-name stem) fold and resolve, and only those whose
+ /// resolved parent IS the parent file remain. A parent's read never
+ /// stats, or walks the artifacts tree for, another parent's edges.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error when the ledger replay fails (an oversized
+ /// ledger); a missing ledger replays empty, and only the liveness
+ /// resolution drops edges.
+ pub fn live_edges_of_parent(&self, parent_file: &Path) -> Result<Vec<RlmLedgerEdge>> {
+ self.seed_once()?;
+ let parent_file = canonical_session_path(parent_file);
+ let state = self.replay_of_parent(&parent_file)?;
+ let mut resolver =
+ LivePathResolver::new(self.agent_dir.clone(), self.canonical_sessions_dir.clone());
+ let mut edges = resolve_live_edges(&state, &mut resolver);
+ edges.retain(|edge| canonical_session_path(Path::new(&edge.parent)) == parent_file);
Ok(edges)
}
@@ -625,8 +730,38 @@ impl RlmSpawnLedger {
}
fn replay(&self) -> Result<ReplayState> {
+ Ok(fold_records(self.read_records()?, |_| true))
+ }
+
+ /// The opened parent file's subset of the replayed ledger: fold only
+ /// the records of children spawned by `parent_file` (matched by its
+ /// durable session id, the file-name stem). Every join, including
+ /// the sole-child-id fallback, stays within one child id, so the
+ /// subset fold equals the full fold restricted to that parent's
+ /// children.
+ fn replay_of_parent(&self, parent_file: &Path) -> Result<ReplayState> {
+ let records = self.read_records()?;
+ let own: HashSet<String> = records
+ .iter()
+ .filter_map(|record| match record {
+ LedgerRecord::Spawn {
+ child_id, parent, ..
+ } if Path::new(parent).file_stem() == parent_file.file_stem() => {
+ Some(child_id.clone())
+ }
+ _ => None,
+ })
+ .collect();
+ Ok(fold_records(records, |child_id| own.contains(child_id)))
+ }
+
+ /// The whole ledger's parsed records within its read bounds: the
+ /// byte and record-count caps apply to the whole file; a missing
+ /// ledger reads empty, and one torn or unknown line skips that
+ /// record only.
+ fn read_records(&self) -> Result<Vec<LedgerRecord>> {
let Ok(bytes) = fs::read(&self.path) else {
- return Ok(ReplayState::default());
+ return Ok(Vec::new());
};
if bytes.len() as u64 > RLM_LEDGER_MAX_BYTES {
bail!(
@@ -637,103 +772,36 @@ impl RlmSpawnLedger {
// A torn write inside a multibyte name must spoil one line, not
// the whole file.
let content = String::from_utf8_lossy(&bytes);
- let mut state = ReplayState::default();
- let mut records = 0usize;
+ let mut records = Vec::new();
+ let mut lines = 0usize;
for (index, line) in content.lines().enumerate() {
if line.trim().is_empty() {
continue;
}
- records += 1;
- if records > RLM_LEDGER_MAX_RECORDS {
+ lines += 1;
+ if lines > RLM_LEDGER_MAX_RECORDS {
bail!(
"RLM ledger {} exceeds {RLM_LEDGER_MAX_RECORDS} records; refusing to read",
self.path.display()
);
}
- let record = match parse_ledger_line(line, index) {
- Ok(Some(record)) => record,
+ match parse_ledger_line(line, index) {
+ Ok(Some(record)) => records.push(record),
Ok(None) => {
self.log(&format!(
"RLM ledger: skipped record with unknown op on line {}",
index + 1
));
- continue;
}
Err(error) => {
self.log(&format!(
"RLM ledger {}: skipped {error:#}",
self.path.display()
));
- continue;
- }
- };
- match record {
- LedgerRecord::Spawn {
- child_id,
- parent,
- child,
- depth,
- name,
- } => {
- let key = edge_key(&child_id, &child);
- if let Some(at) = state.index.get(&key).copied() {
- state.edges[at] = RlmLedgerEdge {
- child_id,
- parent,
- child,
- depth,
- name,
- deleted: None,
- deleted_usage: None,
- };
- } else {
- state.index.insert(key.clone(), state.edges.len());
- state.edges.push(RlmLedgerEdge {
- child_id,
- parent,
- child,
- depth,
- name,
- deleted: None,
- deleted_usage: None,
- });
- }
- }
- LedgerRecord::Rename {
- child_id,
- child,
- name,
- } => {
- let key = edge_key(&child_id, &child);
- if let Some(&at) = state.index.get(&key) {
- state.edges[at].name = name;
- } else if let Some(at) = sole_edge_by_child_id(&state, &child_id) {
- state.edges[at].name = name;
- }
- }
- LedgerRecord::Delete {
- child_id,
- child,
- reason,
- usage,
- } => {
- let key = edge_key(&child_id, &child);
- let at = match state.index.get(&key).copied() {
- Some(at) => Some(at),
- None => sole_edge_by_child_id(&state, &child_id),
- };
- if let Some(at) = at {
- state.edges[at].deleted = Some(reason);
- // The snapshot is sticky: a re-tombstone without a usage
- // block never clears it; a fresh capture replaces it.
- if let Some(usage) = usage {
- state.edges[at].deleted_usage = Some(usage);
- }
- }
}
}
}
- Ok(state)
+ Ok(records)
}
/// One durable append; the first record in a fresh file is the meta
diff --git a/crates/pa-daemon/src/rlm_ledger/replay.rs b/crates/pa-daemon/src/rlm_ledger/replay.rs
index 7d500b41c4..ecd6f97e08 100644
--- a/crates/pa-daemon/src/rlm_ledger/replay.rs
+++ b/crates/pa-daemon/src/rlm_ledger/replay.rs
@@ -172,8 +172,8 @@ pub(super) fn edge_key(child_id: &str, child: &str) -> String {
pub(super) struct LivePathResolver {
agent_dir: PathBuf,
sessions_dir: PathBuf,
- resolved: HashMap<String, Option<PathBuf>>,
- artifact_index: Option<HashMap<String, PathBuf>>,
+ pub(super) resolved: HashMap<String, Option<PathBuf>>,
+ pub(super) artifact_index: Option<HashMap<String, PathBuf>>,
}
impl LivePathResolver {
diff --git a/crates/pa-daemon/src/rlm_ledger/tests.rs b/crates/pa-daemon/src/rlm_ledger/tests.rs
index 5738d8e8a6..e889e084fb 100644
--- a/crates/pa-daemon/src/rlm_ledger/tests.rs
+++ b/crates/pa-daemon/src/rlm_ledger/tests.rs
@@ -254,6 +254,163 @@ fn moved_edge_paths_resolve_through_the_session_id() {
assert_eq!(raw[0].child, recorded_child);
}
+/// The parent-scoped live read bounds the liveness work to the opened
+/// parent's own edges: foreign edges with missing endpoints never
+/// resolve (no per-edge stats, no artifacts-tree walk), while the own
+/// edges resolve - a moved child through its session id - and a
+/// tombstoned own child is skipped before any resolution.
+#[test]
+fn live_edges_of_parent_bounds_the_liveness_work() {
+ let agent_dir = temp_dir("scoped");
+ let sessions_dir = agent_dir.join("sessions");
+ fs::create_dir_all(&sessions_dir).unwrap();
+ let ledger = ledger_over(&agent_dir, &sessions_dir);
+
+ let parent = sessions_dir.join("sess-p.jsonl");
+ let moved_child = sessions_dir.join("sess-a.jsonl");
+ let resident_child = sessions_dir.join("sess-b.jsonl");
+ fs::write(&parent, "{}").unwrap();
+ fs::write(&moved_child, "{}").unwrap();
+ fs::write(&resident_child, "{}").unwrap();
+ ledger
+ .append_spawn(&RlmSpawnInput {
+ child_id: "sub-a".into(),
+ parent: parent.to_string_lossy().into(),
+ child: "/gone-root/sess-a.jsonl".into(),
+ depth: 1,
+ name: "a".into(),
+ })
+ .unwrap();
+ ledger
+ .append_spawn(&RlmSpawnInput {
+ child_id: "sub-b".into(),
+ parent: parent.to_string_lossy().into(),
+ child: resident_child.to_string_lossy().into(),
+ depth: 1,
+ name: "b".into(),
+ })
+ .unwrap();
+ let deleted_child = sessions_dir.join("sess-d.jsonl");
+ fs::write(&deleted_child, "{}").unwrap();
+ ledger
+ .append_spawn(&RlmSpawnInput {
+ child_id: "sub-d".into(),
+ parent: parent.to_string_lossy().into(),
+ child: deleted_child.to_string_lossy().into(),
+ depth: 1,
+ name: "d".into(),
+ })
+ .unwrap();
+ ledger
+ .append_delete(
+ "sub-d",
+ &deleted_child.to_string_lossy(),
+ RlmLedgerDeleteReason::User,
+ )
+ .unwrap();
+ for n in 0..50 {
+ ledger
+ .append_spawn(&RlmSpawnInput {
+ child_id: format!("sub-f{n}"),
+ parent: format!("/gone-root/sess-f{n}.jsonl"),
+ child: format!("/gone-root/sess-f{n}-child.jsonl"),
+ depth: 1,
+ name: format!("f{n}"),
+ })
+ .unwrap();
+ }
+
+ let edges = ledger.live_edges_of_parent(&parent).unwrap();
+ let ids: Vec<&str> = edges.iter().map(|edge| edge.child_id.as_str()).collect();
+ assert_eq!(ids, ["sub-a", "sub-b"], "{edges:?}");
+ assert_eq!(
+ edges[0].child,
+ canonical_session_path(&moved_child)
+ .to_string_lossy()
+ .to_string(),
+ "the moved child resolves through its session id"
+ );
+
+ let state = ledger.replay_of_parent(&parent).unwrap();
+ let mut resolver = LivePathResolver::new(agent_dir.clone(), sessions_dir.clone());
+ let scoped = resolve_live_edges(&state, &mut resolver);
+ assert_eq!(scoped.len(), 2, "{scoped:?}");
+ assert_eq!(
+ resolver.resolved.len(),
+ 3,
+ "{:?}",
+ resolver.resolved.keys().collect::<Vec<_>>()
+ );
+ assert!(
+ resolver.artifact_index.is_none(),
+ "no foreign miss walks the artifacts tree"
+ );
+ let full = ledger.replay_cached().unwrap();
+ let mut unscoped = LivePathResolver::new(agent_dir, sessions_dir);
+ assert_eq!(resolve_live_edges(&full, &mut unscoped).len(), 2);
+ assert!(unscoped.artifact_index.is_some());
+ assert_eq!(unscoped.resolved.len(), 103);
+}
+
+/// A parent that moved (recorded path gone, same stem under the sessions
+/// dir) still lists its children through the scoped read; an edge whose
+/// recorded parent merely shares the opened file's stem but resolves to
+/// another live file is not the opened parent's.
+#[test]
+fn live_edges_of_parent_follows_a_moved_parent() {
+ let agent_dir = temp_dir("scoped-moved");
+ let sessions_dir = agent_dir.join("sessions");
+ fs::create_dir_all(&sessions_dir).unwrap();
+ let ledger = ledger_over(&agent_dir, &sessions_dir);
+
+ let recorded_parent = "/old-root/sessions/sess-p.jsonl";
+ let child = sessions_dir.join("sess-c.jsonl");
+ let live_parent = sessions_dir.join("sess-p.jsonl");
+ fs::write(&child, "{}").unwrap();
+ fs::write(&live_parent, "{}").unwrap();
+ ledger
+ .append_spawn(&RlmSpawnInput {
+ child_id: "sub-1".into(),
+ parent: recorded_parent.into(),
+ child: child.to_string_lossy().into(),
+ depth: 1,
+ name: "w".into(),
+ })
+ .unwrap();
+ let edges = ledger.live_edges_of_parent(&live_parent).unwrap();
+ assert_eq!(edges.len(), 1, "{edges:?}");
+ assert_eq!(
+ edges[0].parent,
+ canonical_session_path(&live_parent)
+ .to_string_lossy()
+ .to_string(),
+ "the moved parent still lists its children"
+ );
+
+ let elsewhere = agent_dir
+ .join("session-artifacts")
+ .join("host")
+ .join("sess-p2.jsonl");
+ fs::create_dir_all(elsewhere.parent().unwrap()).unwrap();
+ fs::write(&elsewhere, "{}").unwrap();
+ fs::write(sessions_dir.join("sess-p2.jsonl"), "{}").unwrap();
+ fs::write(sessions_dir.join("sess-c2.jsonl"), "{}").unwrap();
+ ledger
+ .append_spawn(&RlmSpawnInput {
+ child_id: "sub-2".into(),
+ parent: "/old-root/sessions/sess-p2.jsonl".into(),
+ child: sessions_dir.join("sess-c2.jsonl").to_string_lossy().into(),
+ depth: 1,
+ name: "w2".into(),
+ })
+ .unwrap();
+ let edges = ledger.live_edges_of_parent(&elsewhere).unwrap();
+ assert!(
+ edges.is_empty(),
+ "a same-stem parent resolved to another live file is not the opened parent: {edges:?}"
+ );
+}
+
#[test]
fn duplicate_child_path_and_bad_spawn_inputs_fail_but_bad_lines_skip() {
let dir = temp_dir("dup");
diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rs
index bea876a09d..afcaa92eaa 100644
--- a/crates/pa-daemon/src/worker/create.rs
+++ b/crates/pa-daemon/src/worker/create.rs
@@ -596,7 +596,9 @@ impl Worker {
}) {
return response_failure(None, "create", &error.to_string(), None);
}
- self.reseed_rlm_children().await;
+ if opened_existing_session {
+ self.reseed_rlm_children().await;
+ }
// The built-in Herdr connector binds here, per session: the pane
// identity comes from the create payload's client env (the client
// that owns the pane sent it), never from this process's ambient