A self-improving RLM agent for coding workflows and long-running autonomous tasks.
Rust
Scope startup RLM ledger reseeding to the opened parent (#3414)
Sebastian Müller73acc534673c
diff --git a/.changes/rlm-reseed-scoped-to-opened-parent.md b/.changes/rlm-reseed-scoped-to-opened-parent.mdnew file mode 100644index 0000000000..cc1958df03--- /dev/null+++ b/.changes/rlm-reseed-scoped-to-opened-parent.md@@ -0,0 +1 @@+- Opening a session no longer scans the whole RLM spawn ledger and the whole session-artifacts tree: only the opened parent's own subagents are re-listed, and a create that makes a brand-new session skips the scan, so session opens stay fast as the ledger grows.diff --git a/crates/pa-daemon/src/registry.rs b/crates/pa-daemon/src/registry.rsindex c67c15dbad..9bee2c82d0 100644--- a/crates/pa-daemon/src/registry.rs+++ b/crates/pa-daemon/src/registry.rs@@ -354,7 +354,8 @@ pub(crate) struct WorkerRegistration {}/// Accepted registration state per worker: the identity plus the-/// registration count (epoch 1 = boot, epoch > 1 = re-registration after a supervisor restart).+/// registration count (epoch 1 = boot; every later epoch is an in-place+/// re-registration, e.g. `notify_session_created` after a create).#[derive(Debug, Clone, PartialEq, Eq)]pub(crate) struct RegistrationRecord {pub(crate) registration: WorkerRegistration,diff --git a/crates/pa-daemon/src/rlm_children/registry.rs b/crates/pa-daemon/src/rlm_children/registry.rsindex 466b318ba7..dfd2e74f16 100644--- a/crates/pa-daemon/src/rlm_children/registry.rs+++ b/crates/pa-daemon/src/rlm_children/registry.rs@@ -8,7 +8,6 @@ use super::{bail, json, rlm_child_label, Arc, ChildCloseReason, ChildRecord, Context, DaemonCommand,DeletedChild, Map, Mutex, Result, SupervisorChildSessionsInner, KILL_TIMEOUT_MS,};-use crate::lease::canonical_session_path;/// The already-gone marker inside a close failure: a close walking a/// child that died earlier must not fail (a missing child session is a completed no-op).@@ -311,16 +310,14 @@ fn ledger_child_records(};let ledger =crate::rlm_ledger::RlmSpawnLedger::new(agent_dir, Path::new(&sessions_dir), |_| {});- let edges = ledger.live_edges().unwrap_or_else(|error| {- eprintln!("pa-daemon: RLM ledger reseed skipped: {error:#}");- Vec::new()- });- let parent_file = canonical_session_path(parent_file);+ let edges = ledger+ .live_edges_of_parent(parent_file)+ .unwrap_or_else(|error| {+ eprintln!("pa-daemon: RLM ledger reseed skipped: {error:#}");+ Vec::new()+ });let mut records = Vec::new();for edge in edges {- if canonical_session_path(Path::new(&edge.parent)) != parent_file {- continue;- }let child = PathBuf::from(&edge.child);let session_id = child.file_stem()diff --git a/crates/pa-daemon/src/rlm_ledger.rs b/crates/pa-daemon/src/rlm_ledger.rsindex 3bf550d7b2..a58bd778e0 100644--- a/crates/pa-daemon/src/rlm_ledger.rs+++ b/crates/pa-daemon/src/rlm_ledger.rs@@ -129,6 +129,106 @@ pub fn rlm_ledger_path(agent_dir: &Path, sessions_dir: &Path) -> PathBuf {agent_dir.join(RLM_LEDGER_DIR).join(format!("{hash}.jsonl"))}+/// The liveness pass shared by the live-edge reads: deleted edges never+/// resolve; the survivors' recorded endpoints do - a dead endpoint drops+/// the edge, a moved path resolves through its durable session id - and+/// the resolved paths replace the recorded ones.+fn resolve_live_edges(state: &ReplayState, resolver: &mut LivePathResolver) -> Vec<RlmLedgerEdge> {+ let mut edges = Vec::with_capacity(state.edges.len());+ for edge in &state.edges {+ if edge.deleted.is_some() {+ continue;+ }+ let (Some(child), Some(parent)) = (+ resolver.resolve(&edge.child),+ resolver.resolve(&edge.parent),+ ) else {+ continue;+ };+ edges.push(RlmLedgerEdge {+ parent: parent.to_string_lossy().to_string(),+ child: child.to_string_lossy().to_string(),+ ..edge.clone()+ });+ }+ edges+}++/// Fold parsed records into the replayed edge set, keeping only the+/// records whose child id `keep` selects.+fn fold_records(records: Vec<LedgerRecord>, keep: impl Fn(&str) -> bool) -> ReplayState {+ let mut state = ReplayState::default();+ for record in records {+ match record {+ LedgerRecord::Spawn {+ child_id,+ parent,+ child,+ depth,+ name,+ } if keep(&child_id) => {+ let key = edge_key(&child_id, &child);+ if let Some(at) = state.index.get(&key).copied() {+ state.edges[at] = RlmLedgerEdge {+ child_id,+ parent,+ child,+ depth,+ name,+ deleted: None,+ deleted_usage: None,+ };+ } else {+ state.index.insert(key.clone(), state.edges.len());+ state.edges.push(RlmLedgerEdge {+ child_id,+ parent,+ child,+ depth,+ name,+ deleted: None,+ deleted_usage: None,+ });+ }+ }+ LedgerRecord::Rename {+ child_id,+ child,+ name,+ } if keep(&child_id) => {+ let key = edge_key(&child_id, &child);+ if let Some(&at) = state.index.get(&key) {+ state.edges[at].name = name;+ } else if let Some(at) = sole_edge_by_child_id(&state, &child_id) {+ state.edges[at].name = name;+ }+ }+ LedgerRecord::Delete {+ child_id,+ child,+ reason,+ usage,+ } if keep(&child_id) => {+ let key = edge_key(&child_id, &child);+ let at = match state.index.get(&key).copied() {+ Some(at) => Some(at),+ None => sole_edge_by_child_id(&state, &child_id),+ };+ if let Some(at) = at {+ state.edges[at].deleted = Some(reason);+ // The snapshot is sticky: a re-tombstone without a usage+ // block never clears it; a fresh capture replaces it.+ if let Some(usage) = usage {+ state.edges[at].deleted_usage = Some(usage);+ }+ }+ }+ _ => {}+ }+ }+ state+}+impl RlmSpawnLedger {/// Ledger over one sessions dir, with a caller-supplied log sink for/// degraded reads and seed skips.@@ -546,23 +646,28 @@ impl RlmSpawnLedger {let state = self.replay_cached()?;let mut resolver =LivePathResolver::new(self.agent_dir.clone(), self.canonical_sessions_dir.clone());- let mut edges = Vec::with_capacity(state.edges.len());- for edge in &state.edges {- if edge.deleted.is_some() {- continue;- }- let (Some(child), Some(parent)) = (- resolver.resolve(&edge.child),- resolver.resolve(&edge.parent),- ) else {- continue;- };- edges.push(RlmLedgerEdge {- parent: parent.to_string_lossy().to_string(),- child: child.to_string_lossy().to_string(),- ..edge.clone()- });- }+ Ok(resolve_live_edges(&state, &mut resolver))+ }++ /// The opened parent file's live edges: only the records of children+ /// spawned by that parent file (matched by its durable session id,+ /// the file-name stem) fold and resolve, and only those whose+ /// resolved parent IS the parent file remain. A parent's read never+ /// stats, or walks the artifacts tree for, another parent's edges.+ ///+ /// # Errors+ ///+ /// Returns an error when the ledger replay fails (an oversized+ /// ledger); a missing ledger replays empty, and only the liveness+ /// resolution drops edges.+ pub fn live_edges_of_parent(&self, parent_file: &Path) -> Result<Vec<RlmLedgerEdge>> {+ self.seed_once()?;+ let parent_file = canonical_session_path(parent_file);+ let state = self.replay_of_parent(&parent_file)?;+ let mut resolver =+ LivePathResolver::new(self.agent_dir.clone(), self.canonical_sessions_dir.clone());+ let mut edges = resolve_live_edges(&state, &mut resolver);+ edges.retain(|edge| canonical_session_path(Path::new(&edge.parent)) == parent_file);Ok(edges)}@@ -625,8 +730,38 @@ impl RlmSpawnLedger {}fn replay(&self) -> Result<ReplayState> {+ Ok(fold_records(self.read_records()?, |_| true))+ }++ /// The opened parent file's subset of the replayed ledger: fold only+ /// the records of children spawned by `parent_file` (matched by its+ /// durable session id, the file-name stem). Every join, including+ /// the sole-child-id fallback, stays within one child id, so the+ /// subset fold equals the full fold restricted to that parent's+ /// children.+ fn replay_of_parent(&self, parent_file: &Path) -> Result<ReplayState> {+ let records = self.read_records()?;+ let own: HashSet<String> = records+ .iter()+ .filter_map(|record| match record {+ LedgerRecord::Spawn {+ child_id, parent, ..+ } if Path::new(parent).file_stem() == parent_file.file_stem() => {+ Some(child_id.clone())+ }+ _ => None,+ })+ .collect();+ Ok(fold_records(records, |child_id| own.contains(child_id)))+ }++ /// The whole ledger's parsed records within its read bounds: the+ /// byte and record-count caps apply to the whole file; a missing+ /// ledger reads empty, and one torn or unknown line skips that+ /// record only.+ fn read_records(&self) -> Result<Vec<LedgerRecord>> {let Ok(bytes) = fs::read(&self.path) else {- return Ok(ReplayState::default());+ return Ok(Vec::new());};if bytes.len() as u64 > RLM_LEDGER_MAX_BYTES {bail!(@@ -637,103 +772,36 @@ impl RlmSpawnLedger {// A torn write inside a multibyte name must spoil one line, not// the whole file.let content = String::from_utf8_lossy(&bytes);- let mut state = ReplayState::default();- let mut records = 0usize;+ let mut records = Vec::new();+ let mut lines = 0usize;for (index, line) in content.lines().enumerate() {if line.trim().is_empty() {continue;}- records += 1;- if records > RLM_LEDGER_MAX_RECORDS {+ lines += 1;+ if lines > RLM_LEDGER_MAX_RECORDS {bail!("RLM ledger {} exceeds {RLM_LEDGER_MAX_RECORDS} records; refusing to read",self.path.display());}- let record = match parse_ledger_line(line, index) {- Ok(Some(record)) => record,+ match parse_ledger_line(line, index) {+ Ok(Some(record)) => records.push(record),Ok(None) => {self.log(&format!("RLM ledger: skipped record with unknown op on line {}",index + 1));- continue;}Err(error) => {self.log(&format!("RLM ledger {}: skipped {error:#}",self.path.display()));- continue;- }- };- match record {- LedgerRecord::Spawn {- child_id,- parent,- child,- depth,- name,- } => {- let key = edge_key(&child_id, &child);- if let Some(at) = state.index.get(&key).copied() {- state.edges[at] = RlmLedgerEdge {- child_id,- parent,- child,- depth,- name,- deleted: None,- deleted_usage: None,- };- } else {- state.index.insert(key.clone(), state.edges.len());- state.edges.push(RlmLedgerEdge {- child_id,- parent,- child,- depth,- name,- deleted: None,- deleted_usage: None,- });- }- }- LedgerRecord::Rename {- child_id,- child,- name,- } => {- let key = edge_key(&child_id, &child);- if let Some(&at) = state.index.get(&key) {- state.edges[at].name = name;- } else if let Some(at) = sole_edge_by_child_id(&state, &child_id) {- state.edges[at].name = name;- }- }- LedgerRecord::Delete {- child_id,- child,- reason,- usage,- } => {- let key = edge_key(&child_id, &child);- let at = match state.index.get(&key).copied() {- Some(at) => Some(at),- None => sole_edge_by_child_id(&state, &child_id),- };- if let Some(at) = at {- state.edges[at].deleted = Some(reason);- // The snapshot is sticky: a re-tombstone without a usage- // block never clears it; a fresh capture replaces it.- if let Some(usage) = usage {- state.edges[at].deleted_usage = Some(usage);- }- }}}}- Ok(state)+ Ok(records)}/// One durable append; the first record in a fresh file is the metadiff --git a/crates/pa-daemon/src/rlm_ledger/replay.rs b/crates/pa-daemon/src/rlm_ledger/replay.rsindex 7d500b41c4..ecd6f97e08 100644--- a/crates/pa-daemon/src/rlm_ledger/replay.rs+++ b/crates/pa-daemon/src/rlm_ledger/replay.rs@@ -172,8 +172,8 @@ pub(super) fn edge_key(child_id: &str, child: &str) -> String {pub(super) struct LivePathResolver {agent_dir: PathBuf,sessions_dir: PathBuf,- resolved: HashMap<String, Option<PathBuf>>,- artifact_index: Option<HashMap<String, PathBuf>>,+ pub(super) resolved: HashMap<String, Option<PathBuf>>,+ pub(super) artifact_index: Option<HashMap<String, PathBuf>>,}impl LivePathResolver {diff --git a/crates/pa-daemon/src/rlm_ledger/tests.rs b/crates/pa-daemon/src/rlm_ledger/tests.rsindex 5738d8e8a6..e889e084fb 100644--- a/crates/pa-daemon/src/rlm_ledger/tests.rs+++ b/crates/pa-daemon/src/rlm_ledger/tests.rs@@ -254,6 +254,163 @@ fn moved_edge_paths_resolve_through_the_session_id() {assert_eq!(raw[0].child, recorded_child);}+/// The parent-scoped live read bounds the liveness work to the opened+/// parent's own edges: foreign edges with missing endpoints never+/// resolve (no per-edge stats, no artifacts-tree walk), while the own+/// edges resolve - a moved child through its session id - and a+/// tombstoned own child is skipped before any resolution.+#[test]+fn live_edges_of_parent_bounds_the_liveness_work() {+ let agent_dir = temp_dir("scoped");+ let sessions_dir = agent_dir.join("sessions");+ fs::create_dir_all(&sessions_dir).unwrap();+ let ledger = ledger_over(&agent_dir, &sessions_dir);++ let parent = sessions_dir.join("sess-p.jsonl");+ let moved_child = sessions_dir.join("sess-a.jsonl");+ let resident_child = sessions_dir.join("sess-b.jsonl");+ fs::write(&parent, "{}").unwrap();+ fs::write(&moved_child, "{}").unwrap();+ fs::write(&resident_child, "{}").unwrap();+ ledger+ .append_spawn(&RlmSpawnInput {+ child_id: "sub-a".into(),+ parent: parent.to_string_lossy().into(),+ child: "/gone-root/sess-a.jsonl".into(),+ depth: 1,+ name: "a".into(),+ })+ .unwrap();+ ledger+ .append_spawn(&RlmSpawnInput {+ child_id: "sub-b".into(),+ parent: parent.to_string_lossy().into(),+ child: resident_child.to_string_lossy().into(),+ depth: 1,+ name: "b".into(),+ })+ .unwrap();+ let deleted_child = sessions_dir.join("sess-d.jsonl");+ fs::write(&deleted_child, "{}").unwrap();+ ledger+ .append_spawn(&RlmSpawnInput {+ child_id: "sub-d".into(),+ parent: parent.to_string_lossy().into(),+ child: deleted_child.to_string_lossy().into(),+ depth: 1,+ name: "d".into(),+ })+ .unwrap();+ ledger+ .append_delete(+ "sub-d",+ &deleted_child.to_string_lossy(),+ RlmLedgerDeleteReason::User,+ )+ .unwrap();+ for n in 0..50 {+ ledger+ .append_spawn(&RlmSpawnInput {+ child_id: format!("sub-f{n}"),+ parent: format!("/gone-root/sess-f{n}.jsonl"),+ child: format!("/gone-root/sess-f{n}-child.jsonl"),+ depth: 1,+ name: format!("f{n}"),+ })+ .unwrap();+ }++ let edges = ledger.live_edges_of_parent(&parent).unwrap();+ let ids: Vec<&str> = edges.iter().map(|edge| edge.child_id.as_str()).collect();+ assert_eq!(ids, ["sub-a", "sub-b"], "{edges:?}");+ assert_eq!(+ edges[0].child,+ canonical_session_path(&moved_child)+ .to_string_lossy()+ .to_string(),+ "the moved child resolves through its session id"+ );++ let state = ledger.replay_of_parent(&parent).unwrap();+ let mut resolver = LivePathResolver::new(agent_dir.clone(), sessions_dir.clone());+ let scoped = resolve_live_edges(&state, &mut resolver);+ assert_eq!(scoped.len(), 2, "{scoped:?}");+ assert_eq!(+ resolver.resolved.len(),+ 3,+ "{:?}",+ resolver.resolved.keys().collect::<Vec<_>>()+ );+ assert!(+ resolver.artifact_index.is_none(),+ "no foreign miss walks the artifacts tree"+ );+ let full = ledger.replay_cached().unwrap();+ let mut unscoped = LivePathResolver::new(agent_dir, sessions_dir);+ assert_eq!(resolve_live_edges(&full, &mut unscoped).len(), 2);+ assert!(unscoped.artifact_index.is_some());+ assert_eq!(unscoped.resolved.len(), 103);+}++/// A parent that moved (recorded path gone, same stem under the sessions+/// dir) still lists its children through the scoped read; an edge whose+/// recorded parent merely shares the opened file's stem but resolves to+/// another live file is not the opened parent's.+#[test]+fn live_edges_of_parent_follows_a_moved_parent() {+ let agent_dir = temp_dir("scoped-moved");+ let sessions_dir = agent_dir.join("sessions");+ fs::create_dir_all(&sessions_dir).unwrap();+ let ledger = ledger_over(&agent_dir, &sessions_dir);++ let recorded_parent = "/old-root/sessions/sess-p.jsonl";+ let child = sessions_dir.join("sess-c.jsonl");+ let live_parent = sessions_dir.join("sess-p.jsonl");+ fs::write(&child, "{}").unwrap();+ fs::write(&live_parent, "{}").unwrap();+ ledger+ .append_spawn(&RlmSpawnInput {+ child_id: "sub-1".into(),+ parent: recorded_parent.into(),+ child: child.to_string_lossy().into(),+ depth: 1,+ name: "w".into(),+ })+ .unwrap();+ let edges = ledger.live_edges_of_parent(&live_parent).unwrap();+ assert_eq!(edges.len(), 1, "{edges:?}");+ assert_eq!(+ edges[0].parent,+ canonical_session_path(&live_parent)+ .to_string_lossy()+ .to_string(),+ "the moved parent still lists its children"+ );++ let elsewhere = agent_dir+ .join("session-artifacts")+ .join("host")+ .join("sess-p2.jsonl");+ fs::create_dir_all(elsewhere.parent().unwrap()).unwrap();+ fs::write(&elsewhere, "{}").unwrap();+ fs::write(sessions_dir.join("sess-p2.jsonl"), "{}").unwrap();+ fs::write(sessions_dir.join("sess-c2.jsonl"), "{}").unwrap();+ ledger+ .append_spawn(&RlmSpawnInput {+ child_id: "sub-2".into(),+ parent: "/old-root/sessions/sess-p2.jsonl".into(),+ child: sessions_dir.join("sess-c2.jsonl").to_string_lossy().into(),+ depth: 1,+ name: "w2".into(),+ })+ .unwrap();+ let edges = ledger.live_edges_of_parent(&elsewhere).unwrap();+ assert!(+ edges.is_empty(),+ "a same-stem parent resolved to another live file is not the opened parent: {edges:?}"+ );+}+#[test]fn duplicate_child_path_and_bad_spawn_inputs_fail_but_bad_lines_skip() {let dir = temp_dir("dup");diff --git a/crates/pa-daemon/src/worker/create.rs b/crates/pa-daemon/src/worker/create.rsindex bea876a09d..afcaa92eaa 100644--- a/crates/pa-daemon/src/worker/create.rs+++ b/crates/pa-daemon/src/worker/create.rs@@ -596,7 +596,9 @@ impl Worker {}) {return response_failure(None, "create", &error.to_string(), None);}- self.reseed_rlm_children().await;+ if opened_existing_session {+ self.reseed_rlm_children().await;+ }// The built-in Herdr connector binds here, per session: the pane// identity comes from the create payload's client env (the client// that owns the pane sent it), never from this process's ambient