Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
Reapply "Merge pull request #1696 from akitaonrails/feat/1692-status-consolidation-queue"
AkitaOnRailsdf20d8f192e1
diff --git a/crates/ai-memory-cli/src/commands/status.rs b/crates/ai-memory-cli/src/commands/status.rsindex bf7462ae..3deebab0 100644--- a/crates/ai-memory-cli/src/commands/status.rs+++ b/crates/ai-memory-cli/src/commands/status.rs@@ -606,6 +606,44 @@ mod tests {}assert_eq!(value["recovery"]["journal_pending"], 2);assert_eq!(value["counts"]["observations"], 4);+ assert!(+ value.get("consolidation").is_none(),+ "an older server has no queue, and its shape is unchanged: {value}"+ );++ // A server that reports its queue adds exactly that one key (#1692).+ let mut with_queue = report_json.clone();+ with_queue["consolidation"] = serde_json::json!({+ "pending": 2,+ "running": 0,+ "completed": 9,+ "failed": 1,+ "failed_sessions": [{+ "session_id": "0192b6a1-4c2e-7d3f-8a5b-1234567890ab",+ "attempts": 5,+ "failed_at_us": 1_700_000_000_000_000_i64,+ "last_error": "provider error 400: quota exceeded",+ }],+ });+ let queued = json_report(+ serde_json::from_value(with_queue).unwrap(),+ SpoolHealth::default(),+ 0,+ 0,+ 0,+ None,+ "denylist",+ &endpoint,+ crate::local_auth::AuthPosture::NoneLoopback,+ None,+ );+ assert_eq!(queued["consolidation"]["failed"], 1);+ assert_eq!(queued["consolidation"]["pending"], 2);+ assert_eq!(+ queued["consolidation"]["failed_sessions"][0]["session_id"],+ "0192b6a1-4c2e-7d3f-8a5b-1234567890ab"+ );+ assert_eq!(queued["consolidation"]["failed_sessions"][0]["attempts"], 5);// The server's version stays at the top level; the client's is new.assert_eq!(value["version"], "2.6.0");diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex 593a5ad8..5d721bf5 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -1280,8 +1280,49 @@ pub struct StatusReport {/// Wiki-format state: whether the OKF migration has run, and the/// pre-migration backup archive if its receipt still exists.pub wiki_format: WikiFormatStatus,+ /// SessionEnd consolidation queue: counts plus the sessions that gave up,+ /// so a watchdog can re-run them once the cause is fixed (#1692).+ pub consolidation: ConsolidationStatus,}+/// Consolidation section of [`StatusReport`].+#[derive(Debug, serde::Serialize, serde::Deserialize)]+pub struct ConsolidationStatus {+ /// Jobs waiting for a worker, including ones backing off to retry.+ pub pending: u64,+ /// Jobs a worker holds right now.+ pub running: u64,+ /// Jobs that finished.+ pub completed: u64,+ /// Jobs that gave up after their attempts.+ pub failed: u64,+ /// Most recent failures, newest first, bounded to+ /// [`CONSOLIDATION_FAILED_SESSIONS_CAP`]. The session id is what a+ /// re-consolidation needs; the error is the worker's own bounded text,+ /// scrubbed before it leaves the server.+ pub failed_sessions: Vec<FailedConsolidationSession>,+}++/// One failed session in [`ConsolidationStatus::failed_sessions`].+#[derive(Debug, serde::Serialize, serde::Deserialize)]+pub struct FailedConsolidationSession {+ /// Session whose consolidation gave up.+ pub session_id: String,+ /// Provider attempts spent on it.+ pub attempts: u32,+ /// Unix microseconds of the final failure.+ pub failed_at_us: i64,+ /// Bounded, scrubbed error the worker recorded.+ pub last_error: String,+}++/// How many failed sessions one status response carries. The counts above+/// stay exact; the list is a sample, newest first, so the payload cannot grow+/// with the backlog.+pub const CONSOLIDATION_FAILED_SESSIONS_CAP: u32 = 20;+/// Longest error text one failed session carries, in characters.+const CONSOLIDATION_ERROR_CHARS: usize = 200;+/// Wiki-format section of [`StatusReport`].#[derive(Debug, serde::Serialize, serde::Deserialize)]pub struct WikiFormatStatus {@@ -1394,6 +1435,9 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons.unwrap_or(false);let backup = ai_memory_wiki::backup::BackupReceipt::load(&state.data_dir).filter(ai_memory_wiki::backup::BackupReceipt::archive_present);+ let consolidation =+ consolidation_status(&state, &state.reader.session_consolidation_queue().await)+ .await;let report = StatusReport {version: env!("CARGO_PKG_VERSION").to_string(),data_dir: state.data_dir.display().to_string(),@@ -1412,6 +1456,7 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons.map(|r| r.archive_path.display().to_string()),backup_archive_bytes: backup.as_ref().map(|r| r.size_bytes),},+ consolidation,};(StatusCode::OK,@@ -1430,6 +1475,50 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons}}+/// The consolidation section of a status report.+///+/// A failure to read the queue degrades to zeros, like the storage figures+/// above: `status` is also the health probe, and a queue query must not take+/// the whole answer down.+async fn consolidation_status(+ state: &AdminState,+ queue: &Result<ai_memory_store::SessionConsolidationQueue, ai_memory_store::StoreError>,+) -> ConsolidationStatus {+ let queue = queue.as_ref().copied().unwrap_or_default();+ let failed_sessions = match state+ .reader+ .failed_consolidation_jobs(CONSOLIDATION_FAILED_SESSIONS_CAP)+ .await+ {+ Ok(jobs) => jobs+ .into_iter()+ .map(|job| FailedConsolidationSession {+ session_id: job.session_id,+ attempts: job.attempts,+ failed_at_us: job.failed_at_us,+ // The worker truncates what it records, but that text comes+ // from a provider body: scrub it before an operator's terminal+ // sees it, exactly like `/admin/llm-test`.+ last_error: state+ .wiki+ .sanitizer()+ .scrub(&job.last_error)+ .chars()+ .take(CONSOLIDATION_ERROR_CHARS)+ .collect(),+ })+ .collect(),+ Err(_) => Vec::new(),+ };+ ConsolidationStatus {+ pending: queue.pending,+ running: queue.running,+ completed: queue.completed,+ failed: queue.failed,+ failed_sessions,+ }+}+// ---------------------------------------------------------------------// open-sessions// ---------------------------------------------------------------------diff --git a/crates/ai-memory-mcp/tests/suite/admin_status_search.rs b/crates/ai-memory-mcp/tests/suite/admin_status_search.rsindex 1766a9fe..f6f673d8 100644--- a/crates/ai-memory-mcp/tests/suite/admin_status_search.rs+++ b/crates/ai-memory-mcp/tests/suite/admin_status_search.rs@@ -158,6 +158,89 @@ async fn status_returns_counts_and_paths() {);}+/// `GET /admin/status` carries the consolidation queue: exact counts plus the+/// sessions that gave up, newest first, with the worker's error scrubbed — so a+/// watchdog reads one JSON field instead of grepping the journal for a log+/// line (#1692). The queue's own `/health` view stays counts-only.+#[tokio::test]+async fn status_carries_the_consolidation_queue_and_its_failed_sessions() {+ let tmp = TempDir::new().unwrap();+ let (state, store) = make_admin_state(&tmp).await;+ let ws = store+ .writer+ .get_or_create_workspace("default".to_string())+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();+ let session = ai_memory_core::SessionId::new();+ store+ .writer+ .begin_session(ai_memory_core::NewSession {+ occurred_at: None,+ id: session,+ workspace_id: ws,+ project_id: proj,+ agent_kind: ai_memory_core::AgentKind::Codex,+ cwd: Some("/repo".into()),+ actor_user: None,+ })+ .await+ .unwrap();+ // A provider body can quote the key it rejected; the status copy is+ // scrubbed like `/admin/llm-test`'s.+ let planted = format!("{} from provider", "sk-live-0123456789abcdef");+ rusqlite::Connection::open(store.db_path())+ .unwrap()+ .execute(+ "INSERT INTO session_consolidation_jobs \+ (session_id, workspace_id, project_id, generation, state, requested_at, \+ next_attempt_at, attempts, completed_at, last_error) \+ VALUES (?1, ?2, ?3, 1, 'failed', 0, 0, 5, 1700000000000000, ?4)",+ rusqlite::params![+ session.as_bytes().to_vec(),+ ws.as_bytes().to_vec(),+ proj.as_bytes().to_vec(),+ planted+ ],+ )+ .unwrap();+ let app = admin_router(state);++ let resp = app+ .oneshot(+ Request::builder()+ .method("GET")+ .uri("/admin/status")+ .body(Body::empty())+ .unwrap(),+ )+ .await+ .unwrap();+ assert_eq!(resp.status(), StatusCode::OK);+ let body: serde_json::Value = serde_json::from_slice(&body_bytes(resp).await).unwrap();++ assert_eq!(body["consolidation"]["failed"], 1);+ assert_eq!(body["consolidation"]["pending"], 0);+ assert_eq!(+ body["consolidation"]["failed_sessions"][0]["session_id"],+ serde_json::json!(session.to_string())+ );+ assert_eq!(body["consolidation"]["failed_sessions"][0]["attempts"], 5);+ let rendered = body.to_string();+ assert!(+ !rendered.contains("sk-live-0123456789abcdef"),+ "the provider body must not reach an operator: {rendered}"+ );+ assert!(+ rendered.contains("from provider"),+ "the error class stays useful: {rendered}"+ );+}+/// After an embedding-model switch, `/admin/status` counts a page whose/// only vector was stored under the old triple as pending — status must/// match the embedder's stored identity the way `handle_embed` writes it —diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex efc0c9ae..071df62b 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -1424,6 +1424,26 @@ pub struct SessionConsolidationQueue {pub oldest_pending_requested_at_us: Option<i64>,}+/// One consolidation job that gave up after its attempts, for the operator's+/// `status` (#1692).+///+/// Deliberately **not** part of [`SessionConsolidationQueue`]: that struct+/// feeds the web health panel, which is documented to carry counts only. This+/// one carries the session id (so it can be re-consolidated once the cause is+/// fixed) and the bounded error the worker recorded, and is served only on+/// `/admin/status`, which is root-only on a multi-user server.+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]+pub struct FailedConsolidationJob {+ /// Session whose consolidation gave up.+ pub session_id: String,+ /// Provider attempts spent on it.+ pub attempts: u32,+ /// Unix microseconds of the final failure.+ pub failed_at_us: i64,+ /// The worker's bounded error text, scrubbed by the caller before display.+ pub last_error: String,+}+/// One open session selected by [`ReaderPool::idle_open_sessions`] for the/// server-side idle auto-end tick: its last activity/// (`MAX(observations.created_at)`, falling back to `started_at`) is older@@ -12639,6 +12659,49 @@ impl ReaderPool {.await}+ /// The most recent consolidation jobs that gave up, newest first,+ /// bounded to `limit`.+ ///+ /// # Errors+ /// Propagates any SQL or pool error.+ pub async fn failed_consolidation_jobs(+ &self,+ limit: u32,+ ) -> StoreResult<Vec<FailedConsolidationJob>> {+ let limit = i64::from(limit.clamp(1, 100));+ self.with_conn(move |conn| {+ let mut stmt = conn.prepare_cached(+ "SELECT session_id, attempts, COALESCE(completed_at, 0), \+ COALESCE(last_error, '') \+ FROM session_consolidation_jobs WHERE state = 'failed' \+ ORDER BY COALESCE(completed_at, 0) DESC LIMIT ?1",+ )?;+ let rows = stmt.query_map([limit], |row| {+ Ok((+ row.get::<_, Vec<u8>>(0)?,+ row.get::<_, i64>(1)?,+ row.get::<_, i64>(2)?,+ row.get::<_, String>(3)?,+ ))+ })?;+ let mut out = Vec::new();+ for row in rows {+ let (session, attempts, failed_at_us, last_error) = row?;+ let Ok(session_id) = ai_memory_core::SessionId::from_slice(&session) else {+ continue;+ };+ out.push(FailedConsolidationJob {+ session_id: session_id.to_string(),+ attempts: u32::try_from(attempts).unwrap_or(0),+ failed_at_us,+ last_error,+ });+ }+ Ok(out)+ })+ .await+ }+/// Physical storage figures for the database file.////// Three `PRAGMA` reads, no table scan, so it is cheap enough to sit indiff --git a/crates/ai-memory-store/tests/suite/mod.rs b/crates/ai-memory-store/tests/suite/mod.rsindex ede578cd..8cf34377 100644--- a/crates/ai-memory-store/tests/suite/mod.rs+++ b/crates/ai-memory-store/tests/suite/mod.rs@@ -30,6 +30,7 @@ mod retrieval_observation_stream;mod retrieval_superseded;mod retrieval_tuning_streams;mod session_activity;+mod session_consolidation_status;mod session_ids_touching_scope;mod session_move;mod session_observations;diff --git a/crates/ai-memory-store/tests/suite/session_consolidation_status.rs b/crates/ai-memory-store/tests/suite/session_consolidation_status.rsnew file mode 100644index 00000000..ed0b33fd--- /dev/null+++ b/crates/ai-memory-store/tests/suite/session_consolidation_status.rs@@ -0,0 +1,164 @@+//! The consolidation queue as `status` and the health panel read it (#1692).+//!+//! `SessionConsolidationQueue` stays counts-and-timestamps only — it feeds the+//! web health panel — while `failed_consolidation_jobs` is the operator-facing+//! companion on `/admin/status`: bounded, newest first, and carrying the+//! session id a re-consolidation needs.++use ai_memory_core::{AgentKind, NewSession, ProjectId, SessionId, WorkspaceId};+use ai_memory_store::Store;+use rusqlite::params;++fn workspace_and_project(store: &Store) -> (WorkspaceId, ProjectId) {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();+ rt.block_on(async {+ let ws = store+ .writer+ .get_or_create_workspace("default".to_string())+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();+ (ws, proj)+ })+}++/// One ended session with a consolidation job in `state`.+fn seed_job(+ store: &Store,+ ws: WorkspaceId,+ proj: ProjectId,+ state: &str,+ attempts: i64,+ completed_at: Option<i64>,+ last_error: Option<&str>,+) -> SessionId {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();+ let session = SessionId::new();+ rt.block_on(async {+ store+ .writer+ .begin_session(NewSession {+ occurred_at: None,+ id: session,+ workspace_id: ws,+ project_id: proj,+ agent_kind: AgentKind::Codex,+ cwd: Some("/repo".into()),+ actor_user: None,+ })+ .await+ .unwrap();+ });+ rusqlite::Connection::open(store.db_path())+ .unwrap()+ .execute(+ "INSERT INTO session_consolidation_jobs \+ (session_id, workspace_id, project_id, generation, state, requested_at, \+ next_attempt_at, attempts, completed_at, last_error) \+ VALUES (?1, ?2, ?3, 1, ?4, 0, 0, ?5, ?6, ?7)",+ params![+ session.as_bytes().to_vec(),+ ws.as_bytes().to_vec(),+ proj.as_bytes().to_vec(),+ state,+ attempts,+ completed_at,+ last_error+ ],+ )+ .unwrap();+ session+}++#[test]+fn failed_jobs_are_newest_first_bounded_and_never_a_success() {+ let tmp = tempfile::TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let (ws, proj) = workspace_and_project(&store);++ let older = seed_job(+ &store,+ ws,+ proj,+ "failed",+ 5,+ Some(1_000),+ Some("provider error 400: quota exceeded"),+ );+ let newer = seed_job(+ &store,+ ws,+ proj,+ "failed",+ 5,+ Some(2_000),+ Some("provider error 429: rate limited"),+ );+ // Everything that is not a final failure stays out of the list.+ seed_job(&store, ws, proj, "completed", 1, Some(3_000), None);+ seed_job(&store, ws, proj, "pending", 0, None, None);+ seed_job(&store, ws, proj, "running", 1, None, None);+ seed_job(&store, ws, proj, "superseded", 0, None, None);++ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();+ let jobs = rt+ .block_on(store.reader.failed_consolidation_jobs(20))+ .unwrap();++ assert_eq!(jobs.len(), 2, "{jobs:?}");+ assert_eq!(jobs[0].session_id, newer.to_string(), "newest first");+ assert_eq!(jobs[1].session_id, older.to_string());+ assert_eq!(jobs[0].attempts, 5);+ assert_eq!(jobs[0].failed_at_us, 2_000);+ assert!(jobs[0].last_error.contains("quota") || jobs[0].last_error.contains("rate limited"));++ // The cap is honoured and still answers the newest entries.+ let capped = rt+ .block_on(store.reader.failed_consolidation_jobs(1))+ .unwrap();+ assert_eq!(capped.len(), 1);+ assert_eq!(capped[0].session_id, newer.to_string());+}++/// The health panel's own view keeps the property the new list must not+/// erode: no session ids and no provider text.+#[test]+fn the_health_queue_stays_counts_only() {+ let tmp = tempfile::TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let (ws, proj) = workspace_and_project(&store);+ seed_job(+ &store,+ ws,+ proj,+ "failed",+ 5,+ Some(1_000),+ Some("SECRET_SENTINEL provider body"),+ );++ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();+ let queue = rt+ .block_on(store.reader.session_consolidation_queue())+ .unwrap();+ assert_eq!(queue.failed, 1);+ let rendered = serde_json::to_string(&queue).unwrap();+ assert!(!rendered.contains("SECRET_SENTINEL"), "{rendered}");+ assert!(!rendered.contains("session"), "{rendered}");+}diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex 10b2407d..316f749c 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -925,6 +925,14 @@ turns those into warnings with the fix commands `ai-memory status` printsThe page holds counts, timestamps and configured labels only; provider error text,page content and scope names never reach it.+`GET /admin/status` (and `ai-memory status --json`) additionally carries a+`consolidation` block — the same counts, plus up to 20 `failed_sessions`+(`ReaderPool::failed_consolidation_jobs`: session id, attempts, failure time,+and the worker's bounded error through the privacy strip). The session id is+what a re-consolidation needs after the cause is fixed, so it belongs on the+operator surface and not on the health panel, whose "counts and timestamps+only" contract is unchanged (#1692).+## CLI subcommand surface```