Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

Reapply "Merge pull request #1696 from akitaonrails/feat/1692-status-consolidation-queue"

AkitaOnRailsdf20d8f192e1
diff --git a/crates/ai-memory-cli/src/commands/status.rs b/crates/ai-memory-cli/src/commands/status.rs
index bf7462ae..3deebab0 100644
--- a/crates/ai-memory-cli/src/commands/status.rs
+++ b/crates/ai-memory-cli/src/commands/status.rs
@@ -606,6 +606,44 @@ mod tests {
}
assert_eq!(value["recovery"]["journal_pending"], 2);
assert_eq!(value["counts"]["observations"], 4);
+ assert!(
+ value.get("consolidation").is_none(),
+ "an older server has no queue, and its shape is unchanged: {value}"
+ );
+
+ // A server that reports its queue adds exactly that one key (#1692).
+ let mut with_queue = report_json.clone();
+ with_queue["consolidation"] = serde_json::json!({
+ "pending": 2,
+ "running": 0,
+ "completed": 9,
+ "failed": 1,
+ "failed_sessions": [{
+ "session_id": "0192b6a1-4c2e-7d3f-8a5b-1234567890ab",
+ "attempts": 5,
+ "failed_at_us": 1_700_000_000_000_000_i64,
+ "last_error": "provider error 400: quota exceeded",
+ }],
+ });
+ let queued = json_report(
+ serde_json::from_value(with_queue).unwrap(),
+ SpoolHealth::default(),
+ 0,
+ 0,
+ 0,
+ None,
+ "denylist",
+ &endpoint,
+ crate::local_auth::AuthPosture::NoneLoopback,
+ None,
+ );
+ assert_eq!(queued["consolidation"]["failed"], 1);
+ assert_eq!(queued["consolidation"]["pending"], 2);
+ assert_eq!(
+ queued["consolidation"]["failed_sessions"][0]["session_id"],
+ "0192b6a1-4c2e-7d3f-8a5b-1234567890ab"
+ );
+ assert_eq!(queued["consolidation"]["failed_sessions"][0]["attempts"], 5);
// The server's version stays at the top level; the client's is new.
assert_eq!(value["version"], "2.6.0");
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 593a5ad8..5d721bf5 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -1280,8 +1280,49 @@ pub struct StatusReport {
/// Wiki-format state: whether the OKF migration has run, and the
/// pre-migration backup archive if its receipt still exists.
pub wiki_format: WikiFormatStatus,
+ /// SessionEnd consolidation queue: counts plus the sessions that gave up,
+ /// so a watchdog can re-run them once the cause is fixed (#1692).
+ pub consolidation: ConsolidationStatus,
}
+/// Consolidation section of [`StatusReport`].
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct ConsolidationStatus {
+ /// Jobs waiting for a worker, including ones backing off to retry.
+ pub pending: u64,
+ /// Jobs a worker holds right now.
+ pub running: u64,
+ /// Jobs that finished.
+ pub completed: u64,
+ /// Jobs that gave up after their attempts.
+ pub failed: u64,
+ /// Most recent failures, newest first, bounded to
+ /// [`CONSOLIDATION_FAILED_SESSIONS_CAP`]. The session id is what a
+ /// re-consolidation needs; the error is the worker's own bounded text,
+ /// scrubbed before it leaves the server.
+ pub failed_sessions: Vec<FailedConsolidationSession>,
+}
+
+/// One failed session in [`ConsolidationStatus::failed_sessions`].
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct FailedConsolidationSession {
+ /// Session whose consolidation gave up.
+ pub session_id: String,
+ /// Provider attempts spent on it.
+ pub attempts: u32,
+ /// Unix microseconds of the final failure.
+ pub failed_at_us: i64,
+ /// Bounded, scrubbed error the worker recorded.
+ pub last_error: String,
+}
+
+/// How many failed sessions one status response carries. The counts above
+/// stay exact; the list is a sample, newest first, so the payload cannot grow
+/// with the backlog.
+pub const CONSOLIDATION_FAILED_SESSIONS_CAP: u32 = 20;
+/// Longest error text one failed session carries, in characters.
+const CONSOLIDATION_ERROR_CHARS: usize = 200;
+
/// Wiki-format section of [`StatusReport`].
#[derive(Debug, serde::Serialize, serde::Deserialize)]
pub struct WikiFormatStatus {
@@ -1394,6 +1435,9 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons
.unwrap_or(false);
let backup = ai_memory_wiki::backup::BackupReceipt::load(&state.data_dir)
.filter(ai_memory_wiki::backup::BackupReceipt::archive_present);
+ let consolidation =
+ consolidation_status(&state, &state.reader.session_consolidation_queue().await)
+ .await;
let report = StatusReport {
version: env!("CARGO_PKG_VERSION").to_string(),
data_dir: state.data_dir.display().to_string(),
@@ -1412,6 +1456,7 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons
.map(|r| r.archive_path.display().to_string()),
backup_archive_bytes: backup.as_ref().map(|r| r.size_bytes),
},
+ consolidation,
};
(
StatusCode::OK,
@@ -1430,6 +1475,50 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons
}
}
+/// The consolidation section of a status report.
+///
+/// A failure to read the queue degrades to zeros, like the storage figures
+/// above: `status` is also the health probe, and a queue query must not take
+/// the whole answer down.
+async fn consolidation_status(
+ state: &AdminState,
+ queue: &Result<ai_memory_store::SessionConsolidationQueue, ai_memory_store::StoreError>,
+) -> ConsolidationStatus {
+ let queue = queue.as_ref().copied().unwrap_or_default();
+ let failed_sessions = match state
+ .reader
+ .failed_consolidation_jobs(CONSOLIDATION_FAILED_SESSIONS_CAP)
+ .await
+ {
+ Ok(jobs) => jobs
+ .into_iter()
+ .map(|job| FailedConsolidationSession {
+ session_id: job.session_id,
+ attempts: job.attempts,
+ failed_at_us: job.failed_at_us,
+ // The worker truncates what it records, but that text comes
+ // from a provider body: scrub it before an operator's terminal
+ // sees it, exactly like `/admin/llm-test`.
+ last_error: state
+ .wiki
+ .sanitizer()
+ .scrub(&job.last_error)
+ .chars()
+ .take(CONSOLIDATION_ERROR_CHARS)
+ .collect(),
+ })
+ .collect(),
+ Err(_) => Vec::new(),
+ };
+ ConsolidationStatus {
+ pending: queue.pending,
+ running: queue.running,
+ completed: queue.completed,
+ failed: queue.failed,
+ failed_sessions,
+ }
+}
+
// ---------------------------------------------------------------------
// open-sessions
// ---------------------------------------------------------------------
diff --git a/crates/ai-memory-mcp/tests/suite/admin_status_search.rs b/crates/ai-memory-mcp/tests/suite/admin_status_search.rs
index 1766a9fe..f6f673d8 100644
--- a/crates/ai-memory-mcp/tests/suite/admin_status_search.rs
+++ b/crates/ai-memory-mcp/tests/suite/admin_status_search.rs
@@ -158,6 +158,89 @@ async fn status_returns_counts_and_paths() {
);
}
+/// `GET /admin/status` carries the consolidation queue: exact counts plus the
+/// sessions that gave up, newest first, with the worker's error scrubbed — so a
+/// watchdog reads one JSON field instead of grepping the journal for a log
+/// line (#1692). The queue's own `/health` view stays counts-only.
+#[tokio::test]
+async fn status_carries_the_consolidation_queue_and_its_failed_sessions() {
+ let tmp = TempDir::new().unwrap();
+ let (state, store) = make_admin_state(&tmp).await;
+ let ws = store
+ .writer
+ .get_or_create_workspace("default".to_string())
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+ let session = ai_memory_core::SessionId::new();
+ store
+ .writer
+ .begin_session(ai_memory_core::NewSession {
+ occurred_at: None,
+ id: session,
+ workspace_id: ws,
+ project_id: proj,
+ agent_kind: ai_memory_core::AgentKind::Codex,
+ cwd: Some("/repo".into()),
+ actor_user: None,
+ })
+ .await
+ .unwrap();
+ // A provider body can quote the key it rejected; the status copy is
+ // scrubbed like `/admin/llm-test`'s.
+ let planted = format!("{} from provider", "sk-live-0123456789abcdef");
+ rusqlite::Connection::open(store.db_path())
+ .unwrap()
+ .execute(
+ "INSERT INTO session_consolidation_jobs \
+ (session_id, workspace_id, project_id, generation, state, requested_at, \
+ next_attempt_at, attempts, completed_at, last_error) \
+ VALUES (?1, ?2, ?3, 1, 'failed', 0, 0, 5, 1700000000000000, ?4)",
+ rusqlite::params![
+ session.as_bytes().to_vec(),
+ ws.as_bytes().to_vec(),
+ proj.as_bytes().to_vec(),
+ planted
+ ],
+ )
+ .unwrap();
+ let app = admin_router(state);
+
+ let resp = app
+ .oneshot(
+ Request::builder()
+ .method("GET")
+ .uri("/admin/status")
+ .body(Body::empty())
+ .unwrap(),
+ )
+ .await
+ .unwrap();
+ assert_eq!(resp.status(), StatusCode::OK);
+ let body: serde_json::Value = serde_json::from_slice(&body_bytes(resp).await).unwrap();
+
+ assert_eq!(body["consolidation"]["failed"], 1);
+ assert_eq!(body["consolidation"]["pending"], 0);
+ assert_eq!(
+ body["consolidation"]["failed_sessions"][0]["session_id"],
+ serde_json::json!(session.to_string())
+ );
+ assert_eq!(body["consolidation"]["failed_sessions"][0]["attempts"], 5);
+ let rendered = body.to_string();
+ assert!(
+ !rendered.contains("sk-live-0123456789abcdef"),
+ "the provider body must not reach an operator: {rendered}"
+ );
+ assert!(
+ rendered.contains("from provider"),
+ "the error class stays useful: {rendered}"
+ );
+}
+
/// After an embedding-model switch, `/admin/status` counts a page whose
/// only vector was stored under the old triple as pending — status must
/// match the embedder's stored identity the way `handle_embed` writes it —
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index efc0c9ae..071df62b 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -1424,6 +1424,26 @@ pub struct SessionConsolidationQueue {
pub oldest_pending_requested_at_us: Option<i64>,
}
+/// One consolidation job that gave up after its attempts, for the operator's
+/// `status` (#1692).
+///
+/// Deliberately **not** part of [`SessionConsolidationQueue`]: that struct
+/// feeds the web health panel, which is documented to carry counts only. This
+/// one carries the session id (so it can be re-consolidated once the cause is
+/// fixed) and the bounded error the worker recorded, and is served only on
+/// `/admin/status`, which is root-only on a multi-user server.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
+pub struct FailedConsolidationJob {
+ /// Session whose consolidation gave up.
+ pub session_id: String,
+ /// Provider attempts spent on it.
+ pub attempts: u32,
+ /// Unix microseconds of the final failure.
+ pub failed_at_us: i64,
+ /// The worker's bounded error text, scrubbed by the caller before display.
+ pub last_error: String,
+}
+
/// One open session selected by [`ReaderPool::idle_open_sessions`] for the
/// server-side idle auto-end tick: its last activity
/// (`MAX(observations.created_at)`, falling back to `started_at`) is older
@@ -12639,6 +12659,49 @@ impl ReaderPool {
.await
}
+ /// The most recent consolidation jobs that gave up, newest first,
+ /// bounded to `limit`.
+ ///
+ /// # Errors
+ /// Propagates any SQL or pool error.
+ pub async fn failed_consolidation_jobs(
+ &self,
+ limit: u32,
+ ) -> StoreResult<Vec<FailedConsolidationJob>> {
+ let limit = i64::from(limit.clamp(1, 100));
+ self.with_conn(move |conn| {
+ let mut stmt = conn.prepare_cached(
+ "SELECT session_id, attempts, COALESCE(completed_at, 0), \
+ COALESCE(last_error, '') \
+ FROM session_consolidation_jobs WHERE state = 'failed' \
+ ORDER BY COALESCE(completed_at, 0) DESC LIMIT ?1",
+ )?;
+ let rows = stmt.query_map([limit], |row| {
+ Ok((
+ row.get::<_, Vec<u8>>(0)?,
+ row.get::<_, i64>(1)?,
+ row.get::<_, i64>(2)?,
+ row.get::<_, String>(3)?,
+ ))
+ })?;
+ let mut out = Vec::new();
+ for row in rows {
+ let (session, attempts, failed_at_us, last_error) = row?;
+ let Ok(session_id) = ai_memory_core::SessionId::from_slice(&session) else {
+ continue;
+ };
+ out.push(FailedConsolidationJob {
+ session_id: session_id.to_string(),
+ attempts: u32::try_from(attempts).unwrap_or(0),
+ failed_at_us,
+ last_error,
+ });
+ }
+ Ok(out)
+ })
+ .await
+ }
+
/// Physical storage figures for the database file.
///
/// Three `PRAGMA` reads, no table scan, so it is cheap enough to sit in
diff --git a/crates/ai-memory-store/tests/suite/mod.rs b/crates/ai-memory-store/tests/suite/mod.rs
index ede578cd..8cf34377 100644
--- a/crates/ai-memory-store/tests/suite/mod.rs
+++ b/crates/ai-memory-store/tests/suite/mod.rs
@@ -30,6 +30,7 @@ mod retrieval_observation_stream;
mod retrieval_superseded;
mod retrieval_tuning_streams;
mod session_activity;
+mod session_consolidation_status;
mod session_ids_touching_scope;
mod session_move;
mod session_observations;
diff --git a/crates/ai-memory-store/tests/suite/session_consolidation_status.rs b/crates/ai-memory-store/tests/suite/session_consolidation_status.rs
new file mode 100644
index 00000000..ed0b33fd
--- /dev/null
+++ b/crates/ai-memory-store/tests/suite/session_consolidation_status.rs
@@ -0,0 +1,164 @@
+//! The consolidation queue as `status` and the health panel read it (#1692).
+//!
+//! `SessionConsolidationQueue` stays counts-and-timestamps only — it feeds the
+//! web health panel — while `failed_consolidation_jobs` is the operator-facing
+//! companion on `/admin/status`: bounded, newest first, and carrying the
+//! session id a re-consolidation needs.
+
+use ai_memory_core::{AgentKind, NewSession, ProjectId, SessionId, WorkspaceId};
+use ai_memory_store::Store;
+use rusqlite::params;
+
+fn workspace_and_project(store: &Store) -> (WorkspaceId, ProjectId) {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ rt.block_on(async {
+ let ws = store
+ .writer
+ .get_or_create_workspace("default".to_string())
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+ (ws, proj)
+ })
+}
+
+/// One ended session with a consolidation job in `state`.
+fn seed_job(
+ store: &Store,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ state: &str,
+ attempts: i64,
+ completed_at: Option<i64>,
+ last_error: Option<&str>,
+) -> SessionId {
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ let session = SessionId::new();
+ rt.block_on(async {
+ store
+ .writer
+ .begin_session(NewSession {
+ occurred_at: None,
+ id: session,
+ workspace_id: ws,
+ project_id: proj,
+ agent_kind: AgentKind::Codex,
+ cwd: Some("/repo".into()),
+ actor_user: None,
+ })
+ .await
+ .unwrap();
+ });
+ rusqlite::Connection::open(store.db_path())
+ .unwrap()
+ .execute(
+ "INSERT INTO session_consolidation_jobs \
+ (session_id, workspace_id, project_id, generation, state, requested_at, \
+ next_attempt_at, attempts, completed_at, last_error) \
+ VALUES (?1, ?2, ?3, 1, ?4, 0, 0, ?5, ?6, ?7)",
+ params![
+ session.as_bytes().to_vec(),
+ ws.as_bytes().to_vec(),
+ proj.as_bytes().to_vec(),
+ state,
+ attempts,
+ completed_at,
+ last_error
+ ],
+ )
+ .unwrap();
+ session
+}
+
+#[test]
+fn failed_jobs_are_newest_first_bounded_and_never_a_success() {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (ws, proj) = workspace_and_project(&store);
+
+ let older = seed_job(
+ &store,
+ ws,
+ proj,
+ "failed",
+ 5,
+ Some(1_000),
+ Some("provider error 400: quota exceeded"),
+ );
+ let newer = seed_job(
+ &store,
+ ws,
+ proj,
+ "failed",
+ 5,
+ Some(2_000),
+ Some("provider error 429: rate limited"),
+ );
+ // Everything that is not a final failure stays out of the list.
+ seed_job(&store, ws, proj, "completed", 1, Some(3_000), None);
+ seed_job(&store, ws, proj, "pending", 0, None, None);
+ seed_job(&store, ws, proj, "running", 1, None, None);
+ seed_job(&store, ws, proj, "superseded", 0, None, None);
+
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ let jobs = rt
+ .block_on(store.reader.failed_consolidation_jobs(20))
+ .unwrap();
+
+ assert_eq!(jobs.len(), 2, "{jobs:?}");
+ assert_eq!(jobs[0].session_id, newer.to_string(), "newest first");
+ assert_eq!(jobs[1].session_id, older.to_string());
+ assert_eq!(jobs[0].attempts, 5);
+ assert_eq!(jobs[0].failed_at_us, 2_000);
+ assert!(jobs[0].last_error.contains("quota") || jobs[0].last_error.contains("rate limited"));
+
+ // The cap is honoured and still answers the newest entries.
+ let capped = rt
+ .block_on(store.reader.failed_consolidation_jobs(1))
+ .unwrap();
+ assert_eq!(capped.len(), 1);
+ assert_eq!(capped[0].session_id, newer.to_string());
+}
+
+/// The health panel's own view keeps the property the new list must not
+/// erode: no session ids and no provider text.
+#[test]
+fn the_health_queue_stays_counts_only() {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (ws, proj) = workspace_and_project(&store);
+ seed_job(
+ &store,
+ ws,
+ proj,
+ "failed",
+ 5,
+ Some(1_000),
+ Some("SECRET_SENTINEL provider body"),
+ );
+
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap();
+ let queue = rt
+ .block_on(store.reader.session_consolidation_queue())
+ .unwrap();
+ assert_eq!(queue.failed, 1);
+ let rendered = serde_json::to_string(&queue).unwrap();
+ assert!(!rendered.contains("SECRET_SENTINEL"), "{rendered}");
+ assert!(!rendered.contains("session"), "{rendered}");
+}
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 10b2407d..316f749c 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -925,6 +925,14 @@ turns those into warnings with the fix commands `ai-memory status` prints
The page holds counts, timestamps and configured labels only; provider error text,
page content and scope names never reach it.
+`GET /admin/status` (and `ai-memory status --json`) additionally carries a
+`consolidation` block — the same counts, plus up to 20 `failed_sessions`
+(`ReaderPool::failed_consolidation_jobs`: session id, attempts, failure time,
+and the worker's bounded error through the privacy strip). The session id is
+what a re-consolidation needs after the cause is fixed, so it belongs on the
+operator surface and not on the health panel, whose "counts and timestamps
+only" contract is unchanged (#1692).
+
## CLI subcommand surface
```