Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(status): tell more of the truth - the 2.0 status audit (item 7)
AkitaOnRailsb2870f9e7b2f
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 081081de..fa2a7355 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0## [Unreleased]### Added+- `status` tells more of the truth (2.0 status audit): the wiki-format+ line (OKF migration state and the pre-migration backup archive while+ it exists), stored embedding triples by provider/model/dim, typed-edge+ counts by relation, and a write-queue depth gauge that surfaces a+ backpressured (wedged) writer — shown only when non-zero.- An opt-in cross-session "experience" pass(`[auto_improve.scheduler] experience_every_sessions`): every N newlycompleted sessions, the last few session summaries are reviewed side@@ -83,6 +88,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0`docs/okf.md` and `docs/MIGRATION-2.0.md`.### Fixed+- `status` no longer overstates missing embeddings: empty-body pages —+ which no embedder can ever cover and the backfill skips by rule — are+ excluded from the "latest pages missing" figure and reported on their+ own line (observed live as a permanently stuck 427).- Natural-language searches no longer surface stopword trash: barequeries drop English stopwords before the FTS5 OR-join, so a pagecontaining five "the"s cannot outrank the page whose content matches,diff --git a/crates/ai-memory-cli/src/commands/status.rs b/crates/ai-memory-cli/src/commands/status.rsindex 6cec019c..83169370 100644--- a/crates/ai-memory-cli/src/commands/status.rs+++ b/crates/ai-memory-cli/src/commands/status.rs@@ -50,6 +50,22 @@ struct Report {/// Passive process-scoped provider health.#[serde(default)]providers: ProviderHealthSnapshot,+ /// Write-queue depth `(queued, capacity)` (2.0 servers).+ #[serde(default)]+ write_queue: Option<(usize, usize)>,+ /// Wiki-format state (2.0 servers).+ #[serde(default)]+ wiki_format: Option<WikiFormatReport>,+}++#[derive(Debug, Default, Deserialize, Serialize)]+struct WikiFormatReport {+ #[serde(default)]+ okf_migrated: bool,+ #[serde(default)]+ backup_archive: Option<String>,+ #[serde(default)]+ backup_archive_bytes: Option<u64>,}#[derive(Debug, Deserialize, Serialize)]@@ -68,6 +84,10 @@ struct Derived {observations_fts_rows: u64,latest_pages_missing_embeddings: u64,#[serde(default)]+ latest_pages_unembeddable: u64,+ #[serde(default)]+ typed_links_from_latest_pages: Vec<(String, u64)>,+ #[serde(default)]embed_failures_unresolved: u64,#[serde(default)]embed_failures_recovered: u64,@@ -230,6 +250,18 @@ pub async fn run(config: &Config, args: StatusArgs) -> Result<()> {" embeddings: {} rows; {} latest pages missing",report.derived.embedding_rows, report.derived.latest_pages_missing_embeddings);+ if report.derived.latest_pages_unembeddable > 0 {+ println!(+ " unembeddable: {} (empty body; no embedder can cover these)",+ report.derived.latest_pages_unembeddable+ );+ }+ for triple in &report.derived.embedding_triples {+ println!(+ " {}/{} dim {}: {} rows",+ triple.provider, triple.model, triple.dim, triple.count+ );+ }// Only shown when there is something to act on. A recovered count with// no outstanding failures is history, not a problem.if report.derived.embed_failures_unresolved > 0 {@@ -265,6 +297,36 @@ pub async fn run(config: &Config, args: StatusArgs) -> Result<()> {report.derived.unresolved_links_from_latest_pages,report.derived.stale_links_from_latest_pages);+ if !report.derived.typed_links_from_latest_pages.is_empty() {+ let typed: Vec<String> = report+ .derived+ .typed_links_from_latest_pages+ .iter()+ .map(|(k, v)| format!("{k}: {v}"))+ .collect();+ println!(" typed edges: {}", typed.join(", "));+ }+ if let Some(wiki) = &report.wiki_format {+ let migrated = if wiki.okf_migrated {+ "OKF v0.2 (migrated)"+ } else {+ "OKF v0.2 (native, no migration needed)"+ };+ match (&wiki.backup_archive, wiki.backup_archive_bytes) {+ (Some(path), Some(bytes)) => println!(+ " wiki format: {migrated}; pre-migration backup still on disk: \+ {path} ({})",+ super::compact::human_bytes(bytes)+ ),+ _ => println!(" wiki format: {migrated}"),+ }+ }+ if let Some((queued, capacity)) = report.write_queue {+ // A queue pinned near capacity is the wedged-writer signal.+ if queued > 0 {+ println!(" write queue: {queued}/{capacity}");+ }+ }println!(" spool:");println!(" pending: {}", spool.pending);println!(" oldest: {}", spool_age_line(spool.oldest_age_ms));diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex 4f55418a..61831bb9 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -1083,6 +1083,24 @@ pub struct StatusReport {/// Hook-ingestion counters for this server process. Counts and one/// timestamp only — never captured content (#428).pub ingest: ai_memory_core::IngestMetricsSnapshot,+ /// Instantaneous write-queue depth `(queued, capacity)` — the+ /// wedged-writer signal (2.0 item 7).+ pub write_queue: (usize, usize),+ /// Wiki-format state: whether the OKF migration has run, and the+ /// pre-migration backup archive if its receipt still exists.+ pub wiki_format: WikiFormatStatus,+}++/// Wiki-format section of [`StatusReport`].+#[derive(Debug, serde::Serialize, serde::Deserialize)]+pub struct WikiFormatStatus {+ /// True once the OKF v0.2 wiki migration has been applied.+ pub okf_migrated: bool,+ /// Pre-migration backup archive path, when the receipt exists AND+ /// the archive file is still on disk.+ pub backup_archive: Option<String>,+ /// Size of that archive in bytes.+ pub backup_archive_bytes: Option<u64>,}/// `GET /admin/projects` — the authoritative list of `(workspace, project)`@@ -1110,6 +1128,14 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons// Three pragma reads; a failure here must not take down the// whole status response, which is also the health probe.let storage = state.reader.storage_status().await.unwrap_or_default();+ let okf_migrated = state+ .reader+ .wiki_migration_names()+ .await+ .map(|names| names.iter().any(|n| n.contains("okf")))+ .unwrap_or(false);+ let backup = ai_memory_wiki::backup::BackupReceipt::load(&state.data_dir)+ .filter(ai_memory_wiki::backup::BackupReceipt::archive_present);let report = StatusReport {version: env!("CARGO_PKG_VERSION").to_string(),data_dir: state.data_dir.display().to_string(),@@ -1120,6 +1146,14 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoResponsstorage,providers: state.provider_health.snapshot(),ingest: state.ingest_metrics.snapshot(),+ write_queue: state.writer.queue_depth(),+ wiki_format: WikiFormatStatus {+ okf_migrated,+ backup_archive: backup+ .as_ref()+ .map(|r| r.archive_path.display().to_string()),+ backup_archive_bytes: backup.as_ref().map(|r| r.size_bytes),+ },};(StatusCode::OK,@@ -7192,6 +7226,12 @@ mod tests {let json: serde_json::Value = serde_json::from_slice(&body).unwrap();assert_eq!(json["providers"]["llm"]["status"], "disabled");assert_eq!(json["providers"]["embedding"]["status"], "disabled");+ // 2.0 item 7: the report carries the wedged-writer gauge and the+ // wiki-format state — a fresh store is natively conformant (no+ // migration ran) with no backup archive.+ assert!(json["write_queue"].is_array(), "{json}");+ assert_eq!(json["wiki_format"]["okf_migrated"], false);+ assert!(json["wiki_format"]["backup_archive"].is_null());}/// The MCP-only complement: per-client tool-call counters serveddiff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 7c00b06a..d3ea8628 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -6537,6 +6537,77 @@ mod tests {}}+ /// Item 7 truthfulness: "missing embeddings" counts only pages a+ /// backfill can act on; empty-body pages are reported apart instead+ /// of inflating the actionable number forever.+ #[tokio::test]+ async fn missing_embeddings_excludes_unembeddable_pages() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "s", None)+ .await+ .unwrap();+ store+ .writer+ .upsert_page(sample_page(ws, proj, "notes/real.md", "actual content"))+ .await+ .unwrap();+ store+ .writer+ .upsert_page(sample_page(ws, proj, "notes/empty.md", " "))+ .await+ .unwrap();+ let derived = store.reader.derived_index_status().await.unwrap();+ assert_eq!(derived.latest_pages_missing_embeddings, 1, "{derived:?}");+ assert_eq!(derived.latest_pages_unembeddable, 1, "{derived:?}");+ }++ /// Typed edges surface in the derived status by relation.+ #[tokio::test]+ async fn typed_links_are_counted_by_relation() {+ let tmp = TempDir::new().unwrap();+ let store = Store::open(tmp.path()).unwrap();+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "s", None)+ .await+ .unwrap();+ let mut page = sample_page(ws, proj, "notes/a.md", "body");+ page.links = vec![+ ai_memory_core::LinkTarget {+ workspace: None,+ project: None,+ path: ai_memory_core::PagePath::new("notes/b.md").unwrap(),+ relation: Some(ai_memory_core::Relation::Fixes),+ },+ ai_memory_core::LinkTarget {+ workspace: None,+ project: None,+ path: ai_memory_core::PagePath::new("notes/c.md").unwrap(),+ relation: None,+ },+ ];+ store.writer.upsert_page(page).await.unwrap();+ let derived = store.reader.derived_index_status().await.unwrap();+ assert_eq!(+ derived.typed_links_from_latest_pages,+ vec![("fixes".to_string(), 1)],+ "{derived:?}"+ );+ }+#[tokio::test]async fn entity_stream_finds_and_weights_pages() {let tmp = TempDir::new().unwrap();diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex ab477a42..d391849a 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -946,8 +946,12 @@ pub struct DerivedIndexStatus {pub observations_rows: u64,/// Rows currently present in the observation FTS5 index.pub observations_fts_rows: u64,- /// Latest pages without any embedding row.+ /// Latest pages without any embedding row whose body is non-empty —+ /// i.e. the pages a backfill can actually act on.pub latest_pages_missing_embeddings: u64,+ /// Latest pages without an embedding whose body is empty; no+ /// embedder can ever cover these (the backfill skips them by rule).+ pub latest_pages_unembeddable: u64,/// Latest pages whose last embed attempt failed or was skipped and which/// still have no embedding. These are the pages an operator can act on.pub embed_failures_unresolved: u64,@@ -959,6 +963,9 @@ pub struct DerivedIndexStatus {pub embedding_rows: u64,/// Stored embedding triples and row counts.pub embedding_triples: Vec<EmbeddingTripleCount>,+ /// Typed relation edges (`link_type != 'references'`) from latest+ /// pages, as `(relation, count)` — the 2.0 typed-edge surface.+ pub typed_links_from_latest_pages: Vec<(String, u64)>,/// Outgoing links whose source page is latest.pub links_from_latest_pages: u64,/// Latest-page outgoing links whose target path has not resolved yet.@@ -7079,15 +7086,43 @@ impl ReaderPool {JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \JOIN page_embeddings pe ON pe.page_id = f.page_id",)?,+ // Aligned with the backfill's own skip rule: an+ // empty-body page can never be embedded, so counting it+ // as \"missing\" overstated the actionable number forever+ // (observed live: a stable 427 that no backfill could+ // ever clear). Unembeddable pages are reported apart.latest_pages_missing_embeddings: count(conn,"SELECT COUNT(*) \FROM pages pg \LEFT JOIN page_embeddings pe ON pe.page_id = pg.id \- WHERE pg.is_latest = 1 AND pe.page_id IS NULL",+ WHERE pg.is_latest = 1 AND pe.page_id IS NULL \+ AND TRIM(pg.body) != ''",+ )?,+ latest_pages_unembeddable: count(+ conn,+ "SELECT COUNT(*) \+ FROM pages pg \+ LEFT JOIN page_embeddings pe ON pe.page_id = pg.id \+ WHERE pg.is_latest = 1 AND pe.page_id IS NULL \+ AND TRIM(pg.body) = ''",)?,embedding_rows: count(conn, "SELECT COUNT(*) FROM page_embeddings")?,embedding_triples,+ typed_links_from_latest_pages: {+ let mut stmt = conn.prepare(+ "SELECT l.link_type, COUNT(*) FROM links l \+ JOIN pages fp ON fp.id = l.from_page_id AND fp.is_latest = 1 \+ WHERE l.link_type != 'references' \+ GROUP BY l.link_type ORDER BY l.link_type",+ )?;+ let rows: Vec<(String, i64)> = stmt+ .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?+ .collect::<Result<_, _>>()?;+ rows.into_iter()+ .map(|(k, v)| (k, u64::try_from(v).unwrap_or(0)))+ .collect()+ },links_from_latest_pages: count(conn,"SELECT COUNT(*) \diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex 1f1fffcf..b2a44330 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -1629,6 +1629,15 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Instantaneous write-queue depth as `(queued, capacity)`. A queue+ /// pinned near capacity means writers are being backpressured — the+ /// wedged-writer signal `status` surfaces (2.0 item 7).+ #[must_use]+ pub fn queue_depth(&self) -> (usize, usize) {+ let max = self.inner.tx.max_capacity();+ (max.saturating_sub(self.inner.tx.capacity()), max)+ }+/// Record a completed cross-session ("experience") pass for a scope.////// # Errors