Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(status): tell more of the truth - the 2.0 status audit (item 7)

AkitaOnRailsb2870f9e7b2f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 081081de..fa2a7355 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added
+- `status` tells more of the truth (2.0 status audit): the wiki-format
+ line (OKF migration state and the pre-migration backup archive while
+ it exists), stored embedding triples by provider/model/dim, typed-edge
+ counts by relation, and a write-queue depth gauge that surfaces a
+ backpressured (wedged) writer — shown only when non-zero.
- An opt-in cross-session "experience" pass
(`[auto_improve.scheduler] experience_every_sessions`): every N newly
completed sessions, the last few session summaries are reviewed side
@@ -83,6 +88,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`docs/okf.md` and `docs/MIGRATION-2.0.md`.
### Fixed
+- `status` no longer overstates missing embeddings: empty-body pages —
+ which no embedder can ever cover and the backfill skips by rule — are
+ excluded from the "latest pages missing" figure and reported on their
+ own line (observed live as a permanently stuck 427).
- Natural-language searches no longer surface stopword trash: bare
queries drop English stopwords before the FTS5 OR-join, so a page
containing five "the"s cannot outrank the page whose content matches,
diff --git a/crates/ai-memory-cli/src/commands/status.rs b/crates/ai-memory-cli/src/commands/status.rs
index 6cec019c..83169370 100644
--- a/crates/ai-memory-cli/src/commands/status.rs
+++ b/crates/ai-memory-cli/src/commands/status.rs
@@ -50,6 +50,22 @@ struct Report {
/// Passive process-scoped provider health.
#[serde(default)]
providers: ProviderHealthSnapshot,
+ /// Write-queue depth `(queued, capacity)` (2.0 servers).
+ #[serde(default)]
+ write_queue: Option<(usize, usize)>,
+ /// Wiki-format state (2.0 servers).
+ #[serde(default)]
+ wiki_format: Option<WikiFormatReport>,
+}
+
+#[derive(Debug, Default, Deserialize, Serialize)]
+struct WikiFormatReport {
+ #[serde(default)]
+ okf_migrated: bool,
+ #[serde(default)]
+ backup_archive: Option<String>,
+ #[serde(default)]
+ backup_archive_bytes: Option<u64>,
}
#[derive(Debug, Deserialize, Serialize)]
@@ -68,6 +84,10 @@ struct Derived {
observations_fts_rows: u64,
latest_pages_missing_embeddings: u64,
#[serde(default)]
+ latest_pages_unembeddable: u64,
+ #[serde(default)]
+ typed_links_from_latest_pages: Vec<(String, u64)>,
+ #[serde(default)]
embed_failures_unresolved: u64,
#[serde(default)]
embed_failures_recovered: u64,
@@ -230,6 +250,18 @@ pub async fn run(config: &Config, args: StatusArgs) -> Result<()> {
" embeddings: {} rows; {} latest pages missing",
report.derived.embedding_rows, report.derived.latest_pages_missing_embeddings
);
+ if report.derived.latest_pages_unembeddable > 0 {
+ println!(
+ " unembeddable: {} (empty body; no embedder can cover these)",
+ report.derived.latest_pages_unembeddable
+ );
+ }
+ for triple in &report.derived.embedding_triples {
+ println!(
+ " {}/{} dim {}: {} rows",
+ triple.provider, triple.model, triple.dim, triple.count
+ );
+ }
// Only shown when there is something to act on. A recovered count with
// no outstanding failures is history, not a problem.
if report.derived.embed_failures_unresolved > 0 {
@@ -265,6 +297,36 @@ pub async fn run(config: &Config, args: StatusArgs) -> Result<()> {
report.derived.unresolved_links_from_latest_pages,
report.derived.stale_links_from_latest_pages
);
+ if !report.derived.typed_links_from_latest_pages.is_empty() {
+ let typed: Vec<String> = report
+ .derived
+ .typed_links_from_latest_pages
+ .iter()
+ .map(|(k, v)| format!("{k}: {v}"))
+ .collect();
+ println!(" typed edges: {}", typed.join(", "));
+ }
+ if let Some(wiki) = &report.wiki_format {
+ let migrated = if wiki.okf_migrated {
+ "OKF v0.2 (migrated)"
+ } else {
+ "OKF v0.2 (native, no migration needed)"
+ };
+ match (&wiki.backup_archive, wiki.backup_archive_bytes) {
+ (Some(path), Some(bytes)) => println!(
+ " wiki format: {migrated}; pre-migration backup still on disk: \
+ {path} ({})",
+ super::compact::human_bytes(bytes)
+ ),
+ _ => println!(" wiki format: {migrated}"),
+ }
+ }
+ if let Some((queued, capacity)) = report.write_queue {
+ // A queue pinned near capacity is the wedged-writer signal.
+ if queued > 0 {
+ println!(" write queue: {queued}/{capacity}");
+ }
+ }
println!(" spool:");
println!(" pending: {}", spool.pending);
println!(" oldest: {}", spool_age_line(spool.oldest_age_ms));
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 4f55418a..61831bb9 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -1083,6 +1083,24 @@ pub struct StatusReport {
/// Hook-ingestion counters for this server process. Counts and one
/// timestamp only — never captured content (#428).
pub ingest: ai_memory_core::IngestMetricsSnapshot,
+ /// Instantaneous write-queue depth `(queued, capacity)` — the
+ /// wedged-writer signal (2.0 item 7).
+ pub write_queue: (usize, usize),
+ /// Wiki-format state: whether the OKF migration has run, and the
+ /// pre-migration backup archive if its receipt still exists.
+ pub wiki_format: WikiFormatStatus,
+}
+
+/// Wiki-format section of [`StatusReport`].
+#[derive(Debug, serde::Serialize, serde::Deserialize)]
+pub struct WikiFormatStatus {
+ /// True once the OKF v0.2 wiki migration has been applied.
+ pub okf_migrated: bool,
+ /// Pre-migration backup archive path, when the receipt exists AND
+ /// the archive file is still on disk.
+ pub backup_archive: Option<String>,
+ /// Size of that archive in bytes.
+ pub backup_archive_bytes: Option<u64>,
}
/// `GET /admin/projects` — the authoritative list of `(workspace, project)`
@@ -1110,6 +1128,14 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons
// Three pragma reads; a failure here must not take down the
// whole status response, which is also the health probe.
let storage = state.reader.storage_status().await.unwrap_or_default();
+ let okf_migrated = state
+ .reader
+ .wiki_migration_names()
+ .await
+ .map(|names| names.iter().any(|n| n.contains("okf")))
+ .unwrap_or(false);
+ let backup = ai_memory_wiki::backup::BackupReceipt::load(&state.data_dir)
+ .filter(ai_memory_wiki::backup::BackupReceipt::archive_present);
let report = StatusReport {
version: env!("CARGO_PKG_VERSION").to_string(),
data_dir: state.data_dir.display().to_string(),
@@ -1120,6 +1146,14 @@ async fn handle_status(State(state): State<Arc<AdminState>>) -> impl IntoRespons
storage,
providers: state.provider_health.snapshot(),
ingest: state.ingest_metrics.snapshot(),
+ write_queue: state.writer.queue_depth(),
+ wiki_format: WikiFormatStatus {
+ okf_migrated,
+ backup_archive: backup
+ .as_ref()
+ .map(|r| r.archive_path.display().to_string()),
+ backup_archive_bytes: backup.as_ref().map(|r| r.size_bytes),
+ },
};
(
StatusCode::OK,
@@ -7192,6 +7226,12 @@ mod tests {
let json: serde_json::Value = serde_json::from_slice(&body).unwrap();
assert_eq!(json["providers"]["llm"]["status"], "disabled");
assert_eq!(json["providers"]["embedding"]["status"], "disabled");
+ // 2.0 item 7: the report carries the wedged-writer gauge and the
+ // wiki-format state — a fresh store is natively conformant (no
+ // migration ran) with no backup archive.
+ assert!(json["write_queue"].is_array(), "{json}");
+ assert_eq!(json["wiki_format"]["okf_migrated"], false);
+ assert!(json["wiki_format"]["backup_archive"].is_null());
}
/// The MCP-only complement: per-client tool-call counters served
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 7c00b06a..d3ea8628 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -6537,6 +6537,77 @@ mod tests {
}
}
+ /// Item 7 truthfulness: "missing embeddings" counts only pages a
+ /// backfill can act on; empty-body pages are reported apart instead
+ /// of inflating the actionable number forever.
+ #[tokio::test]
+ async fn missing_embeddings_excludes_unembeddable_pages() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "s", None)
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(sample_page(ws, proj, "notes/real.md", "actual content"))
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(sample_page(ws, proj, "notes/empty.md", " "))
+ .await
+ .unwrap();
+ let derived = store.reader.derived_index_status().await.unwrap();
+ assert_eq!(derived.latest_pages_missing_embeddings, 1, "{derived:?}");
+ assert_eq!(derived.latest_pages_unembeddable, 1, "{derived:?}");
+ }
+
+ /// Typed edges surface in the derived status by relation.
+ #[tokio::test]
+ async fn typed_links_are_counted_by_relation() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "s", None)
+ .await
+ .unwrap();
+ let mut page = sample_page(ws, proj, "notes/a.md", "body");
+ page.links = vec![
+ ai_memory_core::LinkTarget {
+ workspace: None,
+ project: None,
+ path: ai_memory_core::PagePath::new("notes/b.md").unwrap(),
+ relation: Some(ai_memory_core::Relation::Fixes),
+ },
+ ai_memory_core::LinkTarget {
+ workspace: None,
+ project: None,
+ path: ai_memory_core::PagePath::new("notes/c.md").unwrap(),
+ relation: None,
+ },
+ ];
+ store.writer.upsert_page(page).await.unwrap();
+ let derived = store.reader.derived_index_status().await.unwrap();
+ assert_eq!(
+ derived.typed_links_from_latest_pages,
+ vec![("fixes".to_string(), 1)],
+ "{derived:?}"
+ );
+ }
+
#[tokio::test]
async fn entity_stream_finds_and_weights_pages() {
let tmp = TempDir::new().unwrap();
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index ab477a42..d391849a 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -946,8 +946,12 @@ pub struct DerivedIndexStatus {
pub observations_rows: u64,
/// Rows currently present in the observation FTS5 index.
pub observations_fts_rows: u64,
- /// Latest pages without any embedding row.
+ /// Latest pages without any embedding row whose body is non-empty —
+ /// i.e. the pages a backfill can actually act on.
pub latest_pages_missing_embeddings: u64,
+ /// Latest pages without an embedding whose body is empty; no
+ /// embedder can ever cover these (the backfill skips them by rule).
+ pub latest_pages_unembeddable: u64,
/// Latest pages whose last embed attempt failed or was skipped and which
/// still have no embedding. These are the pages an operator can act on.
pub embed_failures_unresolved: u64,
@@ -959,6 +963,9 @@ pub struct DerivedIndexStatus {
pub embedding_rows: u64,
/// Stored embedding triples and row counts.
pub embedding_triples: Vec<EmbeddingTripleCount>,
+ /// Typed relation edges (`link_type != 'references'`) from latest
+ /// pages, as `(relation, count)` — the 2.0 typed-edge surface.
+ pub typed_links_from_latest_pages: Vec<(String, u64)>,
/// Outgoing links whose source page is latest.
pub links_from_latest_pages: u64,
/// Latest-page outgoing links whose target path has not resolved yet.
@@ -7079,15 +7086,43 @@ impl ReaderPool {
JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \
JOIN page_embeddings pe ON pe.page_id = f.page_id",
)?,
+ // Aligned with the backfill's own skip rule: an
+ // empty-body page can never be embedded, so counting it
+ // as \"missing\" overstated the actionable number forever
+ // (observed live: a stable 427 that no backfill could
+ // ever clear). Unembeddable pages are reported apart.
latest_pages_missing_embeddings: count(
conn,
"SELECT COUNT(*) \
FROM pages pg \
LEFT JOIN page_embeddings pe ON pe.page_id = pg.id \
- WHERE pg.is_latest = 1 AND pe.page_id IS NULL",
+ WHERE pg.is_latest = 1 AND pe.page_id IS NULL \
+ AND TRIM(pg.body) != ''",
+ )?,
+ latest_pages_unembeddable: count(
+ conn,
+ "SELECT COUNT(*) \
+ FROM pages pg \
+ LEFT JOIN page_embeddings pe ON pe.page_id = pg.id \
+ WHERE pg.is_latest = 1 AND pe.page_id IS NULL \
+ AND TRIM(pg.body) = ''",
)?,
embedding_rows: count(conn, "SELECT COUNT(*) FROM page_embeddings")?,
embedding_triples,
+ typed_links_from_latest_pages: {
+ let mut stmt = conn.prepare(
+ "SELECT l.link_type, COUNT(*) FROM links l \
+ JOIN pages fp ON fp.id = l.from_page_id AND fp.is_latest = 1 \
+ WHERE l.link_type != 'references' \
+ GROUP BY l.link_type ORDER BY l.link_type",
+ )?;
+ let rows: Vec<(String, i64)> = stmt
+ .query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
+ .collect::<Result<_, _>>()?;
+ rows.into_iter()
+ .map(|(k, v)| (k, u64::try_from(v).unwrap_or(0)))
+ .collect()
+ },
links_from_latest_pages: count(
conn,
"SELECT COUNT(*) \
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index 1f1fffcf..b2a44330 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -1629,6 +1629,15 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Instantaneous write-queue depth as `(queued, capacity)`. A queue
+ /// pinned near capacity means writers are being backpressured — the
+ /// wedged-writer signal `status` surfaces (2.0 item 7).
+ #[must_use]
+ pub fn queue_depth(&self) -> (usize, usize) {
+ let max = self.inner.tx.max_capacity();
+ (max.saturating_sub(self.inner.tx.capacity()), max)
+ }
+
/// Record a completed cross-session ("experience") pass for a scope.
///
/// # Errors