Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(embed): persist a per-page embed outcome (#528) (#544)
Fabio Akita3e5ec5392676
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c43bd266..b5cfaa4b 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -71,6 +71,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0boundary in a table so it is not inferred from the command name.### Fixed+- A failed or skipped embed now leaves a durable record. `page_embeddings`+ stores only successes, and its upsert rewrites `created_at`, so a global+ `embed --force` erased any signal about which row had been stale. A failure+ left nothing at all: the inline write path warned and returned success, and+ the backfill's per-page warnings lived only in container logs that a restart+ took with them (#528).++ A `page_embed_failures` ledger (V53) records the last attempt that produced+ no embedding, across all four sites: the inline `write_page` embed, and the+ backfill's unreadable-page, empty-body and provider-error branches. The+ empty-body case matters most — a page skipped on every pass was previously+ indistinguishable from an idle one, which is what made #509 undiagnosable.++ Failure-only by design. A success writes nothing, because a `page_embeddings`+ row already records it, so the common path pays no extra write and+ `embed --force` cannot erase the history — it never touches this table.+ `status` joins the two to report unresolved failures separately from ones a+ later pass recovered, so a page that failed once and was repaired does not+ read as an outstanding problem forever. One row per page, cascading with it.++ Designed to @barrosohub's three constraints, who also supplied the+ measurement that made the gap concrete: of 1,491 embedding rows on their+ instance, zero predated the `--force` that had overwritten every timestamp.+- The hook drain no longer stalls the whole spool behind one undeliverableentry. A spooled event carries the URL it was captured against, so a spoolcan hold entries addressed to a port nothing listens on any more — an olddiff --git a/crates/ai-memory-cli/src/commands/status.rs b/crates/ai-memory-cli/src/commands/status.rsindex 60b336b1..7cb29220 100644--- a/crates/ai-memory-cli/src/commands/status.rs+++ b/crates/ai-memory-cli/src/commands/status.rs@@ -64,6 +64,10 @@ struct Derived {observations_rows: u64,observations_fts_rows: u64,latest_pages_missing_embeddings: u64,+ #[serde(default)]+ embed_failures_unresolved: u64,+ #[serde(default)]+ embed_failures_recovered: u64,embedding_rows: u64,embedding_triples: Vec<EmbeddingTriple>,links_from_latest_pages: u64,@@ -204,6 +208,14 @@ pub async fn run(config: &Config, args: StatusArgs) -> Result<()> {" embeddings: {} rows; {} latest pages missing",report.derived.embedding_rows, report.derived.latest_pages_missing_embeddings);+ // Only shown when there is something to act on. A recovered count with+ // no outstanding failures is history, not a problem.+ if report.derived.embed_failures_unresolved > 0 {+ println!(+ " embed failures: {} unresolved ({} recovered since)",+ report.derived.embed_failures_unresolved, report.derived.embed_failures_recovered+ );+ }println!(" links: {} latest-page links (unresolved: {}, stale: {})",report.derived.links_from_latest_pages,diff --git a/crates/ai-memory-consolidate/src/embed.rs b/crates/ai-memory-consolidate/src/embed.rsindex 71ad5d40..4e10b3b3 100644--- a/crates/ai-memory-consolidate/src/embed.rs+++ b/crates/ai-memory-consolidate/src/embed.rs@@ -123,11 +123,26 @@ pub async fn run_embedding_backfill(Ok(md) => md,Err(e) => {warn!(path = %cand.path, error = %e, "embed: skip unreadable page");+ // Durable, so the page is attributable after the container+ // that logged this is gone (#528).+ let _ = writer+ .record_embed_failure(+ cand.id,+ ai_memory_store::EmbedOutcome::Unreadable,+ Some(e.to_string()),+ )+ .await;counts.failed += 1;continue;}};if md.body.trim().is_empty() {+ // Permanent until the body changes. Recorded because a page+ // skipped on every pass is otherwise indistinguishable from an+ // idle one, which is what made #509 undiagnosable.+ let _ = writer+ .record_embed_failure(cand.id, ai_memory_store::EmbedOutcome::SkippedEmpty, None)+ .await;counts.skipped += 1;continue;}@@ -135,6 +150,13 @@ pub async fn run_embedding_backfill(Ok(vec) => vec,Err(e) => {warn!(path = %cand.path, error = %e, "embed: provider call failed");+ let _ = writer+ .record_embed_failure(+ cand.id,+ ai_memory_store::EmbedOutcome::Failed,+ Some(e.to_string()),+ )+ .await;counts.failed += 1;continue;}diff --git a/crates/ai-memory-store/migrations/V53__page_embed_failures.sql b/crates/ai-memory-store/migrations/V53__page_embed_failures.sqlnew file mode 100644index 00000000..16801fc0--- /dev/null+++ b/crates/ai-memory-store/migrations/V53__page_embed_failures.sql@@ -0,0 +1,28 @@+-- Durable record of an embed attempt that did NOT produce an embedding (#528).+--+-- `page_embeddings` records only successes, and its upsert rewrites+-- `created_at`, so a global `embed --force` erases any signal about which row+-- was previously stale. A failed or skipped embed left nothing at all: the+-- inline write path warns and returns success, and the backfill's per-page+-- warnings live only in container logs that a restart takes with them. The+-- result was that `status` could report a page missing an embedding with no+-- way to ask when it was last attempted or why it did not take.+--+-- Deliberately failure-only. A success writes nothing here — it is already+-- recorded by the existence of a `page_embeddings` row — so the common path+-- pays no extra write. A row therefore means "the last attempt on this page+-- did not produce an embedding", and joining against `page_embeddings` tells+-- an operator whether that is still true or whether a later pass recovered it.+-- Keeping the row after recovery is intentional: it is the history that+-- `--force` used to destroy.+--+-- One row per page, cascading with it, so the ledger cannot outgrow the corpus.+CREATE TABLE page_embed_failures (+ page_id BLOB PRIMARY KEY NOT NULL REFERENCES pages(id) ON DELETE CASCADE,+ at INTEGER NOT NULL,+ outcome TEXT NOT NULL CHECK (outcome IN ('failed', 'unreadable', 'skipped_empty')),+ -- Truncated provider/IO error. NULL for `skipped_empty`, which has no error.+ detail TEXT+);++CREATE INDEX idx_page_embed_failures_at ON page_embed_failures(at DESC);diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rsindex 61cfb91a..43a61c3e 100644--- a/crates/ai-memory-store/src/lib.rs+++ b/crates/ai-memory-store/src/lib.rs@@ -45,10 +45,10 @@ pub use decay::{pub use error::{StoreError, StoreResult};pub use maintenance::MaintenanceJob;pub use ops::{- AdmittedSession, Compaction, DeleteWorkspaceSummary, EmbeddingWrite, HookSessionAdmission,- IngestObservationOutcome, LifecycleOnlyEndOutcome, MoveSessionSummary, MoveSummary,- ObservationPruneOutcome, PagesMode, PurgeSessionSummary, PurgeSummary, ReorgSummary,- purge_session,+ AdmittedSession, Compaction, DeleteWorkspaceSummary, EmbedOutcome, EmbeddingWrite,+ HookSessionAdmission, IngestObservationOutcome, LifecycleOnlyEndOutcome, MoveSessionSummary,+ MoveSummary, ObservationPruneOutcome, PagesMode, PurgeSessionSummary, PurgeSummary,+ ReorgSummary, purge_session, record_embed_failure,};pub use reader::{ActivityWindow, AgentSessionCount, AuditEvent, AuditLogFilter, AutoImproveCandidateSession,diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rsindex d2a34de6..bffb0ae1 100644--- a/crates/ai-memory-store/src/ops.rs+++ b/crates/ai-memory-store/src/ops.rs@@ -1457,6 +1457,74 @@ fn insert_observation_row(conn: &Connection, obs: &NewObservation) -> StoreResul/// `f32` packing of the unit-normalised vector. Provider/model/dim/// are denormalised onto the row so a single SELECT can detect/// heterogeneity (refuse-on-mismatch path).+/// Why an embed attempt produced no embedding.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum EmbedOutcome {+ /// The embedding provider returned an error.+ Failed,+ /// The page could not be read from the wiki.+ Unreadable,+ /// The page body was empty after trimming, so there was nothing to embed.+ /// Not an error, but permanent until the body changes — and previously+ /// indistinguishable from an idle pass.+ SkippedEmpty,+}++impl EmbedOutcome {+ fn as_str(self) -> &'static str {+ match self {+ Self::Failed => "failed",+ Self::Unreadable => "unreadable",+ Self::SkippedEmpty => "skipped_empty",+ }+ }+}++/// Longest `detail` we keep. A provider error can carry a whole response body;+/// the ledger exists to say what happened, not to archive it.+const EMBED_FAILURE_DETAIL_MAX: usize = 500;++/// Record that an embed attempt on `page_id` did not produce an embedding.+///+/// Failure-only by design: a success writes nothing, because the existence of+/// a `page_embeddings` row already records it. That keeps the common path free+/// of an extra write and means a global `embed --force` cannot erase the+/// history here — it never touches this table (#528).+///+/// Upserts on `page_id`, so the row is always the most recent unsuccessful+/// attempt. It is deliberately left in place after a later pass succeeds: join+/// against `page_embeddings` to tell "still unresolved" from "recovered".+///+/// # Errors+/// Propagates SQL errors.+pub fn record_embed_failure(+ conn: &Connection,+ page_id: &PageId,+ outcome: EmbedOutcome,+ detail: Option<&str>,+) -> StoreResult<()> {+ let detail = detail.map(|d| {+ let mut truncated: String = d.chars().take(EMBED_FAILURE_DETAIL_MAX).collect();+ if truncated.chars().count() < d.chars().count() {+ truncated.push('…');+ }+ truncated+ });+ conn.execute(+ "INSERT INTO page_embed_failures (page_id, at, outcome, detail) \+ VALUES (?1, ?2, ?3, ?4) \+ ON CONFLICT(page_id) DO UPDATE SET \+ at = excluded.at, outcome = excluded.outcome, detail = excluded.detail",+ params![+ page_id.as_bytes(),+ Timestamp::now().as_microsecond(),+ outcome.as_str(),+ detail,+ ],+ )?;+ Ok(())+}+pub fn store_embedding(conn: &mut Connection,page_id: &PageId,@@ -4682,6 +4750,154 @@ pub(crate) mod tests {assert_eq!(state, "accepted", "an accepted handoff is not backlog");}+ fn embed_failure_rows(conn: &Connection) -> i64 {+ conn.query_row("SELECT COUNT(*) FROM page_embed_failures", [], |r| r.get(0))+ .unwrap()+ }++ /// #528: a failed embed left nothing durable — the inline path warned and+ /// returned success, and the backfill's warnings died with the container.+ #[test]+ fn a_failed_embed_is_recorded_with_its_reason_and_detail() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();++ record_embed_failure(+ &conn,+ &page_id,+ EmbedOutcome::Failed,+ Some("400 empty Part"),+ )+ .unwrap();++ let (outcome, detail): (String, Option<String>) = conn+ .query_row(+ "SELECT outcome, detail FROM page_embed_failures WHERE page_id = ?1",+ [page_id.as_bytes()],+ |r| Ok((r.get(0)?, r.get(1)?)),+ )+ .unwrap();+ assert_eq!(outcome, "failed");+ assert_eq!(detail.as_deref(), Some("400 empty Part"));+ }++ /// A success must write nothing here — the `page_embeddings` row already+ /// records it, and the reporter's first constraint was no extra write on+ /// the hot path.+ #[test]+ fn a_successful_embed_writes_no_failure_row() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();++ store_embedding(&mut conn, &page_id, &[0_u8; 8], "p", "m", 2).unwrap();++ assert_eq!(embed_failure_rows(&conn), 0);+ }++ /// The second constraint: a global re-embed must not erase the history.+ #[test]+ fn a_later_successful_embed_preserves_the_failure_history() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();+ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, Some("boom")).unwrap();++ store_embedding(&mut conn, &page_id, &[0_u8; 8], "p", "m", 2).unwrap();++ assert_eq!(+ embed_failure_rows(&conn),+ 1,+ "the record of what went wrong survives the repair"+ );+ }++ /// Re-attempting overwrites rather than accumulating: one row per page.+ #[test]+ fn repeated_failures_keep_only_the_latest_attempt() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();++ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, Some("first")).unwrap();+ record_embed_failure(&conn, &page_id, EmbedOutcome::SkippedEmpty, None).unwrap();++ assert_eq!(embed_failure_rows(&conn), 1);+ let outcome: String = conn+ .query_row("SELECT outcome FROM page_embed_failures", [], |r| r.get(0))+ .unwrap();+ assert_eq!(outcome, "skipped_empty", "latest attempt wins");+ }++ /// A provider error can carry a whole response body; the ledger says what+ /// happened, it does not archive it.+ #[test]+ fn failure_detail_is_truncated() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();+ let huge = "x".repeat(5_000);++ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, Some(&huge)).unwrap();++ let detail: String = conn+ .query_row("SELECT detail FROM page_embed_failures", [], |r| r.get(0))+ .unwrap();+ assert!(detail.chars().count() <= EMBED_FAILURE_DETAIL_MAX + 1);+ assert!(detail.ends_with('…'), "truncation is visible");+ }++ /// The ledger cannot outgrow the corpus: rows go with their page.+ #[test]+ fn a_failure_row_is_removed_with_its_page() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();+ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, None).unwrap();++ delete_page(+ &mut conn,+ ws,+ proj,+ &PagePath::new("notes/a.md").unwrap(),+ None,+ )+ .unwrap();++ assert_eq!(embed_failure_rows(&conn), 0);+ }++ /// A failure row is the *last unsuccessful attempt*, not proof the page is+ /// still broken. `status` must separate the two, or a page that failed+ /// once and recovered reads as an outstanding problem forever.+ #[test]+ fn status_separates_unresolved_embed_failures_from_recovered_ones() {+ let (_tmp, mut conn, ws, proj) = fresh_db();+ let broken = upsert_page(&mut conn, &page(ws, proj, "notes/broken.md", "b")).unwrap();+ let recovered = upsert_page(&mut conn, &page(ws, proj, "notes/ok.md", "b")).unwrap();++ record_embed_failure(&conn, &broken, EmbedOutcome::Failed, Some("boom")).unwrap();+ record_embed_failure(&conn, &recovered, EmbedOutcome::SkippedEmpty, None).unwrap();+ store_embedding(&mut conn, &recovered, &[0_u8; 8], "p", "m", 2).unwrap();++ let unresolved: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM page_embed_failures f \+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \+ LEFT JOIN page_embeddings pe ON pe.page_id = f.page_id \+ WHERE pe.page_id IS NULL",+ [],+ |r| r.get(0),+ )+ .unwrap();+ let recovered_count: i64 = conn+ .query_row(+ "SELECT COUNT(*) FROM page_embed_failures f \+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \+ JOIN page_embeddings pe ON pe.page_id = f.page_id",+ [],+ |r| r.get(0),+ )+ .unwrap();+ assert_eq!(unresolved, 1, "only the page still lacking an embedding");+ assert_eq!(recovered_count, 1, "the repaired page is history");+ }+/// Accepting a handoff does not make its text yours. Purging the/// accepting session must not destroy the authoring session's record.#[test]diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rsindex 2a6c45b1..216a2ca7 100644--- a/crates/ai-memory-store/src/reader.rs+++ b/crates/ai-memory-store/src/reader.rs@@ -948,6 +948,13 @@ pub struct DerivedIndexStatus {pub observations_fts_rows: u64,/// Latest pages without any embedding row.pub latest_pages_missing_embeddings: u64,+ /// Latest pages whose last embed attempt failed or was skipped and which+ /// still have no embedding. These are the pages an operator can act on.+ pub embed_failures_unresolved: u64,+ /// Latest pages that recorded a failed or skipped embed at some point but+ /// have an embedding now. Kept because a global `embed --force` used to+ /// erase exactly this history, leaving a recurrence unattributable (#528).+ pub embed_failures_recovered: u64,/// Stored embedding rows, regardless of provider/model/dim.pub embedding_rows: u64,/// Stored embedding triples and row counts.@@ -6873,6 +6880,25 @@ impl ReaderPool {conn,"SELECT COUNT(*) FROM observations_fts_docsize",)?,+ // Split by whether an embedding exists *now*: a failure row is+ // the last unsuccessful attempt, not proof the page is still+ // broken. Without this join a page that failed once and+ // recovered would read as an outstanding problem forever.+ embed_failures_unresolved: count(+ conn,+ "SELECT COUNT(*) \+ FROM page_embed_failures f \+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \+ LEFT JOIN page_embeddings pe ON pe.page_id = f.page_id \+ WHERE pe.page_id IS NULL",+ )?,+ embed_failures_recovered: count(+ conn,+ "SELECT COUNT(*) \+ FROM page_embed_failures f \+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \+ JOIN page_embeddings pe ON pe.page_id = f.page_id",+ )?,latest_pages_missing_embeddings: count(conn,"SELECT COUNT(*) \diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rsindex 64f276b4..7ea49a05 100644--- a/crates/ai-memory-store/src/writer.rs+++ b/crates/ai-memory-store/src/writer.rs@@ -205,6 +205,13 @@ pub(crate) enum WriteCmd {author_id: Option<ai_memory_core::UserId>,reply: oneshot::Sender<StoreResult<u64>>,},+ /// Record that an embed attempt produced no embedding (#528).+ RecordEmbedFailure {+ page_id: PageId,+ outcome: crate::ops::EmbedOutcome,+ detail: Option<String>,+ reply: oneshot::Sender<StoreResult<()>>,+ },CancelHandoff {handoff_id: HandoffId,workspace_id: WorkspaceId,@@ -973,6 +980,32 @@ impl WriterHandle {rx.await.map_err(|_| StoreError::WriterClosed)?}+ /// Record that an embed attempt on `page_id` did not produce an+ /// embedding, so a failed or skipped page is attributable afterwards+ /// rather than only in container logs (#528).+ ///+ /// Success records nothing — that is already implied by a+ /// `page_embeddings` row — so the common path pays no extra write.+ ///+ /// # Errors+ /// [`StoreError::WriterClosed`], or a propagated SQL error.+ pub async fn record_embed_failure(+ &self,+ page_id: PageId,+ outcome: crate::ops::EmbedOutcome,+ detail: Option<String>,+ ) -> StoreResult<()> {+ let (tx, rx) = oneshot::channel();+ self.send(WriteCmd::RecordEmbedFailure {+ page_id,+ outcome,+ detail,+ reply: tx,+ })+ .await?;+ rx.await.map_err(|_| StoreError::WriterClosed)?+ }+/// Mark an open handoff expired so it will no longer be consumed.////// Returns `true` when an open handoff was changed, `false` when the id was@@ -2142,6 +2175,15 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {);send_or_warn(reply, result, "expire_open_handoffs");}+ WriteCmd::RecordEmbedFailure {+ page_id,+ outcome,+ detail,+ reply,+ } => {+ let result = ops::record_embed_failure(&conn, &page_id, outcome, detail.as_deref());+ send_or_warn(reply, result, "record_embed_failure");+ }WriteCmd::CancelHandoff {handoff_id,workspace_id,diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rsindex 3a03aa80..65834040 100644--- a/crates/ai-memory-wiki/src/wiki.rs+++ b/crates/ai-memory-wiki/src/wiki.rs@@ -1806,6 +1806,18 @@ impl Wiki {}Err(e) => {tracing::warn!(error = %e, path = %page_id, "embedding failed; page indexed without it");+ // The warning alone dies with the container. Record it so+ // the page is attributable later (#528); best-effort,+ // because failing to note a failure must not fail the+ // write that already succeeded.+ let _ = self+ .writer+ .record_embed_failure(+ page_id,+ ai_memory_store::EmbedOutcome::Failed,+ Some(e.to_string()),+ )+ .await;}}}