Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(embed): persist a per-page embed outcome (#528) (#544)

Fabio Akita3e5ec5392676
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c43bd266..b5cfaa4b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -71,6 +71,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
boundary in a table so it is not inferred from the command name.
### Fixed
+- A failed or skipped embed now leaves a durable record. `page_embeddings`
+ stores only successes, and its upsert rewrites `created_at`, so a global
+ `embed --force` erased any signal about which row had been stale. A failure
+ left nothing at all: the inline write path warned and returned success, and
+ the backfill's per-page warnings lived only in container logs that a restart
+ took with them (#528).
+
+ A `page_embed_failures` ledger (V53) records the last attempt that produced
+ no embedding, across all four sites: the inline `write_page` embed, and the
+ backfill's unreadable-page, empty-body and provider-error branches. The
+ empty-body case matters most — a page skipped on every pass was previously
+ indistinguishable from an idle one, which is what made #509 undiagnosable.
+
+ Failure-only by design. A success writes nothing, because a `page_embeddings`
+ row already records it, so the common path pays no extra write and
+ `embed --force` cannot erase the history — it never touches this table.
+ `status` joins the two to report unresolved failures separately from ones a
+ later pass recovered, so a page that failed once and was repaired does not
+ read as an outstanding problem forever. One row per page, cascading with it.
+
+ Designed to @barrosohub's three constraints, who also supplied the
+ measurement that made the gap concrete: of 1,491 embedding rows on their
+ instance, zero predated the `--force` that had overwritten every timestamp.
+
- The hook drain no longer stalls the whole spool behind one undeliverable
entry. A spooled event carries the URL it was captured against, so a spool
can hold entries addressed to a port nothing listens on any more — an old
diff --git a/crates/ai-memory-cli/src/commands/status.rs b/crates/ai-memory-cli/src/commands/status.rs
index 60b336b1..7cb29220 100644
--- a/crates/ai-memory-cli/src/commands/status.rs
+++ b/crates/ai-memory-cli/src/commands/status.rs
@@ -64,6 +64,10 @@ struct Derived {
observations_rows: u64,
observations_fts_rows: u64,
latest_pages_missing_embeddings: u64,
+ #[serde(default)]
+ embed_failures_unresolved: u64,
+ #[serde(default)]
+ embed_failures_recovered: u64,
embedding_rows: u64,
embedding_triples: Vec<EmbeddingTriple>,
links_from_latest_pages: u64,
@@ -204,6 +208,14 @@ pub async fn run(config: &Config, args: StatusArgs) -> Result<()> {
" embeddings: {} rows; {} latest pages missing",
report.derived.embedding_rows, report.derived.latest_pages_missing_embeddings
);
+ // Only shown when there is something to act on. A recovered count with
+ // no outstanding failures is history, not a problem.
+ if report.derived.embed_failures_unresolved > 0 {
+ println!(
+ " embed failures: {} unresolved ({} recovered since)",
+ report.derived.embed_failures_unresolved, report.derived.embed_failures_recovered
+ );
+ }
println!(
" links: {} latest-page links (unresolved: {}, stale: {})",
report.derived.links_from_latest_pages,
diff --git a/crates/ai-memory-consolidate/src/embed.rs b/crates/ai-memory-consolidate/src/embed.rs
index 71ad5d40..4e10b3b3 100644
--- a/crates/ai-memory-consolidate/src/embed.rs
+++ b/crates/ai-memory-consolidate/src/embed.rs
@@ -123,11 +123,26 @@ pub async fn run_embedding_backfill(
Ok(md) => md,
Err(e) => {
warn!(path = %cand.path, error = %e, "embed: skip unreadable page");
+ // Durable, so the page is attributable after the container
+ // that logged this is gone (#528).
+ let _ = writer
+ .record_embed_failure(
+ cand.id,
+ ai_memory_store::EmbedOutcome::Unreadable,
+ Some(e.to_string()),
+ )
+ .await;
counts.failed += 1;
continue;
}
};
if md.body.trim().is_empty() {
+ // Permanent until the body changes. Recorded because a page
+ // skipped on every pass is otherwise indistinguishable from an
+ // idle one, which is what made #509 undiagnosable.
+ let _ = writer
+ .record_embed_failure(cand.id, ai_memory_store::EmbedOutcome::SkippedEmpty, None)
+ .await;
counts.skipped += 1;
continue;
}
@@ -135,6 +150,13 @@ pub async fn run_embedding_backfill(
Ok(vec) => vec,
Err(e) => {
warn!(path = %cand.path, error = %e, "embed: provider call failed");
+ let _ = writer
+ .record_embed_failure(
+ cand.id,
+ ai_memory_store::EmbedOutcome::Failed,
+ Some(e.to_string()),
+ )
+ .await;
counts.failed += 1;
continue;
}
diff --git a/crates/ai-memory-store/migrations/V53__page_embed_failures.sql b/crates/ai-memory-store/migrations/V53__page_embed_failures.sql
new file mode 100644
index 00000000..16801fc0
--- /dev/null
+++ b/crates/ai-memory-store/migrations/V53__page_embed_failures.sql
@@ -0,0 +1,28 @@
+-- Durable record of an embed attempt that did NOT produce an embedding (#528).
+--
+-- `page_embeddings` records only successes, and its upsert rewrites
+-- `created_at`, so a global `embed --force` erases any signal about which row
+-- was previously stale. A failed or skipped embed left nothing at all: the
+-- inline write path warns and returns success, and the backfill's per-page
+-- warnings live only in container logs that a restart takes with them. The
+-- result was that `status` could report a page missing an embedding with no
+-- way to ask when it was last attempted or why it did not take.
+--
+-- Deliberately failure-only. A success writes nothing here — it is already
+-- recorded by the existence of a `page_embeddings` row — so the common path
+-- pays no extra write. A row therefore means "the last attempt on this page
+-- did not produce an embedding", and joining against `page_embeddings` tells
+-- an operator whether that is still true or whether a later pass recovered it.
+-- Keeping the row after recovery is intentional: it is the history that
+-- `--force` used to destroy.
+--
+-- One row per page, cascading with it, so the ledger cannot outgrow the corpus.
+CREATE TABLE page_embed_failures (
+ page_id BLOB PRIMARY KEY NOT NULL REFERENCES pages(id) ON DELETE CASCADE,
+ at INTEGER NOT NULL,
+ outcome TEXT NOT NULL CHECK (outcome IN ('failed', 'unreadable', 'skipped_empty')),
+ -- Truncated provider/IO error. NULL for `skipped_empty`, which has no error.
+ detail TEXT
+);
+
+CREATE INDEX idx_page_embed_failures_at ON page_embed_failures(at DESC);
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 61cfb91a..43a61c3e 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -45,10 +45,10 @@ pub use decay::{
pub use error::{StoreError, StoreResult};
pub use maintenance::MaintenanceJob;
pub use ops::{
- AdmittedSession, Compaction, DeleteWorkspaceSummary, EmbeddingWrite, HookSessionAdmission,
- IngestObservationOutcome, LifecycleOnlyEndOutcome, MoveSessionSummary, MoveSummary,
- ObservationPruneOutcome, PagesMode, PurgeSessionSummary, PurgeSummary, ReorgSummary,
- purge_session,
+ AdmittedSession, Compaction, DeleteWorkspaceSummary, EmbedOutcome, EmbeddingWrite,
+ HookSessionAdmission, IngestObservationOutcome, LifecycleOnlyEndOutcome, MoveSessionSummary,
+ MoveSummary, ObservationPruneOutcome, PagesMode, PurgeSessionSummary, PurgeSummary,
+ ReorgSummary, purge_session, record_embed_failure,
};
pub use reader::{
ActivityWindow, AgentSessionCount, AuditEvent, AuditLogFilter, AutoImproveCandidateSession,
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index d2a34de6..bffb0ae1 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -1457,6 +1457,74 @@ fn insert_observation_row(conn: &Connection, obs: &NewObservation) -> StoreResul
/// `f32` packing of the unit-normalised vector. Provider/model/dim
/// are denormalised onto the row so a single SELECT can detect
/// heterogeneity (refuse-on-mismatch path).
+/// Why an embed attempt produced no embedding.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum EmbedOutcome {
+ /// The embedding provider returned an error.
+ Failed,
+ /// The page could not be read from the wiki.
+ Unreadable,
+ /// The page body was empty after trimming, so there was nothing to embed.
+ /// Not an error, but permanent until the body changes — and previously
+ /// indistinguishable from an idle pass.
+ SkippedEmpty,
+}
+
+impl EmbedOutcome {
+ fn as_str(self) -> &'static str {
+ match self {
+ Self::Failed => "failed",
+ Self::Unreadable => "unreadable",
+ Self::SkippedEmpty => "skipped_empty",
+ }
+ }
+}
+
+/// Longest `detail` we keep. A provider error can carry a whole response body;
+/// the ledger exists to say what happened, not to archive it.
+const EMBED_FAILURE_DETAIL_MAX: usize = 500;
+
+/// Record that an embed attempt on `page_id` did not produce an embedding.
+///
+/// Failure-only by design: a success writes nothing, because the existence of
+/// a `page_embeddings` row already records it. That keeps the common path free
+/// of an extra write and means a global `embed --force` cannot erase the
+/// history here — it never touches this table (#528).
+///
+/// Upserts on `page_id`, so the row is always the most recent unsuccessful
+/// attempt. It is deliberately left in place after a later pass succeeds: join
+/// against `page_embeddings` to tell "still unresolved" from "recovered".
+///
+/// # Errors
+/// Propagates SQL errors.
+pub fn record_embed_failure(
+ conn: &Connection,
+ page_id: &PageId,
+ outcome: EmbedOutcome,
+ detail: Option<&str>,
+) -> StoreResult<()> {
+ let detail = detail.map(|d| {
+ let mut truncated: String = d.chars().take(EMBED_FAILURE_DETAIL_MAX).collect();
+ if truncated.chars().count() < d.chars().count() {
+ truncated.push('…');
+ }
+ truncated
+ });
+ conn.execute(
+ "INSERT INTO page_embed_failures (page_id, at, outcome, detail) \
+ VALUES (?1, ?2, ?3, ?4) \
+ ON CONFLICT(page_id) DO UPDATE SET \
+ at = excluded.at, outcome = excluded.outcome, detail = excluded.detail",
+ params![
+ page_id.as_bytes(),
+ Timestamp::now().as_microsecond(),
+ outcome.as_str(),
+ detail,
+ ],
+ )?;
+ Ok(())
+}
+
pub fn store_embedding(
conn: &mut Connection,
page_id: &PageId,
@@ -4682,6 +4750,154 @@ pub(crate) mod tests {
assert_eq!(state, "accepted", "an accepted handoff is not backlog");
}
+ fn embed_failure_rows(conn: &Connection) -> i64 {
+ conn.query_row("SELECT COUNT(*) FROM page_embed_failures", [], |r| r.get(0))
+ .unwrap()
+ }
+
+ /// #528: a failed embed left nothing durable — the inline path warned and
+ /// returned success, and the backfill's warnings died with the container.
+ #[test]
+ fn a_failed_embed_is_recorded_with_its_reason_and_detail() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();
+
+ record_embed_failure(
+ &conn,
+ &page_id,
+ EmbedOutcome::Failed,
+ Some("400 empty Part"),
+ )
+ .unwrap();
+
+ let (outcome, detail): (String, Option<String>) = conn
+ .query_row(
+ "SELECT outcome, detail FROM page_embed_failures WHERE page_id = ?1",
+ [page_id.as_bytes()],
+ |r| Ok((r.get(0)?, r.get(1)?)),
+ )
+ .unwrap();
+ assert_eq!(outcome, "failed");
+ assert_eq!(detail.as_deref(), Some("400 empty Part"));
+ }
+
+ /// A success must write nothing here — the `page_embeddings` row already
+ /// records it, and the reporter's first constraint was no extra write on
+ /// the hot path.
+ #[test]
+ fn a_successful_embed_writes_no_failure_row() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();
+
+ store_embedding(&mut conn, &page_id, &[0_u8; 8], "p", "m", 2).unwrap();
+
+ assert_eq!(embed_failure_rows(&conn), 0);
+ }
+
+ /// The second constraint: a global re-embed must not erase the history.
+ #[test]
+ fn a_later_successful_embed_preserves_the_failure_history() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();
+ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, Some("boom")).unwrap();
+
+ store_embedding(&mut conn, &page_id, &[0_u8; 8], "p", "m", 2).unwrap();
+
+ assert_eq!(
+ embed_failure_rows(&conn),
+ 1,
+ "the record of what went wrong survives the repair"
+ );
+ }
+
+ /// Re-attempting overwrites rather than accumulating: one row per page.
+ #[test]
+ fn repeated_failures_keep_only_the_latest_attempt() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();
+
+ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, Some("first")).unwrap();
+ record_embed_failure(&conn, &page_id, EmbedOutcome::SkippedEmpty, None).unwrap();
+
+ assert_eq!(embed_failure_rows(&conn), 1);
+ let outcome: String = conn
+ .query_row("SELECT outcome FROM page_embed_failures", [], |r| r.get(0))
+ .unwrap();
+ assert_eq!(outcome, "skipped_empty", "latest attempt wins");
+ }
+
+ /// A provider error can carry a whole response body; the ledger says what
+ /// happened, it does not archive it.
+ #[test]
+ fn failure_detail_is_truncated() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();
+ let huge = "x".repeat(5_000);
+
+ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, Some(&huge)).unwrap();
+
+ let detail: String = conn
+ .query_row("SELECT detail FROM page_embed_failures", [], |r| r.get(0))
+ .unwrap();
+ assert!(detail.chars().count() <= EMBED_FAILURE_DETAIL_MAX + 1);
+ assert!(detail.ends_with('…'), "truncation is visible");
+ }
+
+ /// The ledger cannot outgrow the corpus: rows go with their page.
+ #[test]
+ fn a_failure_row_is_removed_with_its_page() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let page_id = upsert_page(&mut conn, &page(ws, proj, "notes/a.md", "body")).unwrap();
+ record_embed_failure(&conn, &page_id, EmbedOutcome::Failed, None).unwrap();
+
+ delete_page(
+ &mut conn,
+ ws,
+ proj,
+ &PagePath::new("notes/a.md").unwrap(),
+ None,
+ )
+ .unwrap();
+
+ assert_eq!(embed_failure_rows(&conn), 0);
+ }
+
+ /// A failure row is the *last unsuccessful attempt*, not proof the page is
+ /// still broken. `status` must separate the two, or a page that failed
+ /// once and recovered reads as an outstanding problem forever.
+ #[test]
+ fn status_separates_unresolved_embed_failures_from_recovered_ones() {
+ let (_tmp, mut conn, ws, proj) = fresh_db();
+ let broken = upsert_page(&mut conn, &page(ws, proj, "notes/broken.md", "b")).unwrap();
+ let recovered = upsert_page(&mut conn, &page(ws, proj, "notes/ok.md", "b")).unwrap();
+
+ record_embed_failure(&conn, &broken, EmbedOutcome::Failed, Some("boom")).unwrap();
+ record_embed_failure(&conn, &recovered, EmbedOutcome::SkippedEmpty, None).unwrap();
+ store_embedding(&mut conn, &recovered, &[0_u8; 8], "p", "m", 2).unwrap();
+
+ let unresolved: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM page_embed_failures f \
+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \
+ LEFT JOIN page_embeddings pe ON pe.page_id = f.page_id \
+ WHERE pe.page_id IS NULL",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ let recovered_count: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM page_embed_failures f \
+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \
+ JOIN page_embeddings pe ON pe.page_id = f.page_id",
+ [],
+ |r| r.get(0),
+ )
+ .unwrap();
+ assert_eq!(unresolved, 1, "only the page still lacking an embedding");
+ assert_eq!(recovered_count, 1, "the repaired page is history");
+ }
+
/// Accepting a handoff does not make its text yours. Purging the
/// accepting session must not destroy the authoring session's record.
#[test]
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index 2a6c45b1..216a2ca7 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -948,6 +948,13 @@ pub struct DerivedIndexStatus {
pub observations_fts_rows: u64,
/// Latest pages without any embedding row.
pub latest_pages_missing_embeddings: u64,
+ /// Latest pages whose last embed attempt failed or was skipped and which
+ /// still have no embedding. These are the pages an operator can act on.
+ pub embed_failures_unresolved: u64,
+ /// Latest pages that recorded a failed or skipped embed at some point but
+ /// have an embedding now. Kept because a global `embed --force` used to
+ /// erase exactly this history, leaving a recurrence unattributable (#528).
+ pub embed_failures_recovered: u64,
/// Stored embedding rows, regardless of provider/model/dim.
pub embedding_rows: u64,
/// Stored embedding triples and row counts.
@@ -6873,6 +6880,25 @@ impl ReaderPool {
conn,
"SELECT COUNT(*) FROM observations_fts_docsize",
)?,
+ // Split by whether an embedding exists *now*: a failure row is
+ // the last unsuccessful attempt, not proof the page is still
+ // broken. Without this join a page that failed once and
+ // recovered would read as an outstanding problem forever.
+ embed_failures_unresolved: count(
+ conn,
+ "SELECT COUNT(*) \
+ FROM page_embed_failures f \
+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \
+ LEFT JOIN page_embeddings pe ON pe.page_id = f.page_id \
+ WHERE pe.page_id IS NULL",
+ )?,
+ embed_failures_recovered: count(
+ conn,
+ "SELECT COUNT(*) \
+ FROM page_embed_failures f \
+ JOIN pages pg ON pg.id = f.page_id AND pg.is_latest = 1 \
+ JOIN page_embeddings pe ON pe.page_id = f.page_id",
+ )?,
latest_pages_missing_embeddings: count(
conn,
"SELECT COUNT(*) \
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index 64f276b4..7ea49a05 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -205,6 +205,13 @@ pub(crate) enum WriteCmd {
author_id: Option<ai_memory_core::UserId>,
reply: oneshot::Sender<StoreResult<u64>>,
},
+ /// Record that an embed attempt produced no embedding (#528).
+ RecordEmbedFailure {
+ page_id: PageId,
+ outcome: crate::ops::EmbedOutcome,
+ detail: Option<String>,
+ reply: oneshot::Sender<StoreResult<()>>,
+ },
CancelHandoff {
handoff_id: HandoffId,
workspace_id: WorkspaceId,
@@ -973,6 +980,32 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Record that an embed attempt on `page_id` did not produce an
+ /// embedding, so a failed or skipped page is attributable afterwards
+ /// rather than only in container logs (#528).
+ ///
+ /// Success records nothing — that is already implied by a
+ /// `page_embeddings` row — so the common path pays no extra write.
+ ///
+ /// # Errors
+ /// [`StoreError::WriterClosed`], or a propagated SQL error.
+ pub async fn record_embed_failure(
+ &self,
+ page_id: PageId,
+ outcome: crate::ops::EmbedOutcome,
+ detail: Option<String>,
+ ) -> StoreResult<()> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::RecordEmbedFailure {
+ page_id,
+ outcome,
+ detail,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
/// Mark an open handoff expired so it will no longer be consumed.
///
/// Returns `true` when an open handoff was changed, `false` when the id was
@@ -2142,6 +2175,15 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
);
send_or_warn(reply, result, "expire_open_handoffs");
}
+ WriteCmd::RecordEmbedFailure {
+ page_id,
+ outcome,
+ detail,
+ reply,
+ } => {
+ let result = ops::record_embed_failure(&conn, &page_id, outcome, detail.as_deref());
+ send_or_warn(reply, result, "record_embed_failure");
+ }
WriteCmd::CancelHandoff {
handoff_id,
workspace_id,
diff --git a/crates/ai-memory-wiki/src/wiki.rs b/crates/ai-memory-wiki/src/wiki.rs
index 3a03aa80..65834040 100644
--- a/crates/ai-memory-wiki/src/wiki.rs
+++ b/crates/ai-memory-wiki/src/wiki.rs
@@ -1806,6 +1806,18 @@ impl Wiki {
}
Err(e) => {
tracing::warn!(error = %e, path = %page_id, "embedding failed; page indexed without it");
+ // The warning alone dies with the container. Record it so
+ // the page is attributable later (#528); best-effort,
+ // because failing to note a failure must not fail the
+ // write that already succeeded.
+ let _ = self
+ .writer
+ .record_embed_failure(
+ page_id,
+ ai_memory_store::EmbedOutcome::Failed,
+ Some(e.to_string()),
+ )
+ .await;
}
}
}