Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(auto-improve): attach the eval result to the eval evidence entry

AkitaOnRails40b5f721f371
diff --git a/crates/ai-memory-consolidate/src/auto_improve.rs b/crates/ai-memory-consolidate/src/auto_improve.rs
index c570e093..e87e01fa 100644
--- a/crates/ai-memory-consolidate/src/auto_improve.rs
+++ b/crates/ai-memory-consolidate/src/auto_improve.rs
@@ -96,6 +96,9 @@ pub const DEFAULT_AUTO_IMPROVE_MAX_PROPOSALS: usize = 5;
pub const DEFAULT_AUTO_IMPROVE_PROPOSAL_ACTOR: &str = "auto_improve";
/// Default wiki-relative folder for pending proposal sidecar markdown.
pub const DEFAULT_AUTO_IMPROVE_PENDING_PATH: &str = "_pending/auto-improve";
+/// Evidence label of the entry the eval gate adds to a passing proposal; its
+/// structured result attaches to this entry.
+const AUTO_IMPROVE_EVAL_EVIDENCE_PAGE: &str = "auto_improve_eval";
/// Default target prefixes guarded by the optional external eval command.
pub fn default_auto_improve_eval_targets() -> Vec<String> {
@@ -497,10 +500,11 @@ impl AutoImproveReport {
result.status == EvalStatus::Success
&& result.target_path == proposal.path
&& result.after_body_sha256 == sha256_hex(proposal.body_markdown.as_bytes())
- }) && let Some(entry) = evidence
- .as_array_mut()
- .and_then(|entries| entries.last_mut())
- {
+ }) && let Some(entry) = evidence.as_array_mut().and_then(|entries| {
+ entries
+ .iter_mut()
+ .rfind(|entry| entry["page"] == AUTO_IMPROVE_EVAL_EVIDENCE_PAGE)
+ }) {
entry["eval_result"] = serde_json::to_value(result)?;
}
Ok(evidence)
@@ -819,7 +823,7 @@ async fn apply_eval_gate_with_before_bodies(
result.status = EvalStatus::Success;
result.reason = outcome.reason.as_deref().map(sanitize_eval_reason);
proposal.evidence.push(AutoImproveEvidence {
- page: "auto_improve_eval".into(),
+ page: AUTO_IMPROVE_EVAL_EVIDENCE_PAGE.into(),
quote: format_eval_evidence(&outcome, delta),
});
accepted.push(proposal);
diff --git a/crates/ai-memory-consolidate/tests/suite/eval_results.rs b/crates/ai-memory-consolidate/tests/suite/eval_results.rs
index 6d1a60e4..48c4e6ef 100644
--- a/crates/ai-memory-consolidate/tests/suite/eval_results.rs
+++ b/crates/ai-memory-consolidate/tests/suite/eval_results.rs
@@ -13,7 +13,8 @@ use serde_json::{Value, json};
use tempfile::TempDir;
use crate::{
- AutoImproveEvalConfig, AutoImproveReport, AutoImproveReviewConfig, run_auto_improve_review,
+ AutoImproveEvalConfig, AutoImproveEvidence, AutoImproveReport, AutoImproveReviewConfig,
+ run_auto_improve_review,
};
struct ProposingLlm;
@@ -177,6 +178,17 @@ async fn eval_reason_is_sanitized_before_pending_storage_with_legitimate_control
.get("eval_result")
.is_none()
);
+ // The result belongs to the checker's own evidence entry, wherever it
+ // sits, not to whichever entry happens to be last.
+ let mut reordered = candidate.clone();
+ reordered.evidence.push(AutoImproveEvidence {
+ page: "notes/later.md".into(),
+ quote: "later evidence".into(),
+ });
+ let evidence = report.proposal_evidence_json(&reordered).unwrap();
+ assert_eq!(evidence[1]["page"], "auto_improve_eval");
+ assert_eq!(evidence[1]["eval_result"], results[0]);
+ assert!(evidence[2].get("eval_result").is_none());
let staged = store
.writer