Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
feat(auto-improve): stage telemetry reports
AkitaOnRailsc3ccd239f422
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 5c23938e..fb42b74f 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -30,7 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0- Added read-only auto-improvement telemetry reporting via`POST /admin/auto-improve/report` and `ai-memory auto-improve-report`, withJSON and human CLI output for recent run counts, proposal outcomes, terminal- rates, and operational findings without staging pending proposals.+ rates, and operational findings without staging pending proposals; optional+ `--stage` / `stage: true` creates one pending audit report page for approval.- Added `docs/auto-improve-eval-gates.md` plus dependency-free Python and shellscorer templates for `[auto_improve.eval]` proposal gates.- Hook spool drains now use `POST /hook/batch` when the server supports it,diff --git a/README.md b/README.mdindex 1e2c6e44..a749ef43 100644--- a/README.md+++ b/README.md@@ -143,7 +143,8 @@ priors are at the [bottom](#influences-and-prior-art).available for manual catch-up or targeted reruns. `ai-memoryauto-improve-report --workspace <w> --project <p>` returns a read-onlytelemetry report for recent auto-improvement outcomes without staging or- creating proposals. See+ creating proposals; add `--stage` to create one pending report page for+ audit/approval. See[`docs/auto-improve-eval-gates.md`](docs/auto-improve-eval-gates.md) forexample executable eval scorers.diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rsindex d1ff397b..5c99e65b 100644--- a/crates/ai-memory-cli/src/cli.rs+++ b/crates/ai-memory-cli/src/cli.rs@@ -915,6 +915,9 @@ pub struct AutoImproveReportArgs {/// Maximum rows in each top-N count table.#[arg(long, default_value_t = ai_memory_consolidate::DEFAULT_AUTO_IMPROVE_TELEMETRY_TOP_LIMIT)]pub limit: usize,+ /// Stage one pending telemetry report page for later approval.+ #[arg(long)]+ pub stage: bool,/// Emit only machine-readable JSON.#[arg(long)]pub json: bool,@@ -1376,6 +1379,7 @@ mod tests {"14","--limit","3",+ "--stage","--json",]).expect("auto-improve-report parses");@@ -1386,6 +1390,7 @@ mod tests {assert_eq!(args.project.as_deref(), Some("scratch"));assert_eq!(args.days, 14);assert_eq!(args.limit, 3);+ assert!(args.stage);assert!(args.json);}diff --git a/crates/ai-memory-cli/src/commands/auto_improve_report.rs b/crates/ai-memory-cli/src/commands/auto_improve_report.rsindex 2be4768b..55bd15be 100644--- a/crates/ai-memory-cli/src/commands/auto_improve_report.rs+++ b/crates/ai-memory-cli/src/commands/auto_improve_report.rs@@ -2,7 +2,7 @@use ai_memory_consolidate::AutoImproveTelemetryReport;use anyhow::Result;-use serde::Serialize;+use serde::{Deserialize, Serialize};use crate::cli::AutoImproveReportArgs;use crate::config::Config;@@ -14,6 +14,15 @@ struct AutoImproveReportRequest {project: String,since_days: u32,limit: usize,+ stage: bool,+}++#[derive(Deserialize, Serialize)]+struct AutoImproveReportStageResponse {+ run_id: String,+ proposal_ids: Vec<String>,+ sidecar_paths: Vec<String>,+ report: AutoImproveTelemetryReport,}/// Run the `auto-improve-report` subcommand.@@ -28,8 +37,26 @@ pub async fn run(config: &Config, args: AutoImproveReportArgs) -> Result<()> {project: project.clone(),since_days: args.days,limit: args.limit,+ stage: args.stage,};+ if args.stage {+ let response: AutoImproveReportStageResponse =+ post_json(&endpoint, "/admin/auto-improve/report", &request).await?;+ if args.json {+ println!("{}", serde_json::to_string_pretty(&response)?);+ } else {+ println!("\nStaged auto-improve telemetry report for {project}\n");+ println!("Run: {}", response.run_id);+ println!("Proposals: {}", response.proposal_ids.join(", "));+ println!("Sidecars: {}", response.sidecar_paths.join(", "));+ println!("Summary: {}", response.report.summary);+ println!("\n--- machine-readable ---");+ println!("{}", serde_json::to_string_pretty(&response)?);+ }+ return Ok(());+ }+let report: AutoImproveTelemetryReport =post_json(&endpoint, "/admin/auto-improve/report", &request).await?;if args.json {diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rsindex 2cad39a5..92bafb74 100644--- a/crates/ai-memory-mcp/src/admin.rs+++ b/crates/ai-memory-mcp/src/admin.rs@@ -34,10 +34,11 @@ use std::io::Seek;use std::path::PathBuf;use ai_memory_consolidate::{- AutoImproveReviewConfig, AutoImproveTelemetryParams, Bootstrap, BootstrapConfig,- BootstrapOutcome, BootstrapSource, CuratorParams, CuratorReport, SourceCounts,- prune_sources_to_budget, render_curator_report_markdown, run_auto_improve_review,- run_auto_improve_telemetry_report, run_curator_report, run_lint, run_sweep,+ AutoImproveReviewConfig, AutoImproveTelemetryParams, AutoImproveTelemetryReport, Bootstrap,+ BootstrapConfig, BootstrapOutcome, BootstrapSource, CuratorParams, CuratorReport, SourceCounts,+ prune_sources_to_budget, render_auto_improve_telemetry_report_markdown,+ render_curator_report_markdown, run_auto_improve_review, run_auto_improve_telemetry_report,+ run_curator_report, run_lint, run_sweep,};use ai_memory_core::{ActiveProject, AutoImproveProposalId, Capability, DEFAULT_PROJECT_NAME, DEFAULT_WORKSPACE_NAME,@@ -287,6 +288,17 @@ struct AutoImproveTelemetryReportRequest {/// Maximum rows in each top-N count table.#[serde(default = "default_auto_improve_telemetry_top_limit")]limit: usize,+ /// Stage one pending telemetry report page instead of returning a read-only report.+ #[serde(default)]+ stage: bool,+}++#[derive(Debug, Serialize)]+struct AutoImproveTelemetryReportStageResponse {+ run_id: String,+ proposal_ids: Vec<String>,+ sidecar_paths: Vec<String>,+ report: AutoImproveTelemetryReport,}/// JSON request body for `POST /admin/curator`.@@ -1488,8 +1500,41 @@ async fn stage_auto_improve_report(}).await.map_err(|e| internal_err(e.to_string()))?;- let mut sidecar_paths = Vec::with_capacity(staged.proposal_ids.len());- for id in &staged.proposal_ids {+ let sidecar_paths = write_auto_improve_sidecars(state, ws, proj, &staged.proposal_ids).await?;+ Ok(StagedAutoImproveData {+ run_id: staged.run_id,+ proposal_ids: staged.proposal_ids,+ sidecar_paths,+ })+}++async fn target_operation_for_page(+ state: &AdminState,+ ws: WorkspaceId,+ proj: ProjectId,+ target: &PagePath,+) -> Result<AutoImproveProposalOperation, (StatusCode, Json<serde_json::Value>)> {+ if state+ .reader+ .page_body_by_ids(ws, proj, target.as_str())+ .await+ .map_err(|e| internal_err(e.to_string()))?+ .is_some()+ {+ Ok(AutoImproveProposalOperation::Update)+ } else {+ Ok(AutoImproveProposalOperation::Create)+ }+}++async fn write_auto_improve_sidecars(+ state: &AdminState,+ ws: WorkspaceId,+ proj: ProjectId,+ proposal_ids: &[AutoImproveProposalId],+) -> Result<Vec<String>, (StatusCode, Json<serde_json::Value>)> {+ let mut sidecar_paths = Vec::with_capacity(proposal_ids.len());+ for id in proposal_ids {let path = state.wiki.write_auto_improve_sidecar(ws, proj, *id)@@ -1497,11 +1542,7 @@ async fn stage_auto_improve_report(.map_err(|e| internal_err(e.to_string()))?;sidecar_paths.push(path.display().to_string());}- Ok(StagedAutoImproveData {- run_id: staged.run_id,- proposal_ids: staged.proposal_ids,- sidecar_paths,- })+ Ok(sidecar_paths)}fn auto_improve_error_response(@@ -1538,10 +1579,80 @@ async fn handle_auto_improve_report(proj,&req.workspace,&req.project,- params,+ params.clone(),).await.map_err(|e| internal_err(e.to_string()))?;+ if req.stage {+ let body_markdown = render_auto_improve_telemetry_report_markdown(&report);+ let target_path = auto_improve_report_target_path();+ let target = PagePath::new(target_path).map_err(|e| {+ (+ StatusCode::UNPROCESSABLE_ENTITY,+ Json(serde_json::json!({ "error": format!("invalid auto-improve report target path: {e}") })),+ )+ })?;+ let operation = target_operation_for_page(&state, ws, proj, &target).await?;+ let staged = state+ .writer+ .stage_auto_improve_run(StageAutoImproveRun {+ workspace_id: ws,+ project_id: proj,+ session_id: None,+ provider: None,+ model: None,+ summary: Some(format!("Auto-improve telemetry report: {}", report.summary)),+ warnings_json: serde_json::json!([]),+ rejected_candidates_json: serde_json::json!([]),+ config_json: serde_json::json!({+ "mode": "stage",+ "auto_improve_report": true,+ "params": params,+ }),+ proposal_actor: ai_memory_core::ActorContext {+ agent: Some("auto_improve_report".into()),+ ..ai_memory_core::ActorContext::default()+ },+ proposals: vec![NewAutoImproveProposal {+ operation,+ target_path: target,+ kind: "auto_improve_report".into(),+ title: "Auto-improve Telemetry Report".into(),+ confidence: 1.0,+ rationale: "Telemetry report only; approval writes the report page and performs no learning-memory changes.".into(),+ evidence_json: serde_json::json!({+ "summary": report.summary.clone(),+ "findings": report.findings.clone(),+ "params": report.params.clone(),+ }),+ body_markdown,+ artifact_sha256: None,+ edit_mode: None,+ patch_json: None,+ expected_base_body_sha256: None,+ }],+ })+ .await+ .map_err(|e| internal_err(e.to_string()))?;+ let sidecar_paths =+ write_auto_improve_sidecars(&state, ws, proj, &staged.proposal_ids).await?;+ return Ok((+ StatusCode::OK,+ Json(+ serde_json::to_value(AutoImproveTelemetryReportStageResponse {+ run_id: staged.run_id.to_string(),+ proposal_ids: staged+ .proposal_ids+ .iter()+ .map(ToString::to_string)+ .collect(),+ sidecar_paths,+ report,+ })+ .unwrap_or_else(|_| serde_json::json!({})),+ ),+ ));+ }Ok((StatusCode::OK,Json(serde_json::to_value(&report).unwrap_or_else(|_| serde_json::json!({}))),@@ -1600,17 +1711,7 @@ async fn handle_curator(Json(serde_json::json!({ "error": format!("invalid curator target path: {e}") })),)})?;- let operation = if state- .reader- .page_body_by_ids(ws, proj, target.as_str())- .await- .map_err(|e| internal_err(e.to_string()))?- .is_some()- {- AutoImproveProposalOperation::Update- } else {- AutoImproveProposalOperation::Create- };+ let operation = target_operation_for_page(&state, ws, proj, &target).await?;let staged = state.writer@@ -1652,15 +1753,7 @@ async fn handle_curator(}).await.map_err(|e| internal_err(e.to_string()))?;- let mut sidecar_paths = Vec::with_capacity(staged.proposal_ids.len());- for id in &staged.proposal_ids {- let path = state- .wiki- .write_auto_improve_sidecar(ws, proj, *id)- .await- .map_err(|e| internal_err(e.to_string()))?;- sidecar_paths.push(path.display().to_string());- }+ let sidecar_paths = write_auto_improve_sidecars(&state, ws, proj, &staged.proposal_ids).await?;Ok((StatusCode::OK,Json(@@ -1685,6 +1778,11 @@ fn curator_target_path() -> String {format!("notes/curator-{date}.md")}+fn auto_improve_report_target_path() -> String {+ let stamp = jiff::Timestamp::now().as_microsecond();+ format!("notes/auto-improve-report-{stamp}.md")+}+async fn handle_pending_writes_list(State(state): State<Arc<AdminState>>,Query(query): Query<PendingWritesQuery>,diff --git a/crates/ai-memory-mcp/tests/admin_phase3.rs b/crates/ai-memory-mcp/tests/admin_phase3.rsindex a3cdd19c..a631ded1 100644--- a/crates/ai-memory-mcp/tests/admin_phase3.rs+++ b/crates/ai-memory-mcp/tests/admin_phase3.rs@@ -33,7 +33,9 @@ use tower::ServiceExt;/// Build a minimal `AdminState` with no LLM and no embedder.async fn make_state(tmp: &TempDir) -> (AdminState, Store) {let store = Store::open(tmp.path()).unwrap();- let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();+ let wiki = Wiki::new(tmp.path(), store.writer.clone())+ .unwrap()+ .with_store_reader(store.reader.clone());let db_path = store.db_path().to_path_buf();let state = AdminState {writer: store.writer.clone(),@@ -149,8 +151,9 @@ async fn auto_improve_report_returns_telemetry_without_creating_proposals() {json!({ "workspace": "default", "project": "scratch", "since_days": 30, "limit": 3 }),).await;- assert_eq!(resp.status(), StatusCode::OK);+ let status = resp.status();let body = body_json(resp).await;+ assert_eq!(status, StatusCode::OK, "{body:?}");assert!(body["summary"].as_str().unwrap().contains("run(s)"));assert_eq!(body["aggregate"]["run_count"].as_u64().unwrap(), 1);assert_eq!(@@ -201,6 +204,117 @@ async fn auto_improve_report_missing_scope_fails_closed() {assert_eq!(resp.status(), StatusCode::NOT_FOUND);}+#[tokio::test]+async fn auto_improve_report_stage_creates_one_report_proposal_and_approval_writes_page() {+ let tmp = TempDir::new().unwrap();+ let (state, store) = make_state(&tmp).await;+ let ws = store+ .writer+ .get_or_create_workspace("default")+ .await+ .unwrap();+ let proj = store+ .writer+ .get_or_create_project(ws, "scratch", None)+ .await+ .unwrap();++ let before = store+ .reader+ .list_auto_improve_proposals(ws, proj, None, 50)+ .await+ .unwrap()+ .len();+ let resp = post(+ state.clone(),+ "/admin/auto-improve/report",+ json!({ "workspace": "default", "project": "scratch", "since_days": 30, "limit": 3, "stage": true }),+ )+ .await;+ let status = resp.status();+ let body = body_json(resp).await;+ assert_eq!(status, StatusCode::OK, "{body:?}");+ let proposal_ids = body["proposal_ids"].as_array().unwrap();+ assert_eq!(proposal_ids.len(), 1);+ assert_eq!(body["sidecar_paths"].as_array().unwrap().len(), 1);++ let proposals = store+ .reader+ .list_auto_improve_proposals(ws, proj, None, 50)+ .await+ .unwrap();+ assert_eq!(proposals.len(), before + 1);+ let staged = proposals+ .iter()+ .find(|p| p.id.to_string() == proposal_ids[0].as_str().unwrap())+ .unwrap();+ assert_eq!(staged.status, AutoImproveProposalStatus::Pending);+ assert_eq!(staged.kind, "auto_improve_report");+ let target_path = staged.target_path.as_str().to_string();+ assert!(target_path.starts_with("notes/auto-improve-report-"));+ assert!(target_path.ends_with(".md"));+ assert!(+ store+ .reader+ .page_body_by_ids(ws, proj, &target_path)+ .await+ .unwrap()+ .is_none(),+ "staging must not write the target page before approval"+ );++ let approve_resp = post(+ state.clone(),+ &format!(+ "/admin/pending-writes/{}/approve?workspace=default&project=scratch",+ staged.id+ ),+ json!({}),+ )+ .await;+ assert_eq!(approve_resp.status(), StatusCode::OK);+ let report_page = store+ .reader+ .page_body_by_ids(ws, proj, &target_path)+ .await+ .unwrap()+ .expect("approval writes report page");+ assert!(report_page.body.contains("# Auto-Improve Telemetry Report"));+ assert_eq!(+ store+ .reader+ .list_auto_improve_proposals(ws, proj, None, 50)+ .await+ .unwrap()+ .len(),+ before + 1,+ "approval must not create extra proposals"+ );++ let telemetry_resp = post(+ state,+ "/admin/auto-improve/report",+ json!({ "workspace": "default", "project": "scratch", "since_days": 30, "limit": 10 }),+ )+ .await;+ assert_eq!(telemetry_resp.status(), StatusCode::OK);+ let telemetry = body_json(telemetry_resp).await;+ let maintenance = telemetry["aggregate"]["maintenance_proposals_by_kind"]+ .as_array()+ .unwrap();+ assert!(maintenance.iter().any(|row| {+ row["key"].as_str() == Some("auto_improve_report") && row["count"].as_u64() == Some(1)+ }));+ assert_eq!(+ telemetry["aggregate"]["proposals_by_kind"]+ .as_array()+ .unwrap()+ .len(),+ 0,+ "report proposals are maintenance, not learning proposals"+ );+}+// ---------------------------------------------------------------------------// reorg// ---------------------------------------------------------------------------diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.mdindex e0593dff..08e8d59d 100644--- a/docs/ARCHITECTURE.md+++ b/docs/ARCHITECTURE.md@@ -275,6 +275,9 @@ uninstall auth userRun `ai-memory --help` for the full tree.+`auto-improve-report` is read-only by default; `--stage` creates one pending+telemetry report page for audit/approval without staging learning-memory edits.+## Cross-cutting invariantsCarved in M0/M1; every milestone has to respect them. Each comes fromdiff --git a/docs/auto-improvement-loop.md b/docs/auto-improvement-loop.mdindex b1e2ef48..11415b88 100644--- a/docs/auto-improvement-loop.md+++ b/docs/auto-improvement-loop.md@@ -373,7 +373,7 @@ them by default through the wiki mutation path. With `require_approval = true`,|---|---|| Background scheduler | Reviews newly completed sessions after the first-run watermark and applies or stages validated proposals according to approval policy. || `ai-memory auto-improve --session-id <id>` | Manually review one session and apply or stage validated proposals through the auto-improvement approval path. |-| `ai-memory auto-improve-report --workspace <w> --project <p> [--days N] [--limit N]` | Read-only telemetry report for recent auto-improvement runs, proposal outcomes, terminal rates, and findings. Does not stage or create pending proposals. |+| `ai-memory auto-improve-report --workspace <w> --project <p> [--days N] [--limit N] [--stage]` | Read-only telemetry report for recent auto-improvement runs, proposal outcomes, terminal rates, and findings by default. `--stage` creates exactly one pending telemetry report page for audit/approval. || `memory_auto_improve` | Manually review the latest completed session or a named session and apply or stage validated proposals through the same path. || `ai-memory curator` | Rule-based, report-only maintenance review. || `ai-memory curator --stage` | Stage exactly one curator report page for pending-writes approval. |