Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(auto-improve): stage telemetry reports

AkitaOnRailsc3ccd239f422
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5c23938e..fb42b74f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,7 +30,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Added read-only auto-improvement telemetry reporting via
`POST /admin/auto-improve/report` and `ai-memory auto-improve-report`, with
JSON and human CLI output for recent run counts, proposal outcomes, terminal
- rates, and operational findings without staging pending proposals.
+ rates, and operational findings without staging pending proposals; optional
+ `--stage` / `stage: true` creates one pending audit report page for approval.
- Added `docs/auto-improve-eval-gates.md` plus dependency-free Python and shell
scorer templates for `[auto_improve.eval]` proposal gates.
- Hook spool drains now use `POST /hook/batch` when the server supports it,
diff --git a/README.md b/README.md
index 1e2c6e44..a749ef43 100644
--- a/README.md
+++ b/README.md
@@ -143,7 +143,8 @@ priors are at the [bottom](#influences-and-prior-art).
available for manual catch-up or targeted reruns. `ai-memory
auto-improve-report --workspace <w> --project <p>` returns a read-only
telemetry report for recent auto-improvement outcomes without staging or
- creating proposals. See
+ creating proposals; add `--stage` to create one pending report page for
+ audit/approval. See
[`docs/auto-improve-eval-gates.md`](docs/auto-improve-eval-gates.md) for
example executable eval scorers.
diff --git a/crates/ai-memory-cli/src/cli.rs b/crates/ai-memory-cli/src/cli.rs
index d1ff397b..5c99e65b 100644
--- a/crates/ai-memory-cli/src/cli.rs
+++ b/crates/ai-memory-cli/src/cli.rs
@@ -915,6 +915,9 @@ pub struct AutoImproveReportArgs {
/// Maximum rows in each top-N count table.
#[arg(long, default_value_t = ai_memory_consolidate::DEFAULT_AUTO_IMPROVE_TELEMETRY_TOP_LIMIT)]
pub limit: usize,
+ /// Stage one pending telemetry report page for later approval.
+ #[arg(long)]
+ pub stage: bool,
/// Emit only machine-readable JSON.
#[arg(long)]
pub json: bool,
@@ -1376,6 +1379,7 @@ mod tests {
"14",
"--limit",
"3",
+ "--stage",
"--json",
])
.expect("auto-improve-report parses");
@@ -1386,6 +1390,7 @@ mod tests {
assert_eq!(args.project.as_deref(), Some("scratch"));
assert_eq!(args.days, 14);
assert_eq!(args.limit, 3);
+ assert!(args.stage);
assert!(args.json);
}
diff --git a/crates/ai-memory-cli/src/commands/auto_improve_report.rs b/crates/ai-memory-cli/src/commands/auto_improve_report.rs
index 2be4768b..55bd15be 100644
--- a/crates/ai-memory-cli/src/commands/auto_improve_report.rs
+++ b/crates/ai-memory-cli/src/commands/auto_improve_report.rs
@@ -2,7 +2,7 @@
use ai_memory_consolidate::AutoImproveTelemetryReport;
use anyhow::Result;
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
use crate::cli::AutoImproveReportArgs;
use crate::config::Config;
@@ -14,6 +14,15 @@ struct AutoImproveReportRequest {
project: String,
since_days: u32,
limit: usize,
+ stage: bool,
+}
+
+#[derive(Deserialize, Serialize)]
+struct AutoImproveReportStageResponse {
+ run_id: String,
+ proposal_ids: Vec<String>,
+ sidecar_paths: Vec<String>,
+ report: AutoImproveTelemetryReport,
}
/// Run the `auto-improve-report` subcommand.
@@ -28,8 +37,26 @@ pub async fn run(config: &Config, args: AutoImproveReportArgs) -> Result<()> {
project: project.clone(),
since_days: args.days,
limit: args.limit,
+ stage: args.stage,
};
+ if args.stage {
+ let response: AutoImproveReportStageResponse =
+ post_json(&endpoint, "/admin/auto-improve/report", &request).await?;
+ if args.json {
+ println!("{}", serde_json::to_string_pretty(&response)?);
+ } else {
+ println!("\nStaged auto-improve telemetry report for {project}\n");
+ println!("Run: {}", response.run_id);
+ println!("Proposals: {}", response.proposal_ids.join(", "));
+ println!("Sidecars: {}", response.sidecar_paths.join(", "));
+ println!("Summary: {}", response.report.summary);
+ println!("\n--- machine-readable ---");
+ println!("{}", serde_json::to_string_pretty(&response)?);
+ }
+ return Ok(());
+ }
+
let report: AutoImproveTelemetryReport =
post_json(&endpoint, "/admin/auto-improve/report", &request).await?;
if args.json {
diff --git a/crates/ai-memory-mcp/src/admin.rs b/crates/ai-memory-mcp/src/admin.rs
index 2cad39a5..92bafb74 100644
--- a/crates/ai-memory-mcp/src/admin.rs
+++ b/crates/ai-memory-mcp/src/admin.rs
@@ -34,10 +34,11 @@ use std::io::Seek;
use std::path::PathBuf;
use ai_memory_consolidate::{
- AutoImproveReviewConfig, AutoImproveTelemetryParams, Bootstrap, BootstrapConfig,
- BootstrapOutcome, BootstrapSource, CuratorParams, CuratorReport, SourceCounts,
- prune_sources_to_budget, render_curator_report_markdown, run_auto_improve_review,
- run_auto_improve_telemetry_report, run_curator_report, run_lint, run_sweep,
+ AutoImproveReviewConfig, AutoImproveTelemetryParams, AutoImproveTelemetryReport, Bootstrap,
+ BootstrapConfig, BootstrapOutcome, BootstrapSource, CuratorParams, CuratorReport, SourceCounts,
+ prune_sources_to_budget, render_auto_improve_telemetry_report_markdown,
+ render_curator_report_markdown, run_auto_improve_review, run_auto_improve_telemetry_report,
+ run_curator_report, run_lint, run_sweep,
};
use ai_memory_core::{
ActiveProject, AutoImproveProposalId, Capability, DEFAULT_PROJECT_NAME, DEFAULT_WORKSPACE_NAME,
@@ -287,6 +288,17 @@ struct AutoImproveTelemetryReportRequest {
/// Maximum rows in each top-N count table.
#[serde(default = "default_auto_improve_telemetry_top_limit")]
limit: usize,
+ /// Stage one pending telemetry report page instead of returning a read-only report.
+ #[serde(default)]
+ stage: bool,
+}
+
+#[derive(Debug, Serialize)]
+struct AutoImproveTelemetryReportStageResponse {
+ run_id: String,
+ proposal_ids: Vec<String>,
+ sidecar_paths: Vec<String>,
+ report: AutoImproveTelemetryReport,
}
/// JSON request body for `POST /admin/curator`.
@@ -1488,8 +1500,41 @@ async fn stage_auto_improve_report(
})
.await
.map_err(|e| internal_err(e.to_string()))?;
- let mut sidecar_paths = Vec::with_capacity(staged.proposal_ids.len());
- for id in &staged.proposal_ids {
+ let sidecar_paths = write_auto_improve_sidecars(state, ws, proj, &staged.proposal_ids).await?;
+ Ok(StagedAutoImproveData {
+ run_id: staged.run_id,
+ proposal_ids: staged.proposal_ids,
+ sidecar_paths,
+ })
+}
+
+async fn target_operation_for_page(
+ state: &AdminState,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ target: &PagePath,
+) -> Result<AutoImproveProposalOperation, (StatusCode, Json<serde_json::Value>)> {
+ if state
+ .reader
+ .page_body_by_ids(ws, proj, target.as_str())
+ .await
+ .map_err(|e| internal_err(e.to_string()))?
+ .is_some()
+ {
+ Ok(AutoImproveProposalOperation::Update)
+ } else {
+ Ok(AutoImproveProposalOperation::Create)
+ }
+}
+
+async fn write_auto_improve_sidecars(
+ state: &AdminState,
+ ws: WorkspaceId,
+ proj: ProjectId,
+ proposal_ids: &[AutoImproveProposalId],
+) -> Result<Vec<String>, (StatusCode, Json<serde_json::Value>)> {
+ let mut sidecar_paths = Vec::with_capacity(proposal_ids.len());
+ for id in proposal_ids {
let path = state
.wiki
.write_auto_improve_sidecar(ws, proj, *id)
@@ -1497,11 +1542,7 @@ async fn stage_auto_improve_report(
.map_err(|e| internal_err(e.to_string()))?;
sidecar_paths.push(path.display().to_string());
}
- Ok(StagedAutoImproveData {
- run_id: staged.run_id,
- proposal_ids: staged.proposal_ids,
- sidecar_paths,
- })
+ Ok(sidecar_paths)
}
fn auto_improve_error_response(
@@ -1538,10 +1579,80 @@ async fn handle_auto_improve_report(
proj,
&req.workspace,
&req.project,
- params,
+ params.clone(),
)
.await
.map_err(|e| internal_err(e.to_string()))?;
+ if req.stage {
+ let body_markdown = render_auto_improve_telemetry_report_markdown(&report);
+ let target_path = auto_improve_report_target_path();
+ let target = PagePath::new(target_path).map_err(|e| {
+ (
+ StatusCode::UNPROCESSABLE_ENTITY,
+ Json(serde_json::json!({ "error": format!("invalid auto-improve report target path: {e}") })),
+ )
+ })?;
+ let operation = target_operation_for_page(&state, ws, proj, &target).await?;
+ let staged = state
+ .writer
+ .stage_auto_improve_run(StageAutoImproveRun {
+ workspace_id: ws,
+ project_id: proj,
+ session_id: None,
+ provider: None,
+ model: None,
+ summary: Some(format!("Auto-improve telemetry report: {}", report.summary)),
+ warnings_json: serde_json::json!([]),
+ rejected_candidates_json: serde_json::json!([]),
+ config_json: serde_json::json!({
+ "mode": "stage",
+ "auto_improve_report": true,
+ "params": params,
+ }),
+ proposal_actor: ai_memory_core::ActorContext {
+ agent: Some("auto_improve_report".into()),
+ ..ai_memory_core::ActorContext::default()
+ },
+ proposals: vec![NewAutoImproveProposal {
+ operation,
+ target_path: target,
+ kind: "auto_improve_report".into(),
+ title: "Auto-improve Telemetry Report".into(),
+ confidence: 1.0,
+ rationale: "Telemetry report only; approval writes the report page and performs no learning-memory changes.".into(),
+ evidence_json: serde_json::json!({
+ "summary": report.summary.clone(),
+ "findings": report.findings.clone(),
+ "params": report.params.clone(),
+ }),
+ body_markdown,
+ artifact_sha256: None,
+ edit_mode: None,
+ patch_json: None,
+ expected_base_body_sha256: None,
+ }],
+ })
+ .await
+ .map_err(|e| internal_err(e.to_string()))?;
+ let sidecar_paths =
+ write_auto_improve_sidecars(&state, ws, proj, &staged.proposal_ids).await?;
+ return Ok((
+ StatusCode::OK,
+ Json(
+ serde_json::to_value(AutoImproveTelemetryReportStageResponse {
+ run_id: staged.run_id.to_string(),
+ proposal_ids: staged
+ .proposal_ids
+ .iter()
+ .map(ToString::to_string)
+ .collect(),
+ sidecar_paths,
+ report,
+ })
+ .unwrap_or_else(|_| serde_json::json!({})),
+ ),
+ ));
+ }
Ok((
StatusCode::OK,
Json(serde_json::to_value(&report).unwrap_or_else(|_| serde_json::json!({}))),
@@ -1600,17 +1711,7 @@ async fn handle_curator(
Json(serde_json::json!({ "error": format!("invalid curator target path: {e}") })),
)
})?;
- let operation = if state
- .reader
- .page_body_by_ids(ws, proj, target.as_str())
- .await
- .map_err(|e| internal_err(e.to_string()))?
- .is_some()
- {
- AutoImproveProposalOperation::Update
- } else {
- AutoImproveProposalOperation::Create
- };
+ let operation = target_operation_for_page(&state, ws, proj, &target).await?;
let staged = state
.writer
@@ -1652,15 +1753,7 @@ async fn handle_curator(
})
.await
.map_err(|e| internal_err(e.to_string()))?;
- let mut sidecar_paths = Vec::with_capacity(staged.proposal_ids.len());
- for id in &staged.proposal_ids {
- let path = state
- .wiki
- .write_auto_improve_sidecar(ws, proj, *id)
- .await
- .map_err(|e| internal_err(e.to_string()))?;
- sidecar_paths.push(path.display().to_string());
- }
+ let sidecar_paths = write_auto_improve_sidecars(&state, ws, proj, &staged.proposal_ids).await?;
Ok((
StatusCode::OK,
Json(
@@ -1685,6 +1778,11 @@ fn curator_target_path() -> String {
format!("notes/curator-{date}.md")
}
+fn auto_improve_report_target_path() -> String {
+ let stamp = jiff::Timestamp::now().as_microsecond();
+ format!("notes/auto-improve-report-{stamp}.md")
+}
+
async fn handle_pending_writes_list(
State(state): State<Arc<AdminState>>,
Query(query): Query<PendingWritesQuery>,
diff --git a/crates/ai-memory-mcp/tests/admin_phase3.rs b/crates/ai-memory-mcp/tests/admin_phase3.rs
index a3cdd19c..a631ded1 100644
--- a/crates/ai-memory-mcp/tests/admin_phase3.rs
+++ b/crates/ai-memory-mcp/tests/admin_phase3.rs
@@ -33,7 +33,9 @@ use tower::ServiceExt;
/// Build a minimal `AdminState` with no LLM and no embedder.
async fn make_state(tmp: &TempDir) -> (AdminState, Store) {
let store = Store::open(tmp.path()).unwrap();
- let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ let wiki = Wiki::new(tmp.path(), store.writer.clone())
+ .unwrap()
+ .with_store_reader(store.reader.clone());
let db_path = store.db_path().to_path_buf();
let state = AdminState {
writer: store.writer.clone(),
@@ -149,8 +151,9 @@ async fn auto_improve_report_returns_telemetry_without_creating_proposals() {
json!({ "workspace": "default", "project": "scratch", "since_days": 30, "limit": 3 }),
)
.await;
- assert_eq!(resp.status(), StatusCode::OK);
+ let status = resp.status();
let body = body_json(resp).await;
+ assert_eq!(status, StatusCode::OK, "{body:?}");
assert!(body["summary"].as_str().unwrap().contains("run(s)"));
assert_eq!(body["aggregate"]["run_count"].as_u64().unwrap(), 1);
assert_eq!(
@@ -201,6 +204,117 @@ async fn auto_improve_report_missing_scope_fails_closed() {
assert_eq!(resp.status(), StatusCode::NOT_FOUND);
}
+#[tokio::test]
+async fn auto_improve_report_stage_creates_one_report_proposal_and_approval_writes_page() {
+ let tmp = TempDir::new().unwrap();
+ let (state, store) = make_state(&tmp).await;
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "scratch", None)
+ .await
+ .unwrap();
+
+ let before = store
+ .reader
+ .list_auto_improve_proposals(ws, proj, None, 50)
+ .await
+ .unwrap()
+ .len();
+ let resp = post(
+ state.clone(),
+ "/admin/auto-improve/report",
+ json!({ "workspace": "default", "project": "scratch", "since_days": 30, "limit": 3, "stage": true }),
+ )
+ .await;
+ let status = resp.status();
+ let body = body_json(resp).await;
+ assert_eq!(status, StatusCode::OK, "{body:?}");
+ let proposal_ids = body["proposal_ids"].as_array().unwrap();
+ assert_eq!(proposal_ids.len(), 1);
+ assert_eq!(body["sidecar_paths"].as_array().unwrap().len(), 1);
+
+ let proposals = store
+ .reader
+ .list_auto_improve_proposals(ws, proj, None, 50)
+ .await
+ .unwrap();
+ assert_eq!(proposals.len(), before + 1);
+ let staged = proposals
+ .iter()
+ .find(|p| p.id.to_string() == proposal_ids[0].as_str().unwrap())
+ .unwrap();
+ assert_eq!(staged.status, AutoImproveProposalStatus::Pending);
+ assert_eq!(staged.kind, "auto_improve_report");
+ let target_path = staged.target_path.as_str().to_string();
+ assert!(target_path.starts_with("notes/auto-improve-report-"));
+ assert!(target_path.ends_with(".md"));
+ assert!(
+ store
+ .reader
+ .page_body_by_ids(ws, proj, &target_path)
+ .await
+ .unwrap()
+ .is_none(),
+ "staging must not write the target page before approval"
+ );
+
+ let approve_resp = post(
+ state.clone(),
+ &format!(
+ "/admin/pending-writes/{}/approve?workspace=default&project=scratch",
+ staged.id
+ ),
+ json!({}),
+ )
+ .await;
+ assert_eq!(approve_resp.status(), StatusCode::OK);
+ let report_page = store
+ .reader
+ .page_body_by_ids(ws, proj, &target_path)
+ .await
+ .unwrap()
+ .expect("approval writes report page");
+ assert!(report_page.body.contains("# Auto-Improve Telemetry Report"));
+ assert_eq!(
+ store
+ .reader
+ .list_auto_improve_proposals(ws, proj, None, 50)
+ .await
+ .unwrap()
+ .len(),
+ before + 1,
+ "approval must not create extra proposals"
+ );
+
+ let telemetry_resp = post(
+ state,
+ "/admin/auto-improve/report",
+ json!({ "workspace": "default", "project": "scratch", "since_days": 30, "limit": 10 }),
+ )
+ .await;
+ assert_eq!(telemetry_resp.status(), StatusCode::OK);
+ let telemetry = body_json(telemetry_resp).await;
+ let maintenance = telemetry["aggregate"]["maintenance_proposals_by_kind"]
+ .as_array()
+ .unwrap();
+ assert!(maintenance.iter().any(|row| {
+ row["key"].as_str() == Some("auto_improve_report") && row["count"].as_u64() == Some(1)
+ }));
+ assert_eq!(
+ telemetry["aggregate"]["proposals_by_kind"]
+ .as_array()
+ .unwrap()
+ .len(),
+ 0,
+ "report proposals are maintenance, not learning proposals"
+ );
+}
+
// ---------------------------------------------------------------------------
// reorg
// ---------------------------------------------------------------------------
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index e0593dff..08e8d59d 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -275,6 +275,9 @@ uninstall auth user
Run `ai-memory --help` for the full tree.
+`auto-improve-report` is read-only by default; `--stage` creates one pending
+telemetry report page for audit/approval without staging learning-memory edits.
+
## Cross-cutting invariants
Carved in M0/M1; every milestone has to respect them. Each comes from
diff --git a/docs/auto-improvement-loop.md b/docs/auto-improvement-loop.md
index b1e2ef48..11415b88 100644
--- a/docs/auto-improvement-loop.md
+++ b/docs/auto-improvement-loop.md
@@ -373,7 +373,7 @@ them by default through the wiki mutation path. With `require_approval = true`,
|---|---|
| Background scheduler | Reviews newly completed sessions after the first-run watermark and applies or stages validated proposals according to approval policy. |
| `ai-memory auto-improve --session-id <id>` | Manually review one session and apply or stage validated proposals through the auto-improvement approval path. |
-| `ai-memory auto-improve-report --workspace <w> --project <p> [--days N] [--limit N]` | Read-only telemetry report for recent auto-improvement runs, proposal outcomes, terminal rates, and findings. Does not stage or create pending proposals. |
+| `ai-memory auto-improve-report --workspace <w> --project <p> [--days N] [--limit N] [--stage]` | Read-only telemetry report for recent auto-improvement runs, proposal outcomes, terminal rates, and findings by default. `--stage` creates exactly one pending telemetry report page for audit/approval. |
| `memory_auto_improve` | Manually review the latest completed session or a named session and apply or stage validated proposals through the same path. |
| `ai-memory curator` | Rule-based, report-only maintenance review. |
| `ai-memory curator --stage` | Stage exactly one curator report page for pending-writes approval. |