Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

feat(auto-improve): add telemetry report core

AkitaOnRailsc4741d89af03
diff --git a/crates/ai-memory-consolidate/src/auto_improve_telemetry.rs b/crates/ai-memory-consolidate/src/auto_improve_telemetry.rs
new file mode 100644
index 00000000..f67be0fc
--- /dev/null
+++ b/crates/ai-memory-consolidate/src/auto_improve_telemetry.rs
@@ -0,0 +1,451 @@
+//! Report-only telemetry for auto-improvement proposal outcomes.
+
+use ai_memory_core::{ProjectId, WorkspaceId};
+use ai_memory_store::{AutoImproveTelemetryAggregate, AutoImproveTelemetryCount, ReaderPool};
+use jiff::Timestamp;
+use serde::Serialize;
+
+const US_PER_DAY: i64 = 86_400_000_000;
+
+/// Default telemetry lookback window.
+pub const DEFAULT_AUTO_IMPROVE_TELEMETRY_SINCE_DAYS: u32 = 30;
+/// Default number of top count rows rendered per section.
+pub const DEFAULT_AUTO_IMPROVE_TELEMETRY_TOP_LIMIT: usize = 10;
+
+/// Parameters for an auto-improve telemetry report.
+#[derive(Debug, Clone, Serialize)]
+pub struct AutoImproveTelemetryParams {
+ /// Lookback window in days.
+ pub since_days: u32,
+ /// Maximum rows in each top-N count table.
+ pub top_limit: usize,
+}
+
+impl Default for AutoImproveTelemetryParams {
+ fn default() -> Self {
+ Self {
+ since_days: DEFAULT_AUTO_IMPROVE_TELEMETRY_SINCE_DAYS,
+ top_limit: DEFAULT_AUTO_IMPROVE_TELEMETRY_TOP_LIMIT,
+ }
+ }
+}
+
+/// Terminal-rate denominator and rates for learning proposals.
+#[derive(Debug, Clone, Serialize)]
+pub struct AutoImproveTerminalRates {
+ /// Terminal learning proposals only: approved + rejected + conflict + failed.
+ pub denominator: usize,
+ /// Approved / denominator.
+ pub approved_rate: f64,
+ /// Rejected / denominator.
+ pub rejected_rate: f64,
+ /// Conflict / denominator.
+ pub conflict_rate: f64,
+ /// Failed / denominator.
+ pub failed_rate: f64,
+}
+
+/// One bounded operational signal in the telemetry report.
+#[derive(Debug, Clone, Serialize)]
+pub struct AutoImproveTelemetryFinding {
+ /// Machine-readable finding kind.
+ pub kind: String,
+ /// Human severity for CLI/admin rendering.
+ pub severity: String,
+ /// Human-readable finding text.
+ pub message: String,
+ /// Optional structured details containing counts only.
+ #[serde(skip_serializing_if = "Option::is_none")]
+ pub detail: Option<serde_json::Value>,
+}
+
+/// Structured report-only auto-improve telemetry.
+#[derive(Debug, Clone, Serialize)]
+pub struct AutoImproveTelemetryReport {
+ /// Workspace reviewed.
+ pub workspace: String,
+ /// Project reviewed.
+ pub project: String,
+ /// ISO timestamp of report generation.
+ pub generated_at: String,
+ /// Lower bound timestamp for queried rows, in Unix microseconds.
+ pub since_created_at: i64,
+ /// Short summary.
+ pub summary: String,
+ /// Parameters used for this report.
+ pub params: AutoImproveTelemetryParams,
+ /// Store-provided aggregate counts.
+ pub aggregate: AutoImproveTelemetryAggregate,
+ /// Terminal learning proposal rates. Pending proposals are excluded.
+ pub terminal_rates: AutoImproveTerminalRates,
+ /// Bounded operational findings.
+ pub findings: Vec<AutoImproveTelemetryFinding>,
+ /// Known blind spots for this schema-backed report.
+ pub blind_spots: Vec<String>,
+}
+
+/// Build a report-only auto-improve telemetry report from persisted store rows.
+///
+/// # Errors
+/// Propagates store read failures.
+pub async fn run_auto_improve_telemetry_report(
+ reader: &ReaderPool,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ workspace_name: &str,
+ project_name: &str,
+ params: AutoImproveTelemetryParams,
+) -> ai_memory_store::StoreResult<AutoImproveTelemetryReport> {
+ let now = Timestamp::now();
+ let window_us = i64::from(params.since_days).saturating_mul(US_PER_DAY);
+ let since_created_at = now.as_microsecond().saturating_sub(window_us);
+ let aggregate = reader
+ .auto_improve_telemetry_aggregate(
+ workspace_id,
+ project_id,
+ since_created_at,
+ params.top_limit,
+ )
+ .await?;
+ Ok(build_auto_improve_telemetry_report(
+ workspace_name,
+ project_name,
+ now.to_string(),
+ since_created_at,
+ params,
+ aggregate,
+ ))
+}
+
+/// Build report semantics from already-collected aggregate counts.
+#[must_use]
+pub fn build_auto_improve_telemetry_report(
+ workspace_name: &str,
+ project_name: &str,
+ generated_at: String,
+ since_created_at: i64,
+ params: AutoImproveTelemetryParams,
+ aggregate: AutoImproveTelemetryAggregate,
+) -> AutoImproveTelemetryReport {
+ let terminal_rates = terminal_rates(&aggregate.proposals_by_status);
+ let pending = count_for(&aggregate.proposals_by_status, "pending");
+ let maintenance_count: usize = aggregate
+ .maintenance_proposals_by_kind
+ .iter()
+ .map(|row| row.count)
+ .sum();
+ let eval_gate_count = count_for(&aggregate.rejections_by_reason, "eval_gate_failed")
+ + count_for(&aggregate.rejections_by_reason, "eval_gate_timeout")
+ + count_for(&aggregate.rejections_by_reason, "eval_gate_error");
+ let mut findings = Vec::new();
+ if terminal_rates.denominator == 0 {
+ findings.push(AutoImproveTelemetryFinding {
+ kind: "no_terminal_learning_proposals".into(),
+ severity: "info".into(),
+ message: "No terminal learning proposals in the selected window.".into(),
+ detail: None,
+ });
+ }
+ if pending > 0 {
+ findings.push(AutoImproveTelemetryFinding {
+ kind: "pending_learning_proposals_excluded".into(),
+ severity: "info".into(),
+ message: format!(
+ "{pending} pending learning proposal(s) are counted but excluded from terminal rates."
+ ),
+ detail: Some(serde_json::json!({ "pending": pending })),
+ });
+ }
+ if maintenance_count > 0 {
+ findings.push(AutoImproveTelemetryFinding {
+ kind: "maintenance_proposals_excluded".into(),
+ severity: "info".into(),
+ message: format!(
+ "{maintenance_count} maintenance/report proposal(s) are excluded from learning metrics."
+ ),
+ detail: Some(serde_json::json!({ "maintenance_proposals": maintenance_count })),
+ });
+ }
+ if eval_gate_count > 0 {
+ findings.push(AutoImproveTelemetryFinding {
+ kind: "eval_gate_rejections_seen".into(),
+ severity: "info".into(),
+ message: format!(
+ "{eval_gate_count} eval-gate rejection(s) were recorded. This report does not infer eval success rates or score deltas."
+ ),
+ detail: Some(serde_json::json!({ "eval_gate_rejections": eval_gate_count })),
+ });
+ }
+ for row in &aggregate.repeated_rejection_fingerprints {
+ findings.push(AutoImproveTelemetryFinding {
+ kind: "repeated_rejection_fingerprint".into(),
+ severity: "warning".into(),
+ message: format!(
+ "Rejection fingerprint {} appeared {} times.",
+ truncate(&row.key, 64),
+ row.count
+ ),
+ detail: Some(serde_json::json!({
+ "fingerprint": truncate(&row.key, 64),
+ "count": row.count,
+ })),
+ });
+ }
+
+ let summary = if terminal_rates.denominator == 0 {
+ format!(
+ "{} run(s), no terminal learning proposals in the last {} day(s).",
+ aggregate.run_count, params.since_days
+ )
+ } else {
+ format!(
+ "{} run(s), {} terminal learning proposal(s), {:.1}% approved in the last {} day(s).",
+ aggregate.run_count,
+ terminal_rates.denominator,
+ terminal_rates.approved_rate * 100.0,
+ params.since_days
+ )
+ };
+
+ AutoImproveTelemetryReport {
+ workspace: workspace_name.to_string(),
+ project: project_name.to_string(),
+ generated_at,
+ since_created_at,
+ summary,
+ params,
+ aggregate,
+ terminal_rates,
+ findings,
+ blind_spots: vec![
+ "Provider/LLM errors before a run is staged may not appear in proposal or rejection counts."
+ .into(),
+ "Eval-gate rejection reasons are counted, but eval success rates and score deltas are not persisted in this schema."
+ .into(),
+ "Terminal rates use learning proposals only; pending and maintenance/report proposals are excluded from the denominator."
+ .into(),
+ ],
+ }
+}
+
+/// Render the telemetry report as a normal wiki markdown page.
+#[must_use]
+pub fn render_auto_improve_telemetry_report_markdown(
+ report: &AutoImproveTelemetryReport,
+) -> String {
+ let mut out = String::new();
+ out.push_str("# Auto-Improve Telemetry Report\n\n");
+ out.push_str(
+ "> Report-only: approving a staged copy of this report would store this page only. ",
+ );
+ out.push_str("It does not edit rules, procedures, notes, or `_meta` pages.\n\n");
+ out.push_str(&format!("- Workspace: `{}`\n", report.workspace));
+ out.push_str(&format!("- Project: `{}`\n", report.project));
+ out.push_str(&format!("- Generated: `{}`\n", report.generated_at));
+ out.push_str(&format!(
+ "- Window: last {} day(s) (`since_created_at={}`)\n",
+ report.params.since_days, report.since_created_at
+ ));
+ out.push_str(&format!("- Summary: {}\n\n", report.summary));
+
+ out.push_str("## Terminal Learning Proposal Rates\n\n");
+ out.push_str(
+ "Denominator: terminal learning proposals only (`approved`, `rejected`, `conflict`, `failed`). Pending proposals and maintenance/report proposals are excluded.\n\n",
+ );
+ out.push_str(&format!(
+ "- Denominator: {}\n- Approved: {:.1}%\n- Rejected: {:.1}%\n- Conflict: {:.1}%\n- Failed: {:.1}%\n\n",
+ report.terminal_rates.denominator,
+ report.terminal_rates.approved_rate * 100.0,
+ report.terminal_rates.rejected_rate * 100.0,
+ report.terminal_rates.conflict_rate * 100.0,
+ report.terminal_rates.failed_rate * 100.0,
+ ));
+
+ out.push_str("## Counts\n\n");
+ render_counts(
+ &mut out,
+ "Learning proposals by status",
+ &report.aggregate.proposals_by_status,
+ );
+ render_counts(
+ &mut out,
+ "Learning proposals by operation",
+ &report.aggregate.proposals_by_operation,
+ );
+ render_counts(
+ &mut out,
+ "Learning proposals by edit mode",
+ &report.aggregate.proposals_by_edit_mode,
+ );
+ render_counts(
+ &mut out,
+ "Learning proposals by kind",
+ &report.aggregate.proposals_by_kind,
+ );
+ render_counts(
+ &mut out,
+ "Maintenance/report proposals by kind",
+ &report.aggregate.maintenance_proposals_by_kind,
+ );
+ render_counts(
+ &mut out,
+ "Top learning targets",
+ &report.aggregate.top_targets,
+ );
+ render_counts(
+ &mut out,
+ "Rejections by reason",
+ &report.aggregate.rejections_by_reason,
+ );
+ render_counts(
+ &mut out,
+ "Repeated rejection fingerprints",
+ &report.aggregate.repeated_rejection_fingerprints,
+ );
+ render_counts(
+ &mut out,
+ "Rejected targets",
+ &report.aggregate.rejected_targets,
+ );
+
+ out.push_str("## Findings\n\n");
+ if report.findings.is_empty() {
+ out.push_str("No telemetry findings.\n\n");
+ } else {
+ for finding in &report.findings {
+ out.push_str(&format!(
+ "- **{}** ({}) — {}\n",
+ finding.kind, finding.severity, finding.message
+ ));
+ }
+ out.push('\n');
+ }
+
+ out.push_str("## Known Blind Spots\n\n");
+ for blind_spot in &report.blind_spots {
+ out.push_str(&format!("- {blind_spot}\n"));
+ }
+ out
+}
+
+fn terminal_rates(statuses: &[AutoImproveTelemetryCount]) -> AutoImproveTerminalRates {
+ let approved = count_for(statuses, "approved");
+ let rejected = count_for(statuses, "rejected");
+ let conflict = count_for(statuses, "conflict");
+ let failed = count_for(statuses, "failed");
+ let denominator = approved + rejected + conflict + failed;
+ AutoImproveTerminalRates {
+ denominator,
+ approved_rate: rate(approved, denominator),
+ rejected_rate: rate(rejected, denominator),
+ conflict_rate: rate(conflict, denominator),
+ failed_rate: rate(failed, denominator),
+ }
+}
+
+fn count_for(rows: &[AutoImproveTelemetryCount], key: &str) -> usize {
+ rows.iter()
+ .find(|row| row.key == key)
+ .map(|row| row.count)
+ .unwrap_or(0)
+}
+
+fn rate(numerator: usize, denominator: usize) -> f64 {
+ if denominator == 0 {
+ 0.0
+ } else {
+ numerator as f64 / denominator as f64
+ }
+}
+
+fn render_counts(out: &mut String, title: &str, rows: &[AutoImproveTelemetryCount]) {
+ out.push_str(&format!("### {title}\n\n"));
+ if rows.is_empty() {
+ out.push_str("- None\n\n");
+ return;
+ }
+ for row in rows {
+ out.push_str(&format!("- `{}`: {}\n", truncate(&row.key, 96), row.count));
+ }
+ out.push('\n');
+}
+
+fn truncate(value: &str, max_chars: usize) -> String {
+ let mut out = String::new();
+ for (idx, ch) in value.chars().enumerate() {
+ if idx >= max_chars {
+ out.push('…');
+ return out;
+ }
+ out.push(ch);
+ }
+ out
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ fn count(key: &str, n: usize) -> AutoImproveTelemetryCount {
+ AutoImproveTelemetryCount {
+ key: key.to_string(),
+ count: n,
+ }
+ }
+
+ #[test]
+ fn report_uses_terminal_learning_denominator_and_explains_exclusions() {
+ let aggregate = AutoImproveTelemetryAggregate {
+ run_count: 4,
+ runs_with_learning_proposals: 2,
+ proposals_by_status: vec![
+ count("approved", 3),
+ count("pending", 2),
+ count("rejected", 1),
+ count("conflict", 1),
+ ],
+ proposals_by_operation: vec![count("create", 5), count("update", 2)],
+ proposals_by_edit_mode: vec![count("full_page", 6), count("patch", 1)],
+ proposals_by_kind: vec![count("note", 6), count("procedure", 1)],
+ maintenance_proposals_by_kind: vec![count("auto_improve_report", 1)],
+ top_targets: vec![count("notes/a.md", 2)],
+ rejections_by_reason: vec![count("eval_gate_failed", 2)],
+ repeated_rejection_fingerprints: vec![count("abc123", 2)],
+ rejected_targets: vec![count("notes/a.md", 2)],
+ };
+ let report = build_auto_improve_telemetry_report(
+ "default",
+ "app",
+ "2026-06-22T00:00:00Z".into(),
+ 42,
+ AutoImproveTelemetryParams::default(),
+ aggregate,
+ );
+
+ assert_eq!(report.terminal_rates.denominator, 5);
+ assert!((report.terminal_rates.approved_rate - 0.6).abs() < f64::EPSILON);
+ assert!(
+ report
+ .findings
+ .iter()
+ .any(|finding| finding.kind == "pending_learning_proposals_excluded")
+ );
+ assert!(
+ report
+ .findings
+ .iter()
+ .any(|finding| finding.kind == "maintenance_proposals_excluded")
+ );
+ assert!(
+ report
+ .findings
+ .iter()
+ .any(|finding| finding.kind == "eval_gate_rejections_seen")
+ );
+
+ let markdown = render_auto_improve_telemetry_report_markdown(&report);
+ assert!(markdown.contains("# Auto-Improve Telemetry Report"));
+ assert!(markdown.contains("Denominator: terminal learning proposals only"));
+ assert!(markdown.contains("Eval-gate rejection reasons are counted"));
+ }
+}
diff --git a/crates/ai-memory-consolidate/src/lib.rs b/crates/ai-memory-consolidate/src/lib.rs
index f72cab55..542a33b4 100644
--- a/crates/ai-memory-consolidate/src/lib.rs
+++ b/crates/ai-memory-consolidate/src/lib.rs
@@ -9,6 +9,7 @@
//! M7b extends this to multi-page atomic fan-out.
pub mod auto_improve;
+pub mod auto_improve_telemetry;
pub mod bootstrap;
pub mod consolidator;
pub mod curator;
@@ -32,6 +33,12 @@ pub use auto_improve::{
DEFAULT_AUTO_IMPROVE_REJECTION_CONTEXT_DAYS, default_auto_improve_eval_targets,
run_auto_improve_review,
};
+pub use auto_improve_telemetry::{
+ AutoImproveTelemetryFinding, AutoImproveTelemetryParams, AutoImproveTelemetryReport,
+ AutoImproveTerminalRates, DEFAULT_AUTO_IMPROVE_TELEMETRY_SINCE_DAYS,
+ DEFAULT_AUTO_IMPROVE_TELEMETRY_TOP_LIMIT, build_auto_improve_telemetry_report,
+ render_auto_improve_telemetry_report_markdown, run_auto_improve_telemetry_report,
+};
pub use bootstrap::{
Bootstrap, BootstrapConfig, BootstrapError, BootstrapOutcome, BootstrapSource,
DEFAULT_CHUNK_INPUT_TOKENS, ProjectNameStrategy, SourceCounts, SourceKind, collect_sources,
diff --git a/crates/ai-memory-store/src/auto_improve.rs b/crates/ai-memory-store/src/auto_improve.rs
index 8032418a..c4ddd064 100644
--- a/crates/ai-memory-store/src/auto_improve.rs
+++ b/crates/ai-memory-store/src/auto_improve.rs
@@ -188,6 +188,42 @@ pub struct AutoImproveRejectionSummary {
pub created_at: i64,
}
+/// Aggregate telemetry for auto-improvement runs in one scope.
+#[derive(Debug, Clone, Serialize)]
+pub struct AutoImproveTelemetryAggregate {
+ /// Number of auto-improve runs in the window.
+ pub run_count: usize,
+ /// Number of runs that staged at least one learning proposal.
+ pub runs_with_learning_proposals: usize,
+ /// Learning proposal counts by status.
+ pub proposals_by_status: Vec<AutoImproveTelemetryCount>,
+ /// Learning proposal counts by operation.
+ pub proposals_by_operation: Vec<AutoImproveTelemetryCount>,
+ /// Learning proposal counts by edit mode.
+ pub proposals_by_edit_mode: Vec<AutoImproveTelemetryCount>,
+ /// Learning proposal counts by kind.
+ pub proposals_by_kind: Vec<AutoImproveTelemetryCount>,
+ /// Maintenance/report proposal counts by kind.
+ pub maintenance_proposals_by_kind: Vec<AutoImproveTelemetryCount>,
+ /// Most frequently targeted learning pages.
+ pub top_targets: Vec<AutoImproveTelemetryCount>,
+ /// Rejection counts by reason.
+ pub rejections_by_reason: Vec<AutoImproveTelemetryCount>,
+ /// Repeated rejection fingerprints.
+ pub repeated_rejection_fingerprints: Vec<AutoImproveTelemetryCount>,
+ /// Most common rejected targets.
+ pub rejected_targets: Vec<AutoImproveTelemetryCount>,
+}
+
+/// Generic `(key, count)` telemetry row.
+#[derive(Debug, Clone, Serialize)]
+pub struct AutoImproveTelemetryCount {
+ /// Aggregated key; field-specific meaning depends on the source vector.
+ pub key: String,
+ /// Number of rows matching the key.
+ pub count: usize,
+}
+
#[derive(Debug, Clone)]
pub struct RejectAutoImproveProposal {
pub workspace_id: WorkspaceId,
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 4a9cb736..b87cdb89 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -28,9 +28,9 @@ pub use fts_query::prepare_fts5_query;
pub use auto_improve::{
ApproveAutoImproveProposal, ApproveAutoImproveProposalResult, AutoImproveProposalDetail,
AutoImproveProposalEvent, AutoImproveProposalOperation, AutoImproveProposalStatus,
- AutoImproveProposalSummary, AutoImproveRejectionSummary, FailAutoImproveProposal,
- NewAutoImproveProposal, RejectAutoImproveProposal, StageAutoImproveRun, StagedAutoImproveRun,
- artifact_path_for,
+ AutoImproveProposalSummary, AutoImproveRejectionSummary, AutoImproveTelemetryAggregate,
+ AutoImproveTelemetryCount, FailAutoImproveProposal, NewAutoImproveProposal,
+ RejectAutoImproveProposal, StageAutoImproveRun, StagedAutoImproveRun, artifact_path_for,
};
pub use decay::{DecayParams, retention_score};
pub use error::{StoreError, StoreResult};
@@ -210,6 +210,13 @@ mod tests {
)
}
+ fn telemetry_count(rows: &[AutoImproveTelemetryCount], key: &str) -> usize {
+ rows.iter()
+ .find(|row| row.key == key)
+ .map(|row| row.count)
+ .unwrap_or(0)
+ }
+
#[tokio::test]
async fn auto_improve_migration_and_stage_persist_reopen_list_detail_scope() {
let tmp = TempDir::new().unwrap();
@@ -698,6 +705,333 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn auto_improve_telemetry_aggregate_counts_learning_activity_only() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let ws = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let proj = store
+ .writer
+ .get_or_create_project(ws, "app", None)
+ .await
+ .unwrap();
+ let other = store
+ .writer
+ .get_or_create_project(ws, "other", None)
+ .await
+ .unwrap();
+
+ store
+ .writer
+ .upsert_page(sample_page(ws, proj, "notes/update.md", "old update"))
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(sample_page(ws, proj, "procedures/patch.md", "old patch"))
+ .await
+ .unwrap();
+
+ let mut update = proposal(
+ "notes/update.md",
+ AutoImproveProposalOperation::Update,
+ "new update",
+ );
+ update.kind = "decision".into();
+ let mut patch = proposal(
+ "procedures/patch.md",
+ AutoImproveProposalOperation::Update,
+ "new patch",
+ );
+ patch.kind = "procedure".into();
+ patch.edit_mode = Some("patch".into());
+ patch.patch_json = Some(serde_json::json!([{ "op": "append", "content": "new" }]));
+ patch.expected_base_body_sha256 = Some(sha256("old patch"));
+
+ let staged = store
+ .writer
+ .stage_auto_improve_run(stage_input(
+ ws,
+ proj,
+ vec![
+ proposal(
+ "notes/pending.md",
+ AutoImproveProposalOperation::Create,
+ "pending",
+ ),
+ proposal(
+ "notes/approved.md",
+ AutoImproveProposalOperation::Create,
+ "approved",
+ ),
+ proposal(
+ "notes/rejected.md",
+ AutoImproveProposalOperation::Create,
+ "rejected",
+ ),
+ proposal(
+ "notes/failed.md",
+ AutoImproveProposalOperation::Create,
+ "failed",
+ ),
+ proposal(
+ "notes/conflict.md",
+ AutoImproveProposalOperation::Create,
+ "proposal",
+ ),
+ update,
+ patch,
+ ],
+ ))
+ .await
+ .unwrap();
+ let approved_id = staged.proposal_ids[1];
+ let rejected_id = staged.proposal_ids[2];
+ let failed_id = staged.proposal_ids[3];
+ let conflict_id = staged.proposal_ids[4];
+ let actor = ActorContext::default();
+
+ store
+ .writer
+ .approve_auto_improve_proposal(ApproveAutoImproveProposal {
+ workspace_id: ws,
+ project_id: proj,
+ proposal_id: approved_id,
+ page: sample_page(ws, proj, "notes/approved.md", "approved"),
+ actor: actor.clone(),
+ author_id: None,
+ checkpoint: None,
+ })
+ .await
+ .unwrap();
+ store
+ .writer
+ .reject_auto_improve_proposal(RejectAutoImproveProposal {
+ workspace_id: ws,
+ project_id: proj,
+ proposal_id: rejected_id,
+ reason: "human rejected".into(),
+ actor: actor.clone(),
+ author_id: None,
+ })
+ .await
+ .unwrap();
+ store
+ .writer
+ .fail_auto_improve_proposal(FailAutoImproveProposal {
+ workspace_id: ws,
+ project_id: proj,
+ proposal_id: failed_id,
+ reason: "admission denied".into(),
+ actor: actor.clone(),
+ author_id: None,
+ })
+ .await
+ .unwrap();
+ store
+ .writer
+ .upsert_page(sample_page(ws, proj, "notes/conflict.md", "external"))
+ .await
+ .unwrap();
+ assert_eq!(
+ store
+ .writer
+ .approve_auto_improve_proposal(ApproveAutoImproveProposal {
+ workspace_id: ws,
+ project_id: proj,
+ proposal_id: conflict_id,
+ page: sample_page(ws, proj, "notes/conflict.md", "proposal"),
+ actor: actor.clone(),
+ author_id: None,
+ checkpoint: None,
+ })
+ .await
+ .unwrap(),
+ ApproveAutoImproveProposalResult::Conflict
+ );
+
+ let mut curator_report = proposal(
+ "reports/curator.md",
+ AutoImproveProposalOperation::Create,
+ "curator",
+ );
+ curator_report.kind = "curator_report".into();
+ let mut telemetry_report = proposal(
+ "reports/auto-improve.md",
+ AutoImproveProposalOperation::Create,
+ "telemetry",
+ );
+ telemetry_report.kind = "auto_improve_report".into();
+ store
+ .writer
+ .stage_auto_improve_run(stage_input(
+ ws,
+ proj,
+ vec![curator_report, telemetry_report],
+ ))
+ .await
+ .unwrap();
+
+ let mut eval_rejections = stage_input(ws, proj, Vec::new());
+ eval_rejections.rejected_candidates_json = serde_json::json!([
+ {
+ "reason": "eval_gate_failed",
+ "target_path": "eval/repeat.md",
+ "kind": "note",
+ "operation": "create",
+ "edit_mode": "full_page",
+ "summary": "same eval failure"
+ },
+ {
+ "reason": "eval_gate_failed",
+ "target_path": "eval/repeat.md",
+ "kind": "note",
+ "operation": "create",
+ "edit_mode": "full_page",
+ "summary": "same eval failure"
+ },
+ {
+ "reason": "eval_gate_timeout",
+ "target_path": "eval/timeout.md",
+ "summary": "timeout"
+ },
+ {
+ "reason": "eval_gate_error",
+ "target_path": "eval/error.md",
+ "summary": "error"
+ }
+ ]);
+ store
+ .writer
+ .stage_auto_improve_run(eval_rejections)
+ .await
+ .unwrap();
+
+ store
+ .writer
+ .stage_auto_improve_run(stage_input(
+ ws,
+ other,
+ vec![proposal(
+ "notes/other.md",
+ AutoImproveProposalOperation::Create,
+ "other",
+ )],
+ ))
+ .await
+ .unwrap();
+
+ let aggregate = store
+ .reader
+ .auto_improve_telemetry_aggregate(ws, proj, 0, 10)
+ .await
+ .unwrap();
+ assert_eq!(aggregate.run_count, 3);
+ assert_eq!(aggregate.runs_with_learning_proposals, 1);
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_status, "pending"),
+ 3
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_status, "approved"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_status, "rejected"),
+ 1
+ );
+ assert_eq!(telemetry_count(&aggregate.proposals_by_status, "failed"), 1);
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_status, "conflict"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_operation, "create"),
+ 5
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_operation, "update"),
+ 2
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_edit_mode, "full_page"),
+ 6
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_edit_mode, "patch"),
+ 1
+ );
+ assert_eq!(telemetry_count(&aggregate.proposals_by_kind, "note"), 5);
+ assert_eq!(telemetry_count(&aggregate.proposals_by_kind, "decision"), 1);
+ assert_eq!(
+ telemetry_count(&aggregate.proposals_by_kind, "procedure"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.maintenance_proposals_by_kind, "curator_report"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(
+ &aggregate.maintenance_proposals_by_kind,
+ "auto_improve_report"
+ ),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.rejections_by_reason, "human rejected"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.rejections_by_reason, "admission denied"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(
+ &aggregate.rejections_by_reason,
+ "target changed since proposal was staged"
+ ),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.rejections_by_reason, "eval_gate_failed"),
+ 2
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.rejections_by_reason, "eval_gate_timeout"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.rejections_by_reason, "eval_gate_error"),
+ 1
+ );
+ assert_eq!(
+ telemetry_count(&aggregate.rejected_targets, "eval/repeat.md"),
+ 2
+ );
+ assert!(
+ aggregate
+ .repeated_rejection_fingerprints
+ .iter()
+ .any(|row| row.count == 2)
+ );
+
+ let other_aggregate = store
+ .reader
+ .auto_improve_telemetry_aggregate(ws, other, 0, 10)
+ .await
+ .unwrap();
+ assert_eq!(other_aggregate.run_count, 1);
+ assert_eq!(
+ telemetry_count(&other_aggregate.proposals_by_status, "pending"),
+ 1
+ );
+ }
+
#[tokio::test]
async fn auto_improve_approve_upserts_page_and_conflicts_are_sql_atomic() {
let tmp = TempDir::new().unwrap();
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index 9c0aca9a..c688f258 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -27,8 +27,8 @@ use uuid::Uuid;
use crate::auto_improve::{
AutoImproveProposalDetail, AutoImproveProposalEvent, AutoImproveProposalStatus,
- AutoImproveProposalSummary, AutoImproveRejectionSummary, bytes32, opt_bytes32,
- summary_from_row, to_sql_err,
+ AutoImproveProposalSummary, AutoImproveRejectionSummary, AutoImproveTelemetryAggregate,
+ AutoImproveTelemetryCount, bytes32, opt_bytes32, summary_from_row, to_sql_err,
};
use crate::error::{StoreError, StoreResult};
use crate::fts_query::prepare_fts5_query;
@@ -3275,6 +3275,124 @@ impl ReaderPool {
.await
}
+ /// Aggregate auto-improvement telemetry for one workspace/project scope.
+ ///
+ /// `since_created_at` filters by run/proposal/rejection creation time in Unix
+ /// microseconds. Maintenance/report proposals are excluded from learning
+ /// metrics and counted separately.
+ pub async fn auto_improve_telemetry_aggregate(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ since_created_at: i64,
+ top_limit: usize,
+ ) -> StoreResult<AutoImproveTelemetryAggregate> {
+ self.with_conn(move |conn| {
+ let top_limit = i64::try_from(top_limit).unwrap_or(i64::MAX);
+ let run_count: i64 = conn.query_row(
+ "SELECT COUNT(*) FROM auto_improve_runs
+ WHERE workspace_id = ?1 AND project_id = ?2 AND created_at >= ?3",
+ params![
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ since_created_at
+ ],
+ |row| row.get(0),
+ )?;
+ let runs_with_learning_proposals: i64 = conn.query_row(
+ "SELECT COUNT(DISTINCT run_id) FROM auto_improve_proposals
+ WHERE workspace_id = ?1 AND project_id = ?2 AND staged_at >= ?3
+ AND kind NOT IN ('curator_report', 'auto_improve_report')",
+ params![
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ since_created_at
+ ],
+ |row| row.get(0),
+ )?;
+ Ok(AutoImproveTelemetryAggregate {
+ run_count: usize::try_from(run_count).unwrap_or(usize::MAX),
+ runs_with_learning_proposals: usize::try_from(runs_with_learning_proposals)
+ .unwrap_or(usize::MAX),
+ proposals_by_status: auto_improve_group_counts(
+ conn,
+ "status",
+ workspace_id,
+ project_id,
+ since_created_at,
+ None,
+ )?,
+ proposals_by_operation: auto_improve_group_counts(
+ conn,
+ "operation",
+ workspace_id,
+ project_id,
+ since_created_at,
+ None,
+ )?,
+ proposals_by_edit_mode: auto_improve_group_counts(
+ conn,
+ "edit_mode",
+ workspace_id,
+ project_id,
+ since_created_at,
+ None,
+ )?,
+ proposals_by_kind: auto_improve_group_counts(
+ conn,
+ "kind",
+ workspace_id,
+ project_id,
+ since_created_at,
+ None,
+ )?,
+ maintenance_proposals_by_kind: auto_improve_group_counts(
+ conn,
+ "kind",
+ workspace_id,
+ project_id,
+ since_created_at,
+ Some("kind IN ('curator_report', 'auto_improve_report')"),
+ )?,
+ top_targets: auto_improve_group_counts(
+ conn,
+ "target_path",
+ workspace_id,
+ project_id,
+ since_created_at,
+ Some("kind NOT IN ('curator_report', 'auto_improve_report')"),
+ )?
+ .into_iter()
+ .take(usize::try_from(top_limit).unwrap_or(usize::MAX))
+ .collect(),
+ rejections_by_reason: auto_improve_rejection_group_counts(
+ conn,
+ "reason",
+ workspace_id,
+ project_id,
+ since_created_at,
+ top_limit,
+ )?,
+ repeated_rejection_fingerprints: auto_improve_repeated_rejection_fingerprints(
+ conn,
+ workspace_id,
+ project_id,
+ since_created_at,
+ top_limit,
+ )?,
+ rejected_targets: auto_improve_rejection_group_counts(
+ conn,
+ "COALESCE(target_path, '(none)')",
+ workspace_id,
+ project_id,
+ since_created_at,
+ top_limit,
+ )?,
+ })
+ })
+ .await
+ }
+
/// Look up a workspace id by name without creating it.
///
/// Returns `None` when no workspace with the given name exists.
@@ -4125,6 +4243,109 @@ fn materialise_decay_candidate(
})
}
+fn auto_improve_group_counts(
+ conn: &Connection,
+ column: &str,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ since_created_at: i64,
+ extra_where: Option<&str>,
+) -> rusqlite::Result<Vec<AutoImproveTelemetryCount>> {
+ let learning_filter = "kind NOT IN ('curator_report', 'auto_improve_report')";
+ let filter = extra_where.unwrap_or(learning_filter);
+ let sql = format!(
+ "SELECT COALESCE({column}, '(none)') AS key, COUNT(*) AS count
+ FROM auto_improve_proposals
+ WHERE workspace_id = ?1 AND project_id = ?2 AND staged_at >= ?3 AND {filter}
+ GROUP BY key ORDER BY count DESC, key ASC"
+ );
+ let mut stmt = conn.prepare(&sql)?;
+ let rows = stmt.query_map(
+ params![
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ since_created_at
+ ],
+ telemetry_count_from_row,
+ )?;
+ let mut out = Vec::new();
+ for row in rows {
+ out.push(row?);
+ }
+ Ok(out)
+}
+
+fn auto_improve_rejection_group_counts(
+ conn: &Connection,
+ expr: &str,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ since_created_at: i64,
+ limit: i64,
+) -> rusqlite::Result<Vec<AutoImproveTelemetryCount>> {
+ let sql = format!(
+ "SELECT {expr} AS key, COUNT(*) AS count
+ FROM auto_improve_rejections
+ WHERE workspace_id = ?1 AND project_id = ?2 AND created_at >= ?3
+ GROUP BY key ORDER BY count DESC, key ASC LIMIT ?4"
+ );
+ let mut stmt = conn.prepare(&sql)?;
+ let rows = stmt.query_map(
+ params![
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ since_created_at,
+ limit
+ ],
+ telemetry_count_from_row,
+ )?;
+ let mut out = Vec::new();
+ for row in rows {
+ out.push(row?);
+ }
+ Ok(out)
+}
+
+fn auto_improve_repeated_rejection_fingerprints(
+ conn: &Connection,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ since_created_at: i64,
+ limit: i64,
+) -> rusqlite::Result<Vec<AutoImproveTelemetryCount>> {
+ let mut stmt = conn.prepare(
+ "SELECT normalized_fingerprint AS key, COUNT(*) AS count
+ FROM auto_improve_rejections
+ WHERE workspace_id = ?1 AND project_id = ?2 AND created_at >= ?3
+ GROUP BY normalized_fingerprint HAVING count > 1
+ ORDER BY count DESC, key ASC LIMIT ?4",
+ )?;
+ let rows = stmt.query_map(
+ params![
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ since_created_at,
+ limit
+ ],
+ telemetry_count_from_row,
+ )?;
+ let mut out = Vec::new();
+ for row in rows {
+ out.push(row?);
+ }
+ Ok(out)
+}
+
+fn telemetry_count_from_row(
+ row: &rusqlite::Row<'_>,
+) -> rusqlite::Result<AutoImproveTelemetryCount> {
+ let count: i64 = row.get(1)?;
+ Ok(AutoImproveTelemetryCount {
+ key: row.get(0)?,
+ count: usize::try_from(count).unwrap_or(usize::MAX),
+ })
+}
+
/// Normalize a cwd for comparison: treat Windows backslashes as path
/// separators and trim trailing separators while keeping a bare root as `/`.
/// Pure and string-only; it does not touch the filesystem.