Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(maintenance): persist scheduler cadence

AkitaOnRails708939e01398
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 68757971..17d9f1f1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
via MCP `memory_handoff_accept`.
### Changed
+- Scheduled forget sweep and rule-based lint now persist their last successful
+ completion across restarts. Overdue or never-run jobs perform one bounded
+ startup catch-up, while not-due jobs wait only their remaining interval;
+ failures retry without advancing cadence. Embedding backfill remains opt-in
+ and does not gain startup catch-up behavior ([#192]).
- `init`-generated token peppers no longer make operational `/admin/*` routes
root-only before the first user is created. Admin mode now switches
immediately and fail-closed from a store-backed user-row check; expired users
diff --git a/crates/ai-memory-cli/Cargo.toml b/crates/ai-memory-cli/Cargo.toml
index e428888d..c269c047 100644
--- a/crates/ai-memory-cli/Cargo.toml
+++ b/crates/ai-memory-cli/Cargo.toml
@@ -53,6 +53,7 @@ tempfile.workspace = true
[dev-dependencies]
tempfile.workspace = true
tower.workspace = true
+tokio = { workspace = true, features = ["test-util"] }
[lints]
workspace = true
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index 3ef3313e..02995fcd 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -1,6 +1,7 @@
//! `ai-memory serve` — MCP server with optional filesystem watcher.
use std::convert::Infallible;
+use std::future::Future;
use std::path::Path;
use std::sync::Arc;
use std::time::Duration;
@@ -55,6 +56,8 @@ const EMBEDDING_WRITE_BATCH: usize = 100;
/// `POST /admin/bootstrap` may carry a large JSON array of sources even
/// after client-side prune; keep hooks/MCP at [`MAX_BODY_BYTES`].
const BOOTSTRAP_MAX_BODY_BYTES: usize = 32 * 1024 * 1024;
+/// Startup and failure retry delay, capped by each job's configured interval.
+const MAINTENANCE_STARTUP_DELAY_CAP: Duration = Duration::from_secs(60);
/// Validate the credentials that keep an existing multi-user installation
/// closed. Bootstrap installs have no user rows yet and retain their historical
@@ -92,6 +95,75 @@ struct ConsolidatorSetup {
admin_llm: Option<Arc<dyn LlmProvider>>,
}
+fn maintenance_start_delay(
+ last_success_at: Option<i64>,
+ now_microseconds: i64,
+ interval: Duration,
+) -> Duration {
+ let fallback = interval.min(MAINTENANCE_STARTUP_DELAY_CAP);
+ let Some(last_success_at) = last_success_at else {
+ return fallback;
+ };
+ let interval_microseconds = i64::try_from(interval.as_micros()).unwrap_or(i64::MAX);
+ let due_at = last_success_at.saturating_add(interval_microseconds);
+ if due_at <= now_microseconds {
+ fallback
+ } else {
+ // A future persisted timestamp (clock correction / restore) must not
+ // defer maintenance longer than its configured interval.
+ Duration::from_micros(due_at.saturating_sub(now_microseconds) as u64).min(interval)
+ }
+}
+
+async fn run_persisted_maintenance_job<F, Fut, R, RFut, N>(
+ writer: WriterHandle,
+ job: ai_memory_store::MaintenanceJob,
+ interval: Duration,
+ mut load_last_success: R,
+ now_microseconds: N,
+ mut tick: F,
+) where
+ F: FnMut() -> Fut + Send + 'static,
+ Fut: Future<Output = Result<()>> + Send,
+ R: FnMut() -> RFut + Send + 'static,
+ RFut: Future<Output = Result<Option<i64>>> + Send,
+ N: Fn() -> i64 + Send + 'static,
+{
+ let retry_delay = interval.min(MAINTENANCE_STARTUP_DELAY_CAP);
+ let mut cadence_known = false;
+ let mut delay = Duration::ZERO;
+
+ loop {
+ tokio::time::sleep(delay).await;
+ if !cadence_known {
+ match load_last_success().await {
+ Ok(last_success_at) => {
+ delay = maintenance_start_delay(last_success_at, now_microseconds(), interval);
+ cadence_known = true;
+ }
+ Err(error) => {
+ tracing::warn!(%error, job = job.as_str(), "maintenance cadence state read failed; retrying before running work");
+ delay = retry_delay;
+ }
+ }
+ continue;
+ }
+ match tick().await {
+ Ok(()) => match writer.record_maintenance_job_success(job).await {
+ Ok(()) => delay = interval,
+ Err(error) => {
+ tracing::warn!(%error, job = job.as_str(), "maintenance success state write failed; retrying job");
+ delay = retry_delay;
+ }
+ },
+ Err(error) => {
+ tracing::warn!(%error, job = job.as_str(), "scheduled maintenance job failed; retrying");
+ delay = retry_delay;
+ }
+ }
+ }
+}
+
/// Run the `serve` subcommand.
///
/// # Errors
@@ -531,22 +603,51 @@ async fn start_maintenance_scheduler(
let writer = writer.clone();
tasks.push(tokio::spawn(async move {
let interval = std::time::Duration::from_secs(forget_sweep_interval_secs);
- loop {
- tokio::time::sleep(interval).await;
- let started = std::time::Instant::now();
- match run_scheduled_sweep_tick(&reader, &writer, &decay).await {
- Ok(outcome) => info!(
- scopes = outcome.scopes,
- candidates_evaluated = outcome.candidates_evaluated,
- evicted = outcome.evicted,
- hard_deleted = outcome.hard_deleted,
- errors = outcome.errors,
- elapsed_ms = started.elapsed().as_millis(),
- "scheduled forget sweep completed"
- ),
- Err(e) => tracing::warn!(error = %e, "scheduled forget sweep failed"),
- }
- }
+ run_persisted_maintenance_job(
+ writer.clone(),
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ interval,
+ {
+ let reader = reader.clone();
+ move || {
+ let reader = reader.clone();
+ async move {
+ Ok(reader
+ .maintenance_job_last_success(
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ )
+ .await?)
+ }
+ }
+ },
+ || jiff::Timestamp::now().as_microsecond(),
+ move || {
+ let reader = reader.clone();
+ let writer = writer.clone();
+ let decay = decay;
+ async move {
+ let started = std::time::Instant::now();
+ let outcome = run_scheduled_sweep_tick(&reader, &writer, &decay).await?;
+ if outcome.errors > 0 {
+ anyhow::bail!(
+ "scheduled forget sweep had {} scope errors",
+ outcome.errors
+ );
+ }
+ info!(
+ scopes = outcome.scopes,
+ candidates_evaluated = outcome.candidates_evaluated,
+ evicted = outcome.evicted,
+ hard_deleted = outcome.hard_deleted,
+ errors = outcome.errors,
+ elapsed_ms = started.elapsed().as_millis(),
+ "scheduled forget sweep completed"
+ );
+ Ok(())
+ }
+ },
+ )
+ .await;
}));
}
@@ -589,24 +690,54 @@ async fn start_maintenance_scheduler(
if maintenance_enabled && lint_interval_secs > 0 {
let reader = reader.clone();
+ let writer = writer.clone();
let wiki = wiki.clone();
let llm = llm.clone();
tasks.push(tokio::spawn(async move {
let interval = std::time::Duration::from_secs(lint_interval_secs);
- loop {
- tokio::time::sleep(interval).await;
- let started = std::time::Instant::now();
- match run_scheduled_lint_tick(&reader, &wiki, llm.as_ref()).await {
- Ok(outcome) => info!(
- scopes = outcome.scopes,
- findings = outcome.findings,
- errors = outcome.errors,
- elapsed_ms = started.elapsed().as_millis(),
- "scheduled rule-based lint completed"
- ),
- Err(e) => tracing::warn!(error = %e, "scheduled lint failed"),
- }
- }
+ run_persisted_maintenance_job(
+ writer,
+ ai_memory_store::MaintenanceJob::RuleLint,
+ interval,
+ {
+ let reader = reader.clone();
+ move || {
+ let reader = reader.clone();
+ async move {
+ Ok(reader
+ .maintenance_job_last_success(
+ ai_memory_store::MaintenanceJob::RuleLint,
+ )
+ .await?)
+ }
+ }
+ },
+ || jiff::Timestamp::now().as_microsecond(),
+ move || {
+ let reader = reader.clone();
+ let wiki = wiki.clone();
+ let llm = llm.clone();
+ async move {
+ let started = std::time::Instant::now();
+ let outcome = run_scheduled_lint_tick(&reader, &wiki, llm.as_ref()).await?;
+ if outcome.errors > 0 {
+ anyhow::bail!(
+ "scheduled rule-based lint had {} scope errors",
+ outcome.errors
+ );
+ }
+ info!(
+ scopes = outcome.scopes,
+ findings = outcome.findings,
+ errors = outcome.errors,
+ elapsed_ms = started.elapsed().as_millis(),
+ "scheduled rule-based lint completed"
+ );
+ Ok(())
+ }
+ },
+ )
+ .await;
}));
}
@@ -2078,6 +2209,485 @@ mod tests {
}
}
+ #[test]
+ fn maintenance_start_delay_handles_never_overdue_and_remaining_cadence() {
+ let interval = Duration::from_secs(120);
+ let now = 1_000_000_000i64;
+ assert_eq!(
+ maintenance_start_delay(None, now, interval),
+ MAINTENANCE_STARTUP_DELAY_CAP
+ );
+ assert_eq!(
+ maintenance_start_delay(Some(now - 120_000_000), now, interval),
+ MAINTENANCE_STARTUP_DELAY_CAP
+ );
+ assert_eq!(
+ maintenance_start_delay(Some(now - 30_000_000), now, interval),
+ Duration::from_secs(90)
+ );
+ assert_eq!(
+ maintenance_start_delay(None, now, Duration::from_secs(10)),
+ Duration::from_secs(10)
+ );
+ assert_eq!(
+ maintenance_start_delay(Some(i64::MAX), now, interval),
+ interval,
+ "future/extreme persisted timestamps cannot defer beyond interval"
+ );
+ assert_eq!(
+ maintenance_start_delay(Some(i64::MIN), i64::MAX, interval),
+ MAINTENANCE_STARTUP_DELAY_CAP,
+ "opposite-sign extremes must not overflow"
+ );
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn persisted_maintenance_retries_failures_and_waits_after_success() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let attempts = Arc::new(AtomicUsize::new(0));
+ let (events, mut received) = tokio::sync::mpsc::unbounded_channel();
+ let attempts_for_tick = attempts.clone();
+ let reader_for_state = store.reader.clone();
+ let task = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ Duration::from_secs(2),
+ move || {
+ let reader = reader_for_state.clone();
+ async move {
+ Ok(reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::ForgetSweep)
+ .await?)
+ }
+ },
+ || jiff::Timestamp::now().as_microsecond(),
+ move || {
+ let attempts = attempts_for_tick.clone();
+ let events = events.clone();
+ async move {
+ let attempt = attempts.fetch_add(1, Ordering::SeqCst) + 1;
+ events.send(attempt).unwrap();
+ if attempt == 1 {
+ anyhow::bail!("injected failure");
+ }
+ Ok(())
+ }
+ },
+ ));
+
+ tokio::time::advance(Duration::from_secs(10)).await;
+ assert_eq!(received.recv().await, Some(1));
+ assert_eq!(
+ store
+ .reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::ForgetSweep)
+ .await
+ .unwrap(),
+ None,
+ "failed ticks must not advance persisted cadence"
+ );
+
+ tokio::time::advance(Duration::from_millis(1999)).await;
+ tokio::task::yield_now().await;
+ assert!(received.try_recv().is_err(), "failure retry is not early");
+ tokio::time::advance(Duration::from_millis(1)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(received.try_recv(), Ok(2));
+ let mut persisted = false;
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ persisted = store
+ .reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::ForgetSweep)
+ .await
+ .unwrap()
+ .is_some();
+ if persisted {
+ break;
+ }
+ }
+ assert!(persisted, "successful ticks persist cadence");
+
+ tokio::time::advance(Duration::from_secs(1)).await;
+ assert!(
+ received.try_recv().is_err(),
+ "next tick waits full interval"
+ );
+ tokio::time::advance(Duration::from_secs(1)).await;
+ assert_eq!(received.recv().await, Some(3));
+ task.abort();
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn maintenance_state_read_failure_retries_before_running_work() {
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let reads = Arc::new(AtomicUsize::new(0));
+ let work = Arc::new(AtomicUsize::new(0));
+ let reads_for_loader = reads.clone();
+ let work_for_tick = work.clone();
+ let task = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::RuleLint,
+ Duration::from_secs(2),
+ move || {
+ let reads = reads_for_loader.clone();
+ async move {
+ if reads.fetch_add(1, Ordering::SeqCst) == 0 {
+ anyhow::bail!("injected cadence read failure");
+ }
+ Ok(None)
+ }
+ },
+ || jiff::Timestamp::now().as_microsecond(),
+ move || {
+ let work = work_for_tick.clone();
+ async move {
+ work.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ }
+ },
+ ));
+
+ tokio::task::yield_now().await;
+ assert_eq!(reads.load(Ordering::SeqCst), 1);
+ assert_eq!(
+ work.load(Ordering::SeqCst),
+ 0,
+ "read failure must not run work"
+ );
+ tokio::time::advance(Duration::from_secs(2)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(reads.load(Ordering::SeqCst), 2);
+ assert_eq!(
+ work.load(Ordering::SeqCst),
+ 0,
+ "successful retry still observes startup delay"
+ );
+ tokio::time::advance(Duration::from_secs(2)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(work.load(Ordering::SeqCst), 1);
+ task.abort();
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn persisted_loop_runs_one_bounded_startup_catchup_for_never_and_overdue() {
+ const NOW: i64 = 10_000_000;
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (events, mut received) = tokio::sync::mpsc::unbounded_channel();
+ let never_events = events.clone();
+ let overdue_events = events.clone();
+ let never = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ Duration::from_secs(2),
+ || async { Ok(None) },
+ || NOW,
+ move || {
+ let events = never_events.clone();
+ async move {
+ events.send("never").unwrap();
+ Ok(())
+ }
+ },
+ ));
+ let overdue = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::RuleLint,
+ Duration::from_secs(2),
+ || async { Ok(Some(NOW - 2_000_000)) },
+ || NOW,
+ move || {
+ let events = overdue_events.clone();
+ async move {
+ events.send("overdue").unwrap();
+ Ok(())
+ }
+ },
+ ));
+ tokio::task::yield_now().await;
+ assert!(received.try_recv().is_err());
+ tokio::time::advance(Duration::from_secs(2)).await;
+ tokio::task::yield_now().await;
+ let mut seen = [false, false];
+ while let Ok(event) = received.try_recv() {
+ match event {
+ "never" => seen[0] = true,
+ "overdue" => seen[1] = true,
+ _ => unreachable!(),
+ }
+ }
+ assert_eq!(seen, [true, true]);
+ tokio::task::yield_now().await;
+ assert!(
+ received.try_recv().is_err(),
+ "no immediate repeat after catch-up"
+ );
+ never.abort();
+ overdue.abort();
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn persisted_loop_waits_exact_remaining_interval_when_not_due() {
+ const NOW: i64 = 20_000_000;
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (events, mut received) = tokio::sync::mpsc::unbounded_channel();
+ let task = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ Duration::from_secs(4),
+ || async { Ok(Some(NOW - 1_000_000)) },
+ || NOW,
+ move || {
+ let events = events.clone();
+ async move {
+ events.send(()).unwrap();
+ Ok(())
+ }
+ },
+ ));
+ tokio::task::yield_now().await;
+ tokio::time::advance(Duration::from_millis(2999)).await;
+ tokio::task::yield_now().await;
+ assert!(received.try_recv().is_err());
+ tokio::time::advance(Duration::from_millis(1)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(received.try_recv(), Ok(()));
+ task.abort();
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn persisted_loop_waits_after_long_tick_without_overlap() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let (events, mut received) = tokio::sync::mpsc::unbounded_channel();
+ let task = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::RuleLint,
+ Duration::from_secs(2),
+ || async { Ok(None) },
+ || 0,
+ move || {
+ let events = events.clone();
+ async move {
+ events.send("start").unwrap();
+ tokio::time::sleep(Duration::from_secs(3)).await;
+ events.send("end").unwrap();
+ Ok(())
+ }
+ },
+ ));
+ tokio::task::yield_now().await;
+ tokio::time::advance(Duration::from_secs(2)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(received.try_recv(), Ok("start"));
+ tokio::time::advance(Duration::from_secs(2)).await;
+ tokio::task::yield_now().await;
+ assert!(
+ received.try_recv().is_err(),
+ "second tick cannot overlap long first tick"
+ );
+ tokio::time::advance(Duration::from_secs(1)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(received.try_recv(), Ok("end"));
+ for _ in 0..20 {
+ tokio::task::yield_now().await;
+ if store
+ .reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::RuleLint)
+ .await
+ .unwrap()
+ .is_some()
+ {
+ break;
+ }
+ }
+ assert!(
+ store
+ .reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::RuleLint)
+ .await
+ .unwrap()
+ .is_some(),
+ "successful completion must be persisted before measuring the next interval"
+ );
+ tokio::task::yield_now().await;
+ tokio::time::advance(Duration::from_secs(1)).await;
+ tokio::task::yield_now().await;
+ assert!(
+ received.try_recv().is_err(),
+ "interval starts after completion"
+ );
+ tokio::time::advance(Duration::from_secs(1)).await;
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ assert_eq!(
+ received.try_recv(),
+ Ok("start"),
+ "next tick starts after the full post-completion interval"
+ );
+ task.abort();
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn persisted_loop_restart_waits_remaining_interval_from_real_state() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let interval = Duration::from_secs(4);
+ let (first_events, mut first_received) = tokio::sync::mpsc::unbounded_channel();
+ let first = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ interval,
+ || async { Ok(None) },
+ || 0,
+ move || {
+ let events = first_events.clone();
+ async move {
+ events.send(()).unwrap();
+ Ok(())
+ }
+ },
+ ));
+ for _ in 0..10 {
+ tokio::task::yield_now().await;
+ }
+ tokio::time::advance(interval).await;
+ tokio::task::yield_now().await;
+ assert_eq!(first_received.try_recv(), Ok(()));
+
+ let mut persisted = None;
+ for _ in 0..20 {
+ tokio::task::yield_now().await;
+ persisted = store
+ .reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::ForgetSweep)
+ .await
+ .unwrap();
+ if persisted.is_some() {
+ break;
+ }
+ }
+ let persisted = persisted.expect("first scheduler persisted successful completion");
+ first.abort();
+
+ // Restart one second into the persisted cadence: exactly three seconds remain.
+ let (second_events, mut second_received) = tokio::sync::mpsc::unbounded_channel();
+ let (state_loaded, mut state_loaded_received) = tokio::sync::mpsc::unbounded_channel();
+ let second_reader = store.reader.clone();
+ let second = tokio::spawn(run_persisted_maintenance_job(
+ store.writer.clone(),
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ interval,
+ move || {
+ let reader = second_reader.clone();
+ let state_loaded = state_loaded.clone();
+ async move {
+ let state = reader
+ .maintenance_job_last_success(ai_memory_store::MaintenanceJob::ForgetSweep)
+ .await?;
+ state_loaded.send(()).unwrap();
+ Ok(state)
+ }
+ },
+ move || persisted + 1_000_000,
+ move || {
+ let events = second_events.clone();
+ async move {
+ events.send(()).unwrap();
+ Ok(())
+ }
+ },
+ ));
+ assert_eq!(state_loaded_received.recv().await, Some(()));
+ tokio::time::advance(Duration::from_millis(2999)).await;
+ tokio::task::yield_now().await;
+ assert!(second_received.try_recv().is_err());
+ tokio::time::advance(Duration::from_millis(1)).await;
+ tokio::task::yield_now().await;
+ assert_eq!(second_received.try_recv(), Ok(()));
+ second.abort();
+ }
+
+ #[tokio::test]
+ async fn disabled_maintenance_creates_no_persisted_job_state() {
+ let (_tmp, store, wiki, _ws, _first, _second) = two_project_wiki().await;
+ let tasks = start_maintenance_scheduler(
+ MaintenanceSettings {
+ enabled: false,
+ forget_sweep_interval_secs: 1,
+ lint_interval_secs: 1,
+ embedding_backfill_interval_secs: 1,
+ },
+ AutoImproveSettings::default(),
+ store.reader.clone(),
+ store.writer.clone(),
+ wiki,
+ None,
+ None,
+ ai_memory_store::DecayParams::default(),
+ )
+ .await;
+ assert!(tasks.is_empty());
+ for job in [
+ ai_memory_store::MaintenanceJob::ForgetSweep,
+ ai_memory_store::MaintenanceJob::RuleLint,
+ ] {
+ assert_eq!(
+ store
+ .reader
+ .maintenance_job_last_success(job)
+ .await
+ .unwrap(),
+ None
+ );
+ }
+ }
+
+ #[tokio::test]
+ async fn zero_interval_omits_only_that_maintenance_job() {
+ for settings in [
+ MaintenanceSettings {
+ enabled: true,
+ forget_sweep_interval_secs: 0,
+ lint_interval_secs: 60,
+ embedding_backfill_interval_secs: 0,
+ },
+ MaintenanceSettings {
+ enabled: true,
+ forget_sweep_interval_secs: 60,
+ lint_interval_secs: 0,
+ embedding_backfill_interval_secs: 0,
+ },
+ ] {
+ let (_tmp, store, wiki, _ws, _first, _second) = two_project_wiki().await;
+ let tasks = start_maintenance_scheduler(
+ settings,
+ AutoImproveSettings::default(),
+ store.reader.clone(),
+ store.writer.clone(),
+ wiki,
+ None,
+ None,
+ ai_memory_store::DecayParams::default(),
+ )
+ .await;
+ // One enabled lint/sweep job plus the independent hollow-project job.
+ assert_eq!(tasks.len(), 2);
+ for task in tasks {
+ task.abort();
+ }
+ }
+ }
+
struct PanicLlm;
impl LlmProvider for PanicLlm {
@@ -2224,7 +2834,8 @@ mod tests {
.await;
}
- let outcome = run_scheduled_lint_tick(&store.reader, &wiki, None)
+ let panic_llm: Arc<dyn LlmProvider> = Arc::new(PanicLlm);
+ let outcome = run_scheduled_lint_tick(&store.reader, &wiki, Some(&panic_llm))
.await
.unwrap();
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index f3a85898..4b57ea37 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -535,9 +535,13 @@ impl Default for AutoImproveSettings {
pub struct MaintenanceSettings {
/// Master switch for scheduled jobs.
pub enabled: bool,
- /// Interval for the retention forget sweep. `0` disables this job.
+ /// Interval for the retention forget sweep. `0` disables this job. Successful
+ /// runs persist cadence across restarts; overdue work starts after a bounded
+ /// startup delay.
pub forget_sweep_interval_secs: u64,
- /// Interval for rule-based wiki lint. `0` disables this job.
+ /// Interval for rule-based wiki lint. `0` disables this job. Successful runs
+ /// persist cadence across restarts; overdue work starts after a bounded
+ /// startup delay.
pub lint_interval_secs: u64,
/// Interval for embedding backfill. `0` disables this job.
/// Defaults to off because it may call a paid provider.
diff --git a/crates/ai-memory-store/migrations/V29__maintenance_scheduler.sql b/crates/ai-memory-store/migrations/V29__maintenance_scheduler.sql
new file mode 100644
index 00000000..8ea615e0
--- /dev/null
+++ b/crates/ai-memory-store/migrations/V29__maintenance_scheduler.sql
@@ -0,0 +1,5 @@
+-- Global cadence state for maintenance jobs that operate across all scopes.
+CREATE TABLE maintenance_scheduler_state (
+ job TEXT PRIMARY KEY,
+ last_success_at INTEGER NOT NULL
+);
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index 631db471..f53fe6ac 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -16,6 +16,7 @@ mod auto_improve;
pub mod decay;
mod error;
mod fts_query;
+mod maintenance;
mod migrations;
mod ops;
mod reader;
@@ -34,6 +35,7 @@ pub use auto_improve::{
};
pub use decay::{DecayParams, retention_score};
pub use error::{StoreError, StoreResult};
+pub use maintenance::MaintenanceJob;
pub use ops::{DeleteWorkspaceSummary, EmbeddingWrite, MoveSummary, PurgeSummary, ReorgSummary};
pub use reader::{
ActivityWindow, AutoImproveCandidateSession, BriefPageBody, BriefingPage, BriefingSnapshot,
@@ -212,6 +214,45 @@ mod tests {
)
}
+ #[tokio::test]
+ async fn maintenance_scheduler_state_round_trips_per_job() {
+ let tmp = TempDir::new().unwrap();
+ {
+ let store = Store::open(tmp.path()).unwrap();
+ assert_eq!(
+ store
+ .reader
+ .maintenance_job_last_success(MaintenanceJob::ForgetSweep)
+ .await
+ .unwrap(),
+ None
+ );
+
+ store
+ .writer
+ .record_maintenance_job_success(MaintenanceJob::ForgetSweep)
+ .await
+ .unwrap();
+ }
+ let store = Store::open(tmp.path()).unwrap();
+ assert!(
+ store
+ .reader
+ .maintenance_job_last_success(MaintenanceJob::ForgetSweep)
+ .await
+ .unwrap()
+ .is_some()
+ );
+ assert_eq!(
+ store
+ .reader
+ .maintenance_job_last_success(MaintenanceJob::RuleLint)
+ .await
+ .unwrap(),
+ None
+ );
+ }
+
fn telemetry_count(rows: &[AutoImproveTelemetryCount], key: &str) -> usize {
rows.iter()
.find(|row| row.key == key)
diff --git a/crates/ai-memory-store/src/maintenance.rs b/crates/ai-memory-store/src/maintenance.rs
new file mode 100644
index 00000000..455ea7a5
--- /dev/null
+++ b/crates/ai-memory-store/src/maintenance.rs
@@ -0,0 +1,48 @@
+//! Persisted cadence state for global maintenance jobs.
+
+use jiff::Timestamp;
+use rusqlite::{Connection, OptionalExtension, params};
+
+use crate::error::StoreResult;
+
+/// Global maintenance jobs with persisted cadence. Embedding is intentionally
+/// absent: it remains opt-in and has no startup catch-up behavior.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum MaintenanceJob {
+ /// Retention forget sweep across every existing scope.
+ ForgetSweep,
+ /// Rule-based lint across every existing scope.
+ RuleLint,
+}
+
+impl MaintenanceJob {
+ /// Stable database key for this job.
+ #[must_use]
+ pub const fn as_str(self) -> &'static str {
+ match self {
+ Self::ForgetSweep => "forget_sweep",
+ Self::RuleLint => "rule_lint",
+ }
+ }
+}
+
+/// Return the last successful completion time for a maintenance job.
+pub fn last_success(conn: &Connection, job: MaintenanceJob) -> StoreResult<Option<i64>> {
+ conn.query_row(
+ "SELECT last_success_at FROM maintenance_scheduler_state WHERE job = ?1",
+ params![job.as_str()],
+ |row| row.get(0),
+ )
+ .optional()
+ .map_err(Into::into)
+}
+
+/// Record a job completion after its work has succeeded.
+pub fn record_success(conn: &Connection, job: MaintenanceJob) -> StoreResult<()> {
+ conn.execute(
+ "INSERT INTO maintenance_scheduler_state (job, last_success_at) VALUES (?1, ?2) \
+ ON CONFLICT(job) DO UPDATE SET last_success_at = excluded.last_success_at",
+ params![job.as_str(), Timestamp::now().as_microsecond()],
+ )?;
+ Ok(())
+}
diff --git a/crates/ai-memory-store/src/migrations.rs b/crates/ai-memory-store/src/migrations.rs
index 5d5677a0..21703868 100644
--- a/crates/ai-memory-store/src/migrations.rs
+++ b/crates/ai-memory-store/src/migrations.rs
@@ -96,7 +96,7 @@ mod tests {
fn schema_ahead_message_is_actionable() {
let rendered = StoreError::DataSchemaAhead {
applied: "V99 (future_feature)".to_string(),
- supported: 28,
+ supported: 29,
}
.to_string();
@@ -109,6 +109,36 @@ mod tests {
"{rendered}"
);
assert!(rendered.contains("V99 (future_feature)"), "{rendered}");
- assert!(rendered.contains("through V28"), "{rendered}");
+ assert!(rendered.contains("through V29"), "{rendered}");
+ }
+
+ #[test]
+ fn v28_to_v29_preserves_existing_rows() {
+ let mut conn = Connection::open_in_memory().unwrap();
+ run_to(&mut conn, 28).unwrap();
+ let workspace_id = [7_u8; 16];
+ conn.execute(
+ "INSERT INTO workspaces (id, name, created_at) VALUES (?1, 'existing', 1)",
+ params![workspace_id.as_slice()],
+ )
+ .unwrap();
+
+ run(&mut conn).unwrap();
+ let name: String = conn
+ .query_row(
+ "SELECT name FROM workspaces WHERE id = ?1",
+ params![workspace_id.as_slice()],
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(name, "existing");
+ let state_table: i64 = conn
+ .query_row(
+ "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'maintenance_scheduler_state'",
+ [],
+ |row| row.get(0),
+ )
+ .unwrap();
+ assert_eq!(state_table, 1);
}
}
diff --git a/crates/ai-memory-store/src/reader.rs b/crates/ai-memory-store/src/reader.rs
index f200f33c..2dd0741f 100644
--- a/crates/ai-memory-store/src/reader.rs
+++ b/crates/ai-memory-store/src/reader.rs
@@ -32,6 +32,7 @@ use crate::auto_improve::{
};
use crate::error::{StoreError, StoreResult};
use crate::fts_query::prepare_fts5_query;
+use crate::maintenance::MaintenanceJob;
use crate::users::TOKEN_HASH_LEN;
/// One hit returned by [`ReaderPool::search_pages`].
@@ -4337,6 +4338,15 @@ impl ReaderPool {
pub async fn users_exist(&self) -> StoreResult<bool> {
self.with_conn(crate::users::users_exist).await
}
+
+ /// Return the last successful global maintenance completion for `job`.
+ pub async fn maintenance_job_last_success(
+ &self,
+ job: MaintenanceJob,
+ ) -> StoreResult<Option<i64>> {
+ self.with_conn(move |conn| crate::maintenance::last_success(conn, job))
+ .await
+ }
}
/// Map a `(workspace, project, path, title, kind)` row to a [`HealthPage`].
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index 336918b8..b0543f1a 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -254,6 +254,10 @@ pub(crate) enum WriteCmd {
ended_at: i64,
reply: oneshot::Sender<StoreResult<bool>>,
},
+ RecordMaintenanceJobSuccess {
+ job: crate::maintenance::MaintenanceJob,
+ reply: oneshot::Sender<StoreResult<()>>,
+ },
Shutdown,
}
@@ -1013,6 +1017,17 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Persist a global maintenance job's successful completion time.
+ pub async fn record_maintenance_job_success(
+ &self,
+ job: crate::maintenance::MaintenanceJob,
+ ) -> StoreResult<()> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::RecordMaintenanceJobSuccess { job, reply: tx })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
async fn send(&self, cmd: WriteCmd) -> StoreResult<()> {
self.inner
.tx
@@ -1364,6 +1379,10 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
);
send_or_warn(reply, result, "claim_auto_improve_scheduler_session");
}
+ WriteCmd::RecordMaintenanceJobSuccess { job, reply } => {
+ let result = crate::maintenance::record_success(&conn, job);
+ send_or_warn(reply, result, "record_maintenance_job_success");
+ }
}
}
tracing::debug!("writer thread exiting cleanly");
diff --git a/docs/install.md b/docs/install.md
index 8398ffce..43d4271a 100644
--- a/docs/install.md
+++ b/docs/install.md
@@ -1148,6 +1148,12 @@ enable `embedding_backfill_interval_secs` after configuring an embedder,
each scheduled tick backfills every existing workspace/project and may
increase provider usage accordingly.
+Forget sweep and rule-based lint persist their last successful completion. On
+restart, a job that is not due waits only its remaining interval; a never-run
+or overdue job runs once after a bounded startup delay. Failed runs are not
+recorded as successful and retry after that bounded delay. Embedding backfill
+remains opt-in and keeps its interval-only behavior (no startup catch-up).
+
---
## Bootstrap mid-project {#bootstrap-mid-project}