Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(hooks): persist session-end consolidation

AkitaOnRails6b3aedb7a894
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a0dd9a2a..91c541e8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -30,6 +30,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
including subdirectories and linked worktrees. (#259)
### Fixed
+- Opt-in SessionEnd LLM consolidation now runs from a durable, generation-
+ idempotent queue instead of inside the hook batch request. The hook commits
+ its deterministic session page and handoff, persists the provider job, and
+ returns without waiting for LLM latency; a single bounded worker recovers
+ queued or expired-lease work after restart and makes at most five provider
+ attempts with backoff. A stale SessionEnd redelivery also repairs the
+ enqueue when the original request was cancelled just after `ended_at`, so
+ the default hook drain timeout can no longer silently strand the heuristic
+ page as the final result. (#260)
- `purge-project` no longer deletes a project out from under a running agent.
`workstreams` cascades from `projects` and `managed_runs` cascades from
`workstreams`, so purging a scope that still held a live managed run tore
diff --git a/README.md b/README.md
index 34630104..9099a9ba 100644
--- a/README.md
+++ b/README.md
@@ -654,6 +654,10 @@ when you want LLM consolidation (on PreCompact, on demand via
`memory_consolidate`, or opt-in at session end with
`AI_MEMORY_CONSOLIDATE_ON_SESSION_END`), richer linting, and bootstrap.
Session end always writes a rule-based summary page + handoff either way.
+When the session-end opt-in is enabled, provider work is durably queued after
+those deterministic writes and handled by one bounded server worker, so hook
+drain latency does not cancel it. Failed jobs retry with backoff and survive a
+server restart.
Recommended defaults:
diff --git a/crates/ai-memory-cli/src/commands/serve.rs b/crates/ai-memory-cli/src/commands/serve.rs
index b8ff0c52..4786b927 100644
--- a/crates/ai-memory-cli/src/commands/serve.rs
+++ b/crates/ai-memory-cli/src/commands/serve.rs
@@ -50,6 +50,12 @@ const MAX_BODY_BYTES: usize = 10 * 1024 * 1024;
const BOOTSTRAP_MAX_BODY_BYTES: usize = 32 * 1024 * 1024;
/// Startup and failure retry delay, capped by each job's configured interval.
const MAINTENANCE_STARTUP_DELAY_CAP: Duration = Duration::from_secs(60);
+/// How often the durable SessionEnd consolidation worker checks for work when
+/// no hook notification arrives (including jobs recovered after restart).
+const SESSION_CONSOLIDATION_POLL_INTERVAL: Duration = Duration::from_secs(15);
+/// A claimed job may be recovered after this long without completion. Provider
+/// requests are expected to finish well inside this lease.
+const SESSION_CONSOLIDATION_LEASE: Duration = Duration::from_secs(10 * 60);
/// Validate the credentials that keep an existing multi-user installation
/// closed. Bootstrap installs have no user rows yet and retain their historical
@@ -156,6 +162,127 @@ async fn run_persisted_maintenance_job<F, Fut, R, RFut, N>(
}
}
+fn session_consolidation_retry_delay(attempt: u32) -> Duration {
+ let exponent = attempt.saturating_sub(1).min(4);
+ Duration::from_secs(30_u64.saturating_mul(1_u64 << exponent))
+}
+
+async fn run_session_consolidation_worker(
+ writer: WriterHandle,
+ consolidator: Arc<Consolidator>,
+ notify: Arc<tokio::sync::Notify>,
+ cancel: CancellationToken,
+) {
+ loop {
+ let now = jiff::Timestamp::now().as_microsecond();
+ let lease_micros =
+ i64::try_from(SESSION_CONSOLIDATION_LEASE.as_micros()).unwrap_or(i64::MAX);
+ let job = match writer
+ .claim_session_consolidation(now, now.saturating_sub(lease_micros))
+ .await
+ {
+ Ok(job) => job,
+ Err(error) => {
+ tracing::warn!(%error, "SessionEnd consolidation queue claim failed");
+ tokio::select! {
+ () = cancel.cancelled() => return,
+ () = notify.notified() => {},
+ () = tokio::time::sleep(SESSION_CONSOLIDATION_POLL_INTERVAL) => {},
+ }
+ continue;
+ }
+ };
+
+ let Some(job) = job else {
+ tokio::select! {
+ () = cancel.cancelled() => return,
+ () = notify.notified() => {},
+ () = tokio::time::sleep(SESSION_CONSOLIDATION_POLL_INTERVAL) => {},
+ }
+ continue;
+ };
+
+ let session_id = job.session_id();
+ let generation = job.generation();
+ let attempts = job.attempts();
+ let consolidation = consolidator.consolidate_session(
+ session_id,
+ false,
+ ai_memory_core::ActorContext::anonymous(),
+ None,
+ );
+ tokio::pin!(consolidation);
+ let result = tokio::select! {
+ () = cancel.cancelled() => {
+ if let Err(error) = writer.release_session_consolidation(job).await {
+ tracing::warn!(%error, %session_id, generation, "failed to release SessionEnd consolidation during shutdown");
+ }
+ return;
+ }
+ result = &mut consolidation => result,
+ };
+
+ match result {
+ Ok(outcome) => match writer.complete_session_consolidation(job).await {
+ Ok(()) => info!(
+ session = %session_id,
+ generation,
+ attempts,
+ path = %outcome.path,
+ "SessionEnd: queued LLM consolidation written (opt-in)",
+ ),
+ Err(error) => tracing::warn!(
+ %error,
+ session = %session_id,
+ generation,
+ "SessionEnd consolidation succeeded but queue completion failed",
+ ),
+ },
+ Err(error) => {
+ let retry_at = if attempts < ai_memory_store::SESSION_CONSOLIDATION_MAX_ATTEMPTS {
+ let delay = session_consolidation_retry_delay(attempts);
+ let delay_micros = i64::try_from(delay.as_micros()).unwrap_or(i64::MAX);
+ Some(
+ jiff::Timestamp::now()
+ .as_microsecond()
+ .saturating_add(delay_micros),
+ )
+ } else {
+ None
+ };
+ let terminal = retry_at.is_none();
+ if let Err(store_error) = writer
+ .fail_session_consolidation(job, error.to_string(), retry_at)
+ .await
+ {
+ tracing::warn!(
+ error = %store_error,
+ session = %session_id,
+ generation,
+ "failed to persist SessionEnd consolidation failure",
+ );
+ } else if terminal {
+ tracing::error!(
+ %error,
+ session = %session_id,
+ generation,
+ attempts,
+ "SessionEnd LLM consolidation exhausted retries; heuristic page remains",
+ );
+ } else {
+ tracing::warn!(
+ %error,
+ session = %session_id,
+ generation,
+ attempts,
+ "SessionEnd LLM consolidation failed; queued for retry",
+ );
+ }
+ }
+ }
+ }
+}
+
/// Run the `serve` subcommand.
///
/// # Errors
@@ -319,6 +446,27 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
TransportKind::Http => {
let bind = args.bind.unwrap_or_else(|| config.bind.clone());
let cancel = CancellationToken::new();
+ let (session_consolidation_notify, session_consolidation_task) =
+ if config.consolidate_on_session_end {
+ if let Some(consolidator) = consolidator.clone() {
+ let notify = Arc::new(tokio::sync::Notify::new());
+ let task = tokio::spawn(run_session_consolidation_worker(
+ store.writer.clone(),
+ consolidator,
+ notify.clone(),
+ cancel.child_token(),
+ ));
+ info!(
+ max_attempts = ai_memory_store::SESSION_CONSOLIDATION_MAX_ATTEMPTS,
+ "durable SessionEnd consolidation worker started"
+ );
+ (Some(notify), Some(task))
+ } else {
+ (None, None)
+ }
+ } else {
+ (None, None)
+ };
let server_clone = server.clone();
// `Host`-header allowlist for the HTTP DNS-rebinding guard.
// Sourced from Config (which already handles the
@@ -388,6 +536,7 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
)),
ingest_gates: ai_memory_hooks::IngestGates::default(),
consolidate_on_session_end: config.consolidate_on_session_end,
+ session_consolidation_notify,
capture_assistant_enabled: config.capture_assistant,
subagent_sessions: std::sync::Arc::new(tokio::sync::Mutex::new(
ai_memory_hooks::SubagentSessionSet::default(),
@@ -566,13 +715,21 @@ pub async fn run(config: &Config, args: ServeArgs) -> Result<()> {
docs/https-via-proxy.md for copy-paste templates."
);
}
- axum::serve(listener, router)
+ let shutdown_cancel = cancel.clone();
+ let serve_result = axum::serve(listener, router)
.with_graceful_shutdown(async move {
let _ = tokio::signal::ctrl_c().await;
info!("ctrl-c received; shutting down");
- cancel.cancel();
+ shutdown_cancel.cancel();
})
- .await?;
+ .await;
+ cancel.cancel();
+ if let Some(task) = session_consolidation_task
+ && let Err(error) = task.await
+ {
+ tracing::warn!(%error, "SessionEnd consolidation worker join failed");
+ }
+ serve_result?;
}
}
Ok(())
@@ -1265,7 +1422,10 @@ fn host_without_port(host: &str) -> &str {
#[cfg(test)]
mod tests {
use super::*;
- use ai_memory_core::{PagePath, Tier};
+ use ai_memory_core::{
+ AgentKind, NewObservation, NewSession, ObservationKind, PagePath, Sanitized, Sanitizer,
+ SessionId, Tier,
+ };
use ai_memory_llm::{ChatRequest, ChatResponse, LlmResult, SyntheticEmbedder};
use ai_memory_wiki::WritePageRequest;
use axum::http::Request;
@@ -1798,6 +1958,155 @@ mod tests {
}
}
+ struct SuccessfulConsolidationLlm;
+
+ impl LlmProvider for SuccessfulConsolidationLlm {
+ fn name(&self) -> &'static str {
+ "successful-consolidation"
+ }
+
+ fn model(&self) -> &str {
+ "test"
+ }
+
+ fn complete<'life0, 'async_trait>(
+ &'life0 self,
+ _request: ChatRequest,
+ ) -> Pin<Box<dyn Future<Output = LlmResult<ChatResponse>> + Send + 'async_trait>>
+ where
+ 'life0: 'async_trait,
+ Self: 'async_trait,
+ {
+ Box::pin(async move {
+ Ok(ChatResponse {
+ text: "unused".into(),
+ usage: None,
+ model: "test".into(),
+ })
+ })
+ }
+
+ fn complete_structured_raw<'life0, 'async_trait>(
+ &'life0 self,
+ _request: ChatRequest,
+ _schema: serde_json::Value,
+ ) -> Pin<Box<dyn Future<Output = LlmResult<serde_json::Value>> + Send + 'async_trait>>
+ where
+ 'life0: 'async_trait,
+ Self: 'async_trait,
+ {
+ Box::pin(async move {
+ Ok(serde_json::json!({
+ "title": "Compiled session",
+ "body_markdown": "Durable worker completed this session.",
+ "tags": ["test"]
+ }))
+ })
+ }
+ }
+
+ #[tokio::test]
+ async fn session_consolidation_worker_consumes_and_completes_durable_job() {
+ let tmp = TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let workspace_id = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let project_id = store
+ .writer
+ .get_or_create_project(workspace_id, "project", None)
+ .await
+ .unwrap();
+ let session_id = SessionId::new();
+ store
+ .writer
+ .begin_session(NewSession {
+ id: session_id,
+ workspace_id,
+ project_id,
+ agent_kind: AgentKind::Codex,
+ cwd: None,
+ })
+ .await
+ .unwrap();
+ store
+ .writer
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id,
+ project_id,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "finish".into(),
+ body: "complete the durable job".into(),
+ importance: 8,
+ },
+ &Sanitizer::default(),
+ ))
+ .await
+ .unwrap();
+ store.writer.end_session(session_id, None).await.unwrap();
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap();
+
+ let wiki = Wiki::new(tmp.path(), store.writer.clone()).unwrap();
+ let consolidator = Arc::new(Consolidator::new(
+ store.reader.clone(),
+ store.writer.clone(),
+ wiki,
+ Arc::new(SuccessfulConsolidationLlm),
+ workspace_id,
+ project_id,
+ ));
+ let notify = Arc::new(tokio::sync::Notify::new());
+ let cancel = CancellationToken::new();
+ let task = tokio::spawn(run_session_consolidation_worker(
+ store.writer.clone(),
+ consolidator,
+ notify.clone(),
+ cancel.child_token(),
+ ));
+ notify.notify_one();
+
+ let path = format!("sessions/{session_id}.md");
+ tokio::time::timeout(Duration::from_secs(5), async {
+ loop {
+ if store
+ .reader
+ .page_body_by_ids(workspace_id, project_id, &path)
+ .await
+ .unwrap()
+ .is_some_and(|page| page.body.contains("Durable worker completed"))
+ {
+ break;
+ }
+ tokio::time::sleep(Duration::from_millis(10)).await;
+ }
+ })
+ .await
+ .expect("worker should consume the queued job");
+
+ cancel.cancel();
+ task.await.unwrap();
+ let now = jiff::Timestamp::now().as_microsecond();
+ assert!(
+ store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .is_none(),
+ "completed work must not be claimed again"
+ );
+ }
+
async fn two_project_wiki() -> (TempDir, Store, Wiki, WorkspaceId, ProjectId, ProjectId) {
let tmp = TempDir::new().unwrap();
let store = Store::open(tmp.path()).unwrap();
diff --git a/crates/ai-memory-cli/src/config.rs b/crates/ai-memory-cli/src/config.rs
index 8323064f..6e061b7d 100644
--- a/crates/ai-memory-cli/src/config.rs
+++ b/crates/ai-memory-cli/src/config.rs
@@ -91,9 +91,10 @@ pub struct Config {
pub llm_compat_strict: bool,
/// Opt-in: run LLM consolidation on SessionEnd (in addition to the
/// always-written heuristic session page), when an LLM provider is
- /// configured. Off by default — SessionEnd stays cheap and
- /// fire-and-forget; the LLM checkpoint otherwise happens on PreCompact
- /// and via manual `memory_consolidate`. Set with
+ /// configured. Off by default. Provider work is durably queued after the
+ /// deterministic session page and handoff, then handled outside the hook
+ /// response by one bounded retrying worker. The LLM checkpoint otherwise
+ /// happens on PreCompact and via manual `memory_consolidate`. Set with
/// `AI_MEMORY_CONSOLIDATE_ON_SESSION_END=true`.
pub consolidate_on_session_end: bool,
/// Server-side opt-in for assistant/Stop capture (#196). When true, the
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index 4b9051be..440370d2 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -453,6 +453,9 @@ pub struct HookState {
/// session close stays cheap; the LLM checkpoint otherwise happens on
/// PreCompact and via manual `memory_consolidate`.
pub consolidate_on_session_end: bool,
+ /// Coalescing wake-up for the durable SessionEnd consolidation worker.
+ /// The database is the queue; notifications only reduce pickup latency.
+ pub session_consolidation_notify: Option<Arc<tokio::sync::Notify>>,
/// Opt-in (`AI_MEMORY_CAPTURE_ASSISTANT`): when true, the server honors the
/// client's `_ai_memory_assistant` protocol on a `Stop` event and persists
/// the sanitized excerpt as the Stop body. Off by default; when off the
@@ -1850,6 +1853,35 @@ async fn process_envelope(state: Arc<HookState>, env: HookEnvelope, actor_user:
}
}
+async fn enqueue_session_end_consolidation(
+ state: &HookState,
+ session_id: SessionId,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+) -> anyhow::Result<()> {
+ if !state.consolidate_on_session_end || state.consolidator.is_none() {
+ return Ok(());
+ }
+ let Some(notify) = state.session_consolidation_notify.as_ref() else {
+ warn!(
+ session = %session_id,
+ "SessionEnd LLM consolidation enabled without a queue worker"
+ );
+ return Ok(());
+ };
+ let inserted = state
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await?;
+ notify.notify_one();
+ debug!(
+ session = %session_id,
+ inserted,
+ "SessionEnd LLM consolidation queued"
+ );
+ Ok(())
+}
+
async fn process(
state: &HookState,
env: HookEnvelope,
@@ -1938,6 +1970,11 @@ async fn process(
{
ai_memory_store::SessionEndDisposition::Open => {}
ai_memory_store::SessionEndDisposition::DropStale => {
+ // A first delivery can be cancelled after `end_session` but
+ // before its durable LLM job is queued. The replay must ensure
+ // that job exists before acknowledging the already-ended
+ // session; enqueue is generation-idempotent.
+ enqueue_session_end_consolidation(state, session_id, ws, proj).await?;
info!(
session = %session_id,
agent = %env.agent.as_str(),
@@ -2158,35 +2195,6 @@ async fn process(
);
Some(state.writer.insert_handoff(handoff).await?)
};
- // Opt-in (AI_MEMORY_CONSOLIDATE_ON_SESSION_END): additionally run LLM
- // consolidation so the session's knowledge is compiled into topical
- // pages, not just the heuristic session record. The heuristic page
- // above is always written first, so an LLM failure here is non-fatal —
- // warn and keep the deterministic result. Runs before the commit so
- // the consolidated pages land in the same atomic snapshot.
- if state.consolidate_on_session_end
- && let Some(c) = state.consolidator.as_ref()
- {
- match c
- .consolidate_session(
- session_id,
- false,
- ai_memory_core::ActorContext::anonymous(),
- None,
- )
- .await
- {
- Ok(outcome) => info!(
- session = %session_id,
- path = %outcome.path,
- "SessionEnd: LLM consolidation written (opt-in)",
- ),
- Err(e) => warn!(
- error = %e,
- "SessionEnd LLM consolidation failed; heuristic page already written",
- ),
- }
- }
// Auto-commit the wiki tree so the session/handoff/log.md
// changes land in git in one atomic snapshot.
let commit_msg = format!(
@@ -2199,6 +2207,11 @@ async fn process(
Ok(None) => debug!("wiki clean; no auto-commit"),
Err(e) => warn!(error = %e, "auto-commit failed"),
}
+ // Persist the optional LLM work before acknowledging the hook, after
+ // deterministic wiki writes are committed so the worker cannot race
+ // their git snapshot. Stale redelivery above repairs cancellation in
+ // the narrow window after `end_session`.
+ enqueue_session_end_consolidation(state, session_id, ws, proj).await?;
if let Some(handoff_id) = handoff_id {
info!(
session = %session_id,
@@ -2486,6 +2499,7 @@ mod tests {
project_cache: Arc::new(tokio::sync::Mutex::new(ProjectCacheStore::default())),
active_project: ActiveProject::new(),
consolidate_on_session_end: false,
+ session_consolidation_notify: None,
capture_assistant_enabled: false,
subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),
ingest_rate: Arc::new(tokio::sync::Mutex::new(IngestRateLimiter::disabled())),
@@ -5362,6 +5376,104 @@ mod tests {
);
}
+ #[tokio::test]
+ async fn session_end_queues_llm_work_without_waiting_for_provider() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ let llm = Arc::new(RecordingLlm(Mutex::new(None)));
+ state.consolidator = Some(Arc::new(Consolidator::new(
+ state.reader.clone(),
+ state.writer.clone(),
+ state.wiki.clone(),
+ llm.clone(),
+ state.workspace_id,
+ state.project_id,
+ )));
+ state.consolidate_on_session_end = true;
+ state.session_consolidation_notify = Some(Arc::new(tokio::sync::Notify::new()));
+ let sid = "10101010-1010-1010-1010-101010101010";
+ let fire = |event: &str| {
+ HookEnvelope::from_query_and_body(
+ HookQuery {
+ event: event.into(),
+ agent: Some("codex".into()),
+ ..Default::default()
+ },
+ serde_json::json!({ "session_id": sid, "prompt": "finish" }),
+ )
+ };
+ process(&state, fire("user-prompt-submit"), None)
+ .await
+ .unwrap();
+ tokio::time::timeout(
+ std::time::Duration::from_secs(1),
+ process(&state, fire("session-end"), None),
+ )
+ .await
+ .expect("SessionEnd must not wait for provider I/O")
+ .unwrap();
+
+ assert!(
+ llm.0.lock().unwrap().is_none(),
+ "the hook path must leave provider work to the queue worker"
+ );
+ let now = Timestamp::now().as_microsecond();
+ let job = state
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .expect("SessionEnd persisted a consolidation job");
+ assert_eq!(job.session_id(), sid.parse().unwrap());
+ assert_eq!(job.generation(), 2);
+ }
+
+ #[tokio::test]
+ async fn stale_session_end_redelivery_repairs_missing_queue_job() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ let llm = Arc::new(RecordingLlm(Mutex::new(None)));
+ state.consolidator = Some(Arc::new(Consolidator::new(
+ state.reader.clone(),
+ state.writer.clone(),
+ state.wiki.clone(),
+ llm,
+ state.workspace_id,
+ state.project_id,
+ )));
+ state.consolidate_on_session_end = true;
+ state.session_consolidation_notify = Some(Arc::new(tokio::sync::Notify::new()));
+ let sid = "20202020-2020-2020-2020-202020202020";
+ let session_id: SessionId = sid.parse().unwrap();
+ let fire = |event: &str| {
+ HookEnvelope::from_query_and_body(
+ HookQuery {
+ event: event.into(),
+ agent: Some("codex".into()),
+ ..Default::default()
+ },
+ serde_json::json!({ "session_id": sid, "prompt": "finish" }),
+ )
+ };
+ process(&state, fire("user-prompt-submit"), None)
+ .await
+ .unwrap();
+ // Reproduce the persisted boundary from #260: the original request
+ // reached end_session, then was cancelled before it could enqueue.
+ state.writer.end_session(session_id, None).await.unwrap();
+
+ process(&state, fire("session-end"), None).await.unwrap();
+ let now = Timestamp::now().as_microsecond();
+ let job = state
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .expect("stale redelivery must restore the missing durable job");
+ assert_eq!(job.session_id(), session_id);
+ assert_eq!(job.generation(), 1);
+ }
+
// Issue #152: an agent that resumes an ended session under the same id
// and keeps working must get its page re-compiled by the second
// SessionEnd instead of that end being dropped as "already-ended".
diff --git a/crates/ai-memory-mcp/tests/autoscope_multiuser.rs b/crates/ai-memory-mcp/tests/autoscope_multiuser.rs
index e551595b..7314fede 100644
--- a/crates/ai-memory-mcp/tests/autoscope_multiuser.rs
+++ b/crates/ai-memory-mcp/tests/autoscope_multiuser.rs
@@ -159,6 +159,7 @@ impl MultiUserHarness {
)),
ingest_gates: ai_memory_hooks::IngestGates::default(),
consolidate_on_session_end: false,
+ session_consolidation_notify: None,
capture_assistant_enabled: false,
subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),
ingest_rate: Arc::new(tokio::sync::Mutex::new(
diff --git a/crates/ai-memory-mcp/tests/autoscope_stress.rs b/crates/ai-memory-mcp/tests/autoscope_stress.rs
index 9ece92c4..406d3038 100644
--- a/crates/ai-memory-mcp/tests/autoscope_stress.rs
+++ b/crates/ai-memory-mcp/tests/autoscope_stress.rs
@@ -158,6 +158,7 @@ impl Harness {
)),
ingest_gates: ai_memory_hooks::IngestGates::default(),
consolidate_on_session_end: false,
+ session_consolidation_notify: None,
capture_assistant_enabled: false,
subagent_sessions: Arc::new(tokio::sync::Mutex::new(SubagentSessionSet::default())),
ingest_rate: Arc::new(tokio::sync::Mutex::new(
diff --git a/crates/ai-memory-store/migrations/V34__session_consolidation_jobs.sql b/crates/ai-memory-store/migrations/V34__session_consolidation_jobs.sql
new file mode 100644
index 00000000..2ced9273
--- /dev/null
+++ b/crates/ai-memory-store/migrations/V34__session_consolidation_jobs.sql
@@ -0,0 +1,32 @@
+-- Durable work queue for opt-in SessionEnd LLM consolidation. The hook path
+-- only enqueues a job; one bounded server worker claims jobs outside the HTTP
+-- request lifetime and retries transient failures. `generation` is the number
+-- of observations present when the session was ended, so resumed sessions can
+-- be consolidated again without duplicating work for the same observation set.
+CREATE TABLE session_consolidation_jobs (
+ session_id BLOB NOT NULL REFERENCES sessions(id) ON DELETE CASCADE,
+ workspace_id BLOB NOT NULL REFERENCES workspaces(id) ON DELETE CASCADE,
+ project_id BLOB NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+ generation INTEGER NOT NULL CHECK (generation > 0),
+ state TEXT NOT NULL CHECK (state IN ('pending', 'running', 'completed', 'failed', 'superseded')),
+ requested_at INTEGER NOT NULL,
+ next_attempt_at INTEGER NOT NULL,
+ started_at INTEGER,
+ completed_at INTEGER,
+ attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0),
+ claim_id BLOB,
+ last_error TEXT,
+ PRIMARY KEY (session_id, generation)
+) WITHOUT ROWID;
+
+CREATE INDEX idx_session_consolidation_jobs_due
+ ON session_consolidation_jobs (state, next_attempt_at, requested_at);
+
+CREATE TRIGGER session_consolidation_jobs_session_pairing_ai
+BEFORE INSERT ON session_consolidation_jobs
+FOR EACH ROW
+WHEN NEW.workspace_id IS NOT (SELECT workspace_id FROM sessions WHERE id = NEW.session_id)
+ OR NEW.project_id IS NOT (SELECT project_id FROM sessions WHERE id = NEW.session_id)
+BEGIN
+ SELECT RAISE(ABORT, 'session_consolidation_jobs scope does not match the session scope');
+END;
diff --git a/crates/ai-memory-store/src/lib.rs b/crates/ai-memory-store/src/lib.rs
index ced1ada8..47370ca4 100644
--- a/crates/ai-memory-store/src/lib.rs
+++ b/crates/ai-memory-store/src/lib.rs
@@ -21,6 +21,7 @@ mod migrations;
mod ops;
mod reader;
mod scope;
+mod session_consolidation;
pub mod users;
mod workstream;
mod writer;
@@ -55,6 +56,7 @@ pub use scope::{
create_explicit_scope, create_global_scope, lookup_existing_scope, lookup_existing_workspace,
lookup_global_scope, resolve_many_existing_scopes,
};
+pub use session_consolidation::{SESSION_CONSOLIDATION_MAX_ATTEMPTS, SessionConsolidationJob};
pub use users::{TOKEN_HASH_LEN, TOKEN_RAW_LEN, TokenPepper, generate_token, hash_token};
pub use workstream::{
FinishWorkstreamRun, FinishedWorkstreamRun, ManagedRunContext, PrepareWorkstreamRun,
diff --git a/crates/ai-memory-store/src/migrations.rs b/crates/ai-memory-store/src/migrations.rs
index 131be7d9..1a257666 100644
--- a/crates/ai-memory-store/src/migrations.rs
+++ b/crates/ai-memory-store/src/migrations.rs
@@ -52,6 +52,7 @@ pub(crate) fn run_to(conn: &mut rusqlite::Connection, target: u32) -> Result<(),
#[cfg(test)]
mod tests {
use super::*;
+ use ai_memory_core::{AgentKind, NewObservation, NewSession, ObservationKind, SessionId};
use rusqlite::{Connection, params};
/// A store migrated by a newer build (an applied version above anything
@@ -235,4 +236,63 @@ mod tests {
.unwrap();
assert_eq!(state_table, 1);
}
+
+ #[test]
+ fn v33_to_v34_preserves_sessions_and_activates_durable_queue() {
+ let mut conn = Connection::open_in_memory().unwrap();
+ run_to(&mut conn, 33).unwrap();
+ conn.pragma_update(None, "foreign_keys", "ON").unwrap();
+ let workspace_id = crate::ops::get_or_create_workspace(&mut conn, "default").unwrap();
+ let project_id =
+ crate::ops::get_or_create_project(&mut conn, &workspace_id, "project", None).unwrap();
+ let session_id = SessionId::new();
+ crate::ops::begin_session(
+ &mut conn,
+ &NewSession {
+ id: session_id,
+ workspace_id,
+ project_id,
+ agent_kind: AgentKind::Codex,
+ cwd: None,
+ },
+ )
+ .unwrap();
+ crate::ops::insert_observation(
+ &mut conn,
+ &NewObservation {
+ session_id,
+ workspace_id,
+ project_id,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "continue".into(),
+ importance: 8,
+ },
+ )
+ .unwrap();
+ crate::ops::end_session(&mut conn, &session_id, None).unwrap();
+
+ conn.pragma_update(None, "foreign_keys", "OFF").unwrap();
+ run(&mut conn).unwrap();
+ conn.pragma_update(None, "foreign_keys", "ON").unwrap();
+ assert_eq!(schema_object_count(&conn, "table", "sessions"), 1);
+ assert_eq!(
+ schema_object_count(&conn, "table", "session_consolidation_jobs"),
+ 1
+ );
+ assert_eq!(
+ schema_object_count(
+ &conn,
+ "trigger",
+ "session_consolidation_jobs_session_pairing_ai"
+ ),
+ 1
+ );
+ assert!(
+ crate::session_consolidation::enqueue(&mut conn, workspace_id, project_id, session_id,)
+ .unwrap()
+ );
+ }
}
diff --git a/crates/ai-memory-store/src/ops.rs b/crates/ai-memory-store/src/ops.rs
index 60340ba8..d69343ab 100644
--- a/crates/ai-memory-store/src/ops.rs
+++ b/crates/ai-memory-store/src/ops.rs
@@ -1782,6 +1782,8 @@ pub struct MoveSummary {
pub auto_improve_scheduler_state_moved: u64,
/// `auto_improve_scheduler_claims` rows re-stamped.
pub auto_improve_scheduler_claims_moved: u64,
+ /// Durable SessionEnd consolidation jobs re-stamped.
+ pub session_consolidation_jobs_moved: u64,
}
/// Re-stamp a project's `workspace_id` across every domain table in ONE
@@ -1851,6 +1853,10 @@ pub fn move_project_workspace(
"UPDATE auto_improve_scheduler_claims SET workspace_id = ?1 WHERE project_id = ?2 AND workspace_id = ?3",
params![&to[..], &pid[..], &from[..]],
)? as u64;
+ let session_consolidation_jobs_moved = tx.execute(
+ "UPDATE session_consolidation_jobs SET workspace_id = ?1 WHERE project_id = ?2 AND workspace_id = ?3",
+ params![&to[..], &pid[..], &from[..]],
+ )? as u64;
let projects_updated = tx.execute(
"UPDATE projects SET workspace_id = ?1 WHERE id = ?2 AND workspace_id = ?3",
@@ -1873,6 +1879,7 @@ pub fn move_project_workspace(
auto_improve_proposals_moved,
auto_improve_scheduler_state_moved,
auto_improve_scheduler_claims_moved,
+ session_consolidation_jobs_moved,
})
}
@@ -3134,11 +3141,14 @@ mod tests {
},
)
.unwrap();
+ end_session(&mut conn, &sid, None).unwrap();
+ crate::session_consolidation::enqueue(&mut conn, src_ws, proj, sid).unwrap();
let summary = move_project_workspace(&mut conn, &proj, &src_ws, &dst_ws).unwrap();
assert_eq!(summary.pages_moved, 1);
assert_eq!(summary.sessions_moved, 1);
assert_eq!(summary.observations_moved, 1);
+ assert_eq!(summary.session_consolidation_jobs_moved, 1);
// The project_id is unchanged; every row now points at dst_ws.
// `projects` keys the project by `id`; child tables by `project_id`.
@@ -3155,7 +3165,13 @@ mod tests {
)
.unwrap()
};
- for table in ["projects", "pages", "sessions", "observations"] {
+ for table in [
+ "projects",
+ "pages",
+ "sessions",
+ "observations",
+ "session_consolidation_jobs",
+ ] {
assert_eq!(count_in(table, &dst_ws), 1, "{table} must move to dst ws");
assert_eq!(count_in(table, &src_ws), 0, "{table} must leave src ws");
}
diff --git a/crates/ai-memory-store/src/session_consolidation.rs b/crates/ai-memory-store/src/session_consolidation.rs
new file mode 100644
index 00000000..7454e9e4
--- /dev/null
+++ b/crates/ai-memory-store/src/session_consolidation.rs
@@ -0,0 +1,626 @@
+//! Durable queue state for opt-in SessionEnd LLM consolidation.
+
+use ai_memory_core::{ProjectId, SessionId, WorkspaceId};
+use jiff::Timestamp;
+use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
+use uuid::Uuid;
+
+use crate::error::{StoreError, StoreResult};
+
+/// Maximum provider attempts for one immutable observation generation.
+pub const SESSION_CONSOLIDATION_MAX_ATTEMPTS: u32 = 5;
+
+/// One atomically claimed session-consolidation job.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SessionConsolidationJob {
+ session_id: SessionId,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ generation: u64,
+ attempts: u32,
+ claim_id: Uuid,
+ attempt_spent: bool,
+}
+
+impl SessionConsolidationJob {
+ /// Session whose observations must be consolidated.
+ #[must_use]
+ pub const fn session_id(&self) -> SessionId {
+ self.session_id
+ }
+
+ /// Owning workspace.
+ #[must_use]
+ pub const fn workspace_id(&self) -> WorkspaceId {
+ self.workspace_id
+ }
+
+ /// Owning project.
+ #[must_use]
+ pub const fn project_id(&self) -> ProjectId {
+ self.project_id
+ }
+
+ /// Observation count covered by this immutable job.
+ #[must_use]
+ pub const fn generation(&self) -> u64 {
+ self.generation
+ }
+
+ /// One-based provider attempt number.
+ #[must_use]
+ pub const fn attempts(&self) -> u32 {
+ self.attempts
+ }
+}
+
+/// Enqueue the current observation generation for an ended session.
+///
+/// Returns `true` only when a new generation was inserted. A duplicate end,
+/// missing session, mismatched scope, open session, or observation-less session
+/// returns `false` without creating work.
+pub fn enqueue(
+ conn: &mut Connection,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ session_id: SessionId,
+) -> StoreResult<bool> {
+ let now = Timestamp::now().as_microsecond();
+ let tx = conn.transaction()?;
+ let inserted = tx.execute(
+ "INSERT OR IGNORE INTO session_consolidation_jobs \
+ (session_id, workspace_id, project_id, generation, state, requested_at, \
+ next_attempt_at, started_at, completed_at, attempts, claim_id, last_error) \
+ SELECT s.id, s.workspace_id, s.project_id, COUNT(o.id), 'pending', ?4, \
+ ?4, NULL, NULL, 0, NULL, NULL \
+ FROM sessions s \
+ JOIN observations o ON o.session_id = s.id \
+ WHERE s.id = ?1 AND s.workspace_id = ?2 AND s.project_id = ?3 \
+ AND s.ended_at IS NOT NULL \
+ GROUP BY s.id, s.workspace_id, s.project_id \
+ HAVING COUNT(o.id) > 0",
+ params![
+ session_id.as_bytes(),
+ workspace_id.as_bytes(),
+ project_id.as_bytes(),
+ now,
+ ],
+ )?;
+ // A newer pending generation subsumes an older one because consolidation
+ // reads the session's current observations. Running work keeps its lease:
+ // it may already have read the older snapshot, so the new generation must
+ // still run afterward.
+ tx.execute(
+ "UPDATE session_consolidation_jobs \
+ SET state = 'superseded', completed_at = ?1, claim_id = NULL, \
+ last_error = 'superseded by a newer observation generation' \
+ WHERE session_id = ?2 AND state = 'pending' \
+ AND generation < ( \
+ SELECT COUNT(*) FROM observations WHERE session_id = ?2 \
+ )",
+ params![now, session_id.as_bytes()],
+ )?;
+ tx.commit()?;
+ Ok(inserted == 1)
+}
+
+/// Claim the oldest due job, including a running job whose lease expired.
+pub fn claim_next(
+ conn: &mut Connection,
+ now: i64,
+ stale_before: i64,
+) -> StoreResult<Option<SessionConsolidationJob>> {
+ let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
+ let row = tx
+ .query_row(
+ "SELECT session_id, workspace_id, project_id, generation, attempts, state \
+ FROM session_consolidation_jobs \
+ WHERE (state = 'pending' AND attempts < ?1 AND next_attempt_at <= ?2) \
+ OR (state = 'running' AND attempts <= ?1 AND started_at <= ?3) \
+ ORDER BY requested_at ASC, generation ASC \
+ LIMIT 1",
+ params![
+ i64::from(SESSION_CONSOLIDATION_MAX_ATTEMPTS),
+ now,
+ stale_before
+ ],
+ |row| {
+ Ok((
+ row.get::<_, Vec<u8>>(0)?,
+ row.get::<_, Vec<u8>>(1)?,
+ row.get::<_, Vec<u8>>(2)?,
+ row.get::<_, i64>(3)?,
+ row.get::<_, i64>(4)?,
+ row.get::<_, String>(5)?,
+ ))
+ },
+ )
+ .optional()?;
+ let Some((session, workspace, project, generation, attempts, state)) = row else {
+ tx.commit()?;
+ return Ok(None);
+ };
+
+ let session_id = parse_session_id(&session)?;
+ let workspace_id = parse_workspace_id(&workspace)?;
+ let project_id = parse_project_id(&project)?;
+ let generation = u64::try_from(generation).map_err(|_| {
+ StoreError::MalformedRecord("negative session consolidation generation".into())
+ })?;
+ let attempts = u32::try_from(attempts).map_err(|_| {
+ StoreError::MalformedRecord("invalid session consolidation attempt count".into())
+ })?;
+ let claimed_attempts = match state.as_str() {
+ "pending" => attempts.saturating_add(1),
+ "running" => attempts,
+ other => {
+ return Err(StoreError::MalformedRecord(format!(
+ "invalid claimable session consolidation state: {other}"
+ )));
+ }
+ };
+ let attempt_spent = state == "pending";
+ let claim_id = Uuid::new_v4();
+ let changed = tx.execute(
+ "UPDATE session_consolidation_jobs \
+ SET state = 'running', started_at = ?1, completed_at = NULL, \
+ attempts = CASE WHEN state = 'pending' THEN attempts + 1 ELSE attempts END, \
+ claim_id = ?2, last_error = NULL \
+ WHERE session_id = ?3 AND generation = ?4 \
+ AND attempts = ?5 \
+ AND ((state = 'pending' AND next_attempt_at <= ?1) \
+ OR (state = 'running' AND started_at <= ?6))",
+ params![
+ now,
+ claim_id.as_bytes(),
+ session_id.as_bytes(),
+ i64::try_from(generation).unwrap_or(i64::MAX),
+ i64::from(attempts),
+ stale_before,
+ ],
+ )?;
+ if changed != 1 {
+ return Err(StoreError::InvalidState(
+ "session consolidation job changed while being claimed".into(),
+ ));
+ }
+ tx.commit()?;
+ Ok(Some(SessionConsolidationJob {
+ session_id,
+ workspace_id,
+ project_id,
+ generation,
+ attempts: claimed_attempts,
+ claim_id,
+ attempt_spent,
+ }))
+}
+
+/// Mark a claimed job complete. A stale claim cannot complete a newer lease.
+pub fn complete(conn: &mut Connection, job: &SessionConsolidationJob) -> StoreResult<()> {
+ let now = Timestamp::now().as_microsecond();
+ let changed = conn.execute(
+ "UPDATE session_consolidation_jobs \
+ SET state = 'completed', completed_at = ?1, claim_id = NULL, last_error = NULL \
+ WHERE session_id = ?2 AND generation = ?3 \
+ AND state = 'running' AND claim_id = ?4",
+ params![
+ now,
+ job.session_id.as_bytes(),
+ generation_i64(job),
+ job.claim_id.as_bytes(),
+ ],
+ )?;
+ require_claim_update(changed)
+}
+
+/// Record a provider failure and either schedule another attempt or terminate.
+pub fn fail(
+ conn: &mut Connection,
+ job: &SessionConsolidationJob,
+ error: &str,
+ retry_at: Option<i64>,
+) -> StoreResult<()> {
+ let now = Timestamp::now().as_microsecond();
+ let error = truncate_error(error);
+ if job.attempts >= SESSION_CONSOLIDATION_MAX_ATTEMPTS || retry_at.is_none() {
+ let changed = conn.execute(
+ "UPDATE session_consolidation_jobs \
+ SET state = 'failed', completed_at = ?1, claim_id = NULL, last_error = ?2 \
+ WHERE session_id = ?3 AND generation = ?4 \
+ AND state = 'running' AND claim_id = ?5",
+ params![
+ now,
+ error,
+ job.session_id.as_bytes(),
+ generation_i64(job),
+ job.claim_id.as_bytes(),
+ ],
+ )?;
+ require_claim_update(changed)
+ } else {
+ let retry_at = retry_at.unwrap_or(now);
+ let changed = conn.execute(
+ "UPDATE session_consolidation_jobs \
+ SET state = 'pending', next_attempt_at = ?1, started_at = NULL, \
+ claim_id = NULL, last_error = ?2 \
+ WHERE session_id = ?3 AND generation = ?4 \
+ AND state = 'running' AND claim_id = ?5",
+ params![
+ retry_at,
+ error,
+ job.session_id.as_bytes(),
+ generation_i64(job),
+ job.claim_id.as_bytes(),
+ ],
+ )?;
+ require_claim_update(changed)
+ }
+}
+
+/// Release an in-flight claim during graceful shutdown without spending an
+/// attempt. The next server process can claim it immediately.
+pub fn release(conn: &mut Connection, job: &SessionConsolidationJob) -> StoreResult<()> {
+ let now = Timestamp::now().as_microsecond();
+ let changed = conn.execute(
+ "UPDATE session_consolidation_jobs \
+ SET state = 'pending', next_attempt_at = ?1, started_at = NULL, \
+ attempts = CASE WHEN ?2 = 1 AND attempts > 0 THEN attempts - 1 ELSE attempts END, \
+ claim_id = NULL \
+ WHERE session_id = ?3 AND generation = ?4 \
+ AND state = 'running' AND claim_id = ?5",
+ params![
+ now,
+ i64::from(job.attempt_spent),
+ job.session_id.as_bytes(),
+ generation_i64(job),
+ job.claim_id.as_bytes(),
+ ],
+ )?;
+ require_claim_update(changed)
+}
+
+fn require_claim_update(changed: usize) -> StoreResult<()> {
+ if changed == 1 {
+ Ok(())
+ } else {
+ Err(StoreError::InvalidState(
+ "session consolidation claim is no longer active".into(),
+ ))
+ }
+}
+
+fn generation_i64(job: &SessionConsolidationJob) -> i64 {
+ i64::try_from(job.generation).unwrap_or(i64::MAX)
+}
+
+fn truncate_error(error: &str) -> String {
+ error.chars().take(1_024).collect()
+}
+
+fn parse_session_id(bytes: &[u8]) -> StoreResult<SessionId> {
+ SessionId::from_slice(bytes).map_err(|error| StoreError::MalformedRecord(error.to_string()))
+}
+
+fn parse_workspace_id(bytes: &[u8]) -> StoreResult<WorkspaceId> {
+ WorkspaceId::from_slice(bytes).map_err(|error| StoreError::MalformedRecord(error.to_string()))
+}
+
+fn parse_project_id(bytes: &[u8]) -> StoreResult<ProjectId> {
+ ProjectId::from_slice(bytes).map_err(|error| StoreError::MalformedRecord(error.to_string()))
+}
+
+#[cfg(test)]
+mod tests {
+ use ai_memory_core::{
+ AgentKind, NewObservation, NewSession, ObservationKind, Sanitized, Sanitizer,
+ };
+
+ use super::*;
+ use crate::Store;
+
+ async fn ended_session() -> (tempfile::TempDir, Store, WorkspaceId, ProjectId, SessionId) {
+ let tmp = tempfile::TempDir::new().unwrap();
+ let store = Store::open(tmp.path()).unwrap();
+ let workspace_id = store
+ .writer
+ .get_or_create_workspace("default")
+ .await
+ .unwrap();
+ let project_id = store
+ .writer
+ .get_or_create_project(workspace_id, "project", None)
+ .await
+ .unwrap();
+ let session_id = SessionId::new();
+ store
+ .writer
+ .begin_session(NewSession {
+ id: session_id,
+ workspace_id,
+ project_id,
+ agent_kind: AgentKind::Codex,
+ cwd: None,
+ })
+ .await
+ .unwrap();
+ insert_observation(&store, workspace_id, project_id, session_id).await;
+ store.writer.end_session(session_id, None).await.unwrap();
+ (tmp, store, workspace_id, project_id, session_id)
+ }
+
+ async fn insert_observation(
+ store: &Store,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ session_id: SessionId,
+ ) {
+ store
+ .writer
+ .insert_observation(Sanitized::new(
+ NewObservation {
+ session_id,
+ workspace_id,
+ project_id,
+ kind: ObservationKind::UserPrompt,
+ extension: None,
+ source_event: None,
+ title: "prompt".into(),
+ body: "continue".into(),
+ importance: 8,
+ },
+ &Sanitizer::default(),
+ ))
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn queue_is_idempotent_per_observation_generation() {
+ let (_tmp, store, workspace_id, project_id, session_id) = ended_session().await;
+ assert!(
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap()
+ );
+ assert!(
+ !store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap()
+ );
+
+ let now = Timestamp::now().as_microsecond();
+ let job = store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(job.session_id(), session_id);
+ assert_eq!(job.generation(), 1);
+ assert_eq!(job.attempts(), 1);
+ store
+ .writer
+ .complete_session_consolidation(job)
+ .await
+ .unwrap();
+ assert!(
+ store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .is_none()
+ );
+
+ insert_observation(&store, workspace_id, project_id, session_id).await;
+ store.writer.end_session(session_id, None).await.unwrap();
+ assert!(
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap(),
+ "resumed work creates a new immutable generation"
+ );
+ let job = store
+ .writer
+ .claim_session_consolidation(Timestamp::now().as_microsecond(), now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(job.generation(), 2);
+ }
+
+ #[tokio::test]
+ async fn failed_claim_retries_and_graceful_release_does_not_spend_attempt() {
+ let (_tmp, store, workspace_id, project_id, session_id) = ended_session().await;
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap();
+ let now = Timestamp::now().as_microsecond();
+ let first = store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ store
+ .writer
+ .fail_session_consolidation(first, "provider unavailable".into(), Some(now + 100))
+ .await
+ .unwrap();
+ assert!(
+ store
+ .writer
+ .claim_session_consolidation(now + 99, now - 1)
+ .await
+ .unwrap()
+ .is_none()
+ );
+ let second = store
+ .writer
+ .claim_session_consolidation(now + 100, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(second.attempts(), 2);
+ store
+ .writer
+ .release_session_consolidation(second)
+ .await
+ .unwrap();
+ let reclaim_at = Timestamp::now().as_microsecond() + 100;
+ let reclaimed = store
+ .writer
+ .claim_session_consolidation(reclaim_at, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(reclaimed.attempts(), 2);
+ }
+
+ #[tokio::test]
+ async fn newer_generation_supersedes_older_pending_work() {
+ let (_tmp, store, workspace_id, project_id, session_id) = ended_session().await;
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap();
+ insert_observation(&store, workspace_id, project_id, session_id).await;
+ store.writer.end_session(session_id, None).await.unwrap();
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap();
+
+ let now = Timestamp::now().as_microsecond();
+ let latest = store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(latest.generation(), 2);
+ store
+ .writer
+ .complete_session_consolidation(latest)
+ .await
+ .unwrap();
+ assert!(
+ store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .is_none(),
+ "the superseded first generation must not run"
+ );
+ }
+
+ #[tokio::test]
+ async fn expired_lease_rejects_completion_from_the_old_claim() {
+ let (_tmp, store, workspace_id, project_id, session_id) = ended_session().await;
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap();
+ let now = Timestamp::now().as_microsecond();
+ let first = store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ let second = store
+ .writer
+ .claim_session_consolidation(now + 100, now)
+ .await
+ .unwrap()
+ .unwrap();
+ assert!(
+ store
+ .writer
+ .complete_session_consolidation(first)
+ .await
+ .is_err(),
+ "an expired claim must not complete its replacement's lease"
+ );
+ assert_eq!(second.attempts(), 1);
+ store
+ .writer
+ .release_session_consolidation(second)
+ .await
+ .unwrap();
+ let reclaimed = store
+ .writer
+ .claim_session_consolidation(Timestamp::now().as_microsecond() + 100, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(
+ reclaimed.attempts(),
+ 2,
+ "release after a recovered lease must retain the original attempt count"
+ );
+ store
+ .writer
+ .complete_session_consolidation(reclaimed)
+ .await
+ .unwrap();
+ }
+
+ #[tokio::test]
+ async fn expired_final_attempt_replays_without_spending_an_extra_attempt() {
+ let (_tmp, store, workspace_id, project_id, session_id) = ended_session().await;
+ store
+ .writer
+ .enqueue_session_consolidation(workspace_id, project_id, session_id)
+ .await
+ .unwrap();
+ let now = Timestamp::now().as_microsecond();
+ for expected_attempt in 1..SESSION_CONSOLIDATION_MAX_ATTEMPTS {
+ let job = store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(job.attempts(), expected_attempt);
+ store
+ .writer
+ .fail_session_consolidation(job, "retry".into(), Some(now))
+ .await
+ .unwrap();
+ }
+ let final_claim = store
+ .writer
+ .claim_session_consolidation(now, now - 1)
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(final_claim.attempts(), SESSION_CONSOLIDATION_MAX_ATTEMPTS);
+
+ let recovered = store
+ .writer
+ .claim_session_consolidation(now + 1, now)
+ .await
+ .unwrap()
+ .expect("a crashed final attempt must be recovered from its expired lease");
+ assert_eq!(recovered.attempts(), SESSION_CONSOLIDATION_MAX_ATTEMPTS);
+ store
+ .writer
+ .fail_session_consolidation(recovered, "terminal".into(), None)
+ .await
+ .unwrap();
+ }
+}
diff --git a/crates/ai-memory-store/src/writer.rs b/crates/ai-memory-store/src/writer.rs
index fb41f415..135630be 100644
--- a/crates/ai-memory-store/src/writer.rs
+++ b/crates/ai-memory-store/src/writer.rs
@@ -26,6 +26,7 @@ use crate::ops::{
self, DeleteWorkspaceSummary, EmbeddingWrite, IngestObservationOutcome, MoveSummary,
PurgeSummary, ReorgSummary,
};
+use crate::session_consolidation::SessionConsolidationJob;
use crate::users::{self, TOKEN_HASH_LEN};
use crate::workstream::{
FinishWorkstreamRun, FinishedWorkstreamRun, PrepareWorkstreamRun, PreparedWorkstreamRun,
@@ -113,6 +114,31 @@ pub(crate) enum WriteCmd {
ingest_key: String,
reply: oneshot::Sender<StoreResult<()>>,
},
+ EnqueueSessionConsolidation {
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ session_id: SessionId,
+ reply: oneshot::Sender<StoreResult<bool>>,
+ },
+ ClaimSessionConsolidation {
+ now: i64,
+ stale_before: i64,
+ reply: oneshot::Sender<StoreResult<Option<SessionConsolidationJob>>>,
+ },
+ CompleteSessionConsolidation {
+ job: SessionConsolidationJob,
+ reply: oneshot::Sender<StoreResult<()>>,
+ },
+ FailSessionConsolidation {
+ job: SessionConsolidationJob,
+ error: String,
+ retry_at: Option<i64>,
+ reply: oneshot::Sender<StoreResult<()>>,
+ },
+ ReleaseSessionConsolidation {
+ job: SessionConsolidationJob,
+ reply: oneshot::Sender<StoreResult<()>>,
+ },
InsertHandoff {
handoff: NewHandoff,
reply: oneshot::Sender<StoreResult<HandoffId>>,
@@ -565,6 +591,81 @@ impl WriterHandle {
rx.await.map_err(|_| StoreError::WriterClosed)?
}
+ /// Persist one opt-in SessionEnd consolidation job for the current
+ /// observation generation. Duplicate generations are idempotent.
+ pub async fn enqueue_session_consolidation(
+ &self,
+ workspace_id: WorkspaceId,
+ project_id: ProjectId,
+ session_id: SessionId,
+ ) -> StoreResult<bool> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::EnqueueSessionConsolidation {
+ workspace_id,
+ project_id,
+ session_id,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
+ /// Atomically claim the oldest due SessionEnd consolidation job.
+ pub async fn claim_session_consolidation(
+ &self,
+ now: i64,
+ stale_before: i64,
+ ) -> StoreResult<Option<SessionConsolidationJob>> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::ClaimSessionConsolidation {
+ now,
+ stale_before,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
+ /// Mark a claimed SessionEnd consolidation job complete.
+ pub async fn complete_session_consolidation(
+ &self,
+ job: SessionConsolidationJob,
+ ) -> StoreResult<()> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::CompleteSessionConsolidation { job, reply: tx })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
+ /// Record a failed SessionEnd consolidation attempt.
+ pub async fn fail_session_consolidation(
+ &self,
+ job: SessionConsolidationJob,
+ error: String,
+ retry_at: Option<i64>,
+ ) -> StoreResult<()> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::FailSessionConsolidation {
+ job,
+ error,
+ retry_at,
+ reply: tx,
+ })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
+ /// Return an in-flight SessionEnd consolidation job to the durable queue.
+ pub async fn release_session_consolidation(
+ &self,
+ job: SessionConsolidationJob,
+ ) -> StoreResult<()> {
+ let (tx, rx) = oneshot::channel();
+ self.send(WriteCmd::ReleaseSessionConsolidation { job, reply: tx })
+ .await?;
+ rx.await.map_err(|_| StoreError::WriterClosed)?
+ }
+
/// Insert a new handoff in `open` state.
///
/// # Errors
@@ -1371,6 +1472,45 @@ fn worker_loop(mut conn: Connection, mut rx: mpsc::Receiver<WriteCmd>) {
let result = ops::complete_observation_ingest(&mut conn, &project_id, &ingest_key);
send_or_warn(reply, result, "complete_observation_ingest");
}
+ WriteCmd::EnqueueSessionConsolidation {
+ workspace_id,
+ project_id,
+ session_id,
+ reply,
+ } => {
+ let result = crate::session_consolidation::enqueue(
+ &mut conn,
+ workspace_id,
+ project_id,
+ session_id,
+ );
+ send_or_warn(reply, result, "enqueue_session_consolidation");
+ }
+ WriteCmd::ClaimSessionConsolidation {
+ now,
+ stale_before,
+ reply,
+ } => {
+ let result = crate::session_consolidation::claim_next(&mut conn, now, stale_before);
+ send_or_warn(reply, result, "claim_session_consolidation");
+ }
+ WriteCmd::CompleteSessionConsolidation { job, reply } => {
+ let result = crate::session_consolidation::complete(&mut conn, &job);
+ send_or_warn(reply, result, "complete_session_consolidation");
+ }
+ WriteCmd::FailSessionConsolidation {
+ job,
+ error,
+ retry_at,
+ reply,
+ } => {
+ let result = crate::session_consolidation::fail(&mut conn, &job, &error, retry_at);
+ send_or_warn(reply, result, "fail_session_consolidation");
+ }
+ WriteCmd::ReleaseSessionConsolidation { job, reply } => {
+ let result = crate::session_consolidation::release(&mut conn, &job);
+ send_or_warn(reply, result, "release_session_consolidation");
+ }
WriteCmd::InsertHandoff { handoff, reply } => {
let result = ops::insert_handoff(&mut conn, &handoff);
send_or_warn(reply, result, "insert_handoff");
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 1317b43b..03a776cf 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -206,6 +206,7 @@ separately gated Claude Code assistant/Stop excerpt remains capped at 2 KB.
| `pages` | Versioned wiki pages with `is_latest` + `supersedes` chain. M8 columns: `last_accessed_at`, `access_count`, `superseded_at`. M9 cols: `embedding_provider`, `embedding_model`, `embedding_dim`. |
| `pages_fts` | FTS5 virtual table over `(title, body)`, auto-synced by triggers. |
| `sessions`, `observations` | Sanitized, bounded lifecycle-hook projections. They are an operational audit trail, not a complete native transcript. |
+| `session_consolidation_jobs` | Durable, observation-generation-idempotent queue for opt-in SessionEnd LLM consolidation. One bounded server worker leases jobs, retries provider failures with backoff, and recovers expired leases after restart. |
| `observations_fts` | FTS5 virtual table over raw observation `(title, body)`, used only as bounded fallback. |
| `workstreams`, `managed_runs`, `workstream_native_sessions` | Optional lease state plus per-harness native source and delivery cursors for `ai-memory run`. |
| `workstream_events`, `workstream_events_fts` | Append-only normalized visible transcript events and full-text search; immutable sanitized source batches also live under `raw/workstreams/`. |
diff --git a/docs/design-decisions.md b/docs/design-decisions.md
index d5a82fcc..8266890b 100644
--- a/docs/design-decisions.md
+++ b/docs/design-decisions.md
@@ -96,7 +96,7 @@ Why not LanceDB/Qdrant/Kuzu/CozoDB/SurrealDB?
**LLM for consolidation passes:**
- **Off by default**, behaves like agentmemory after #138's fix. Without a provider, the system still works: synthetic compression (rule-based), no LLM-generated summaries, no `memory_consolidate` page-rewrite.
-- With a provider, LLM consolidation runs on PreCompact, on demand via `memory_consolidate`, and at session end only when `AI_MEMORY_CONSOLIDATE_ON_SESSION_END=true` (off by default — session end always writes a rule-based summary page + handoff regardless). Optional 6h maintenance timer.
+- With a provider, LLM consolidation runs on PreCompact, on demand via `memory_consolidate`, and at session end only when `AI_MEMORY_CONSOLIDATE_ON_SESSION_END=true` (off by default — session end always writes a rule-based summary page + handoff regardless). SessionEnd provider work is persisted by observation generation and consumed outside the hook request by one bounded retrying worker, so client drain cancellation cannot lose it. Optional 6h maintenance timer.
- Providers implement `LlmProvider { complete(...); complete_structured(...) }`. The current provider and authentication matrix lives in [`ARCHITECTURE.md`](ARCHITECTURE.md); this design boundary also covers OpenAI-compatible endpoints such as Ollama, vLLM, and LM Studio.
- **Native HTTP per provider** - no LiteLLM-equivalent. The cognee tracker (#2412/#2430/#2537/#2608/#2749/#2782/#2840/#2842) showed silent-kwarg-drop in a generic gateway is the #1 source of provider bugs. Each provider's typed JSON, errors on unknown fields. Hand-coded but correct.
- **Structured output via JSON schema, not XML, not Instructor-style wrapping.** Use each provider's native JSON-mode where available; for Anthropic, request a tool-use response with a typed schema. Validate with `serde_json` + `schemars`-derived schemas.
diff --git a/docs/usage.md b/docs/usage.md
index 490a48fb..7778eb57 100644
--- a/docs/usage.md
+++ b/docs/usage.md
@@ -62,7 +62,7 @@ at the managed ai-memory Agent Skills that carry detailed tool routing.
| "Where did we leave off?" | Existing handoff block, or `memory_handoff_accept` if no block exists | Resumes from the latest pending handoff. |
| "Save context for the next session" | `memory_handoff_begin` | Writes a terse session-end handoff with open questions and next steps. Do not use for status or briefing requests. |
| "Discard that handoff" / "I created a handoff by mistake" | `memory_handoff_cancel` | Marks an exact open handoff id expired before the next session can consume it. |
-| "Consolidate this session" | `memory_consolidate` | Manually runs LLM consolidation. Also runs on PreCompact, and at session end only when `AI_MEMORY_CONSOLIDATE_ON_SESSION_END` is set (off by default; session end otherwise writes a rule-based summary page). |
+| "Consolidate this session" | `memory_consolidate` | Manually runs LLM consolidation. Also runs on PreCompact, and at session end only when `AI_MEMORY_CONSOLIDATE_ON_SESSION_END` is set (off by default; session end otherwise writes a rule-based summary page). Opt-in SessionEnd provider work is durably queued outside the hook response, retried with backoff, and recovered after server restart. |
| "What did we learn from this session?" / "what memory should we add?" | `memory_auto_improve` | Manually reviews the latest completed session by default. The server also runs scheduled auto-improvement for new completed sessions when an LLM is configured. `[auto_improve.scheduler] enabled = false` disables automatic review; `[auto_improve] require_approval = true` leaves scheduled and manual proposals in pending-writes for review. |
| "Remember this permanently" / "add an annotation" | `memory_write_page` | Writes durable wiki knowledge; not a single-use handoff. |
| "Delete this page" / "remove the note about X" | `memory_delete_page` | Removes a page by exact path. Pass `workspace` + `project` together when the page lives in a sibling workspace, so a project name shared between workspaces never silently routes the delete to the wrong slot. |
diff --git a/tests/e2e/handoff_smoke.sh b/tests/e2e/handoff_smoke.sh
index 2fb396c6..3a014b5f 100755
--- a/tests/e2e/handoff_smoke.sh
+++ b/tests/e2e/handoff_smoke.sh
@@ -84,6 +84,7 @@ fi
# Free port for ai-memory.
PORT="$(python3 -c "import socket; s=socket.socket(); s.bind(('127.0.0.1',0)); print(s.getsockname()[1]); s.close()")"
SERVER_URL="http://127.0.0.1:$PORT"
+HOOK_SCOPE="workspace=e2e-test&project=blog"
# Two deliberately different Gemini variants. Both are free-tier;
# overridable via env when the defaults rotate out.
@@ -213,7 +214,7 @@ SESSION_ID_1="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"
step "Session 1 ($MODEL_A): session-start hook"
echo "{\"session_id\":\"$SESSION_ID_1\",\"cwd\":\"$TEST_DIR/blog\",\"model\":\"$MODEL_A\"}" \
| curl -sS --max-time 3 \
- -X POST "$SERVER_URL/hook?event=session-start&agent=open-code" \
+ -X POST "$SERVER_URL/hook?event=session-start&agent=open-code&$HOOK_SCOPE" \
-H "Content-Type: application/json" \
-H "$AUTH_HEADER" \
--data-binary @- >>"$LOG_FILE" 2>&1
@@ -240,7 +241,7 @@ echo "--- end model A response ---"
step "Session 1: forward user-prompt + session-end hooks"
echo "{\"session_id\":\"$SESSION_ID_1\",\"prompt\":$(jq -Rs <<<"$S1_PROMPT")}" \
| curl -sS --max-time 3 \
- -X POST "$SERVER_URL/hook?event=user-prompt&agent=open-code" \
+ -X POST "$SERVER_URL/hook?event=user-prompt&agent=open-code&$HOOK_SCOPE" \
-H "Content-Type: application/json" \
-H "$AUTH_HEADER" \
--data-binary @- >>"$LOG_FILE" 2>&1
@@ -250,7 +251,7 @@ echo "{\"session_id\":\"$SESSION_ID_1\",\"prompt\":$(jq -Rs <<<"$S1_PROMPT")}" \
for tool in "Read" "Edit" "Write"; do
echo "{\"session_id\":\"$SESSION_ID_1\",\"tool\":\"$tool\"}" \
| curl -sS --max-time 2 \
- -X POST "$SERVER_URL/hook?event=post-tool-use&agent=open-code" \
+ -X POST "$SERVER_URL/hook?event=post-tool-use&agent=open-code&$HOOK_SCOPE" \
-H "Content-Type: application/json" \
-H "$AUTH_HEADER" \
--data-binary @- >>"$LOG_FILE" 2>&1
@@ -258,7 +259,7 @@ done
echo "{\"session_id\":\"$SESSION_ID_1\",\"cwd\":\"$TEST_DIR/blog\"}" \
| curl -sS --max-time 10 \
- -X POST "$SERVER_URL/hook?event=session-end&agent=open-code" \
+ -X POST "$SERVER_URL/hook?event=session-end&agent=open-code&$HOOK_SCOPE" \
-H "Content-Type: application/json" \
-H "$AUTH_HEADER" \
--data-binary @- >>"$LOG_FILE" 2>&1
@@ -287,7 +288,7 @@ SESSION_ID_2="$(uuidgen 2>/dev/null || cat /proc/sys/kernel/random/uuid)"
step "Session 2 ($MODEL_B): session-start hook"
echo "{\"session_id\":\"$SESSION_ID_2\",\"cwd\":\"$TEST_DIR/blog\",\"model\":\"$MODEL_B\"}" \
| curl -sS --max-time 3 \
- -X POST "$SERVER_URL/hook?event=session-start&agent=open-code" \
+ -X POST "$SERVER_URL/hook?event=session-start&agent=open-code&$HOOK_SCOPE" \
-H "Content-Type: application/json" \
-H "$AUTH_HEADER" \
--data-binary @- >>"$LOG_FILE" 2>&1