Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(hooks): move the ingest counters on /hook/batch, and stamp last write only on a real write (#517)

Aguinaldo Tupy4844020335bc
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 229d556f..f7b79708 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -38,6 +38,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
HTTP 400 and disappear from memory while ASCII events still worked. A native
Windows loopback test now round-trips Chinese and Portuguese text byte for
byte (#500).
+- The `ingest (server, this process)` counters in `ai-memory status` now move
+ for events delivered over `POST /hook/batch`. They were instrumented only in
+ the per-event `POST /hook` handler, while the spool drain posts batches and
+ falls back to `/hook` only against a pre-upgrade server, so a current client
+ against a current server left the whole section reading `accepted 0`,
+ `last write: -` and zero sheds while observations were landing normally — the
+ section exists precisely to tell "hooks are not arriving" apart from "hooks
+ are arriving but nothing is stored", and it reported the same thing for both.
+ A batch 429 also now counts every item it rejects, not just the one that
+ found no permit, so shed volume is comparable between the two routes instead
+ of understated by up to the batch size (#516).
+- `last write` in the same section now advances only when an event actually
+ cleared the writer. `handle_hook` stamped it unconditionally after processing
+ returned, and processing swallows its errors, so a store rejecting every
+ event — read-only database, exhausted disk, a failed migration — still
+ reported a fresh write time and looked healthy (#516).
## [1.32.2] - 2026-08-26
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index 38ed071f..327517bd 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -583,6 +583,14 @@ pub fn hook_router(state: HookState) -> Router {
.with_state(Arc::new(state))
}
+/// Unix milliseconds for an ingest metric stamp, saturating rather than
+/// failing: a clock before the epoch or past `u64` must not cost an event.
+fn now_unix_ms() -> u64 {
+ std::time::SystemTime::now()
+ .duration_since(std::time::UNIX_EPOCH)
+ .map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX))
+}
+
async fn handle_hook(
State(state): State<Arc<HookState>>,
Query(query): Query<HookQuery>,
@@ -642,7 +650,7 @@ async fn handle_hook(
tokio::spawn(async move {
let _permit = permit;
let metrics = state.ingest_metrics.clone();
- process_envelope(
+ let persisted = process_envelope(
state,
env,
actor,
@@ -650,16 +658,15 @@ async fn handle_hook(
skip_webhooks,
)
.await;
- // Stamped after `process_envelope` returns, which is the point the
- // event has been through the writer. This is the signal an operator
- // uses to tell "hooks are arriving but nothing is landing" from
- // "nothing is arriving" — the two look identical from the accepted
- // count alone.
- metrics.record_persisted(
- std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX)),
- );
+ // Stamped only when `process_envelope` actually cleared the writer.
+ // This is the signal an operator uses to tell "hooks are arriving but
+ // nothing is landing" from "nothing is arriving" — the two look
+ // identical from the accepted count alone — so a failed write must
+ // NOT advance it, or a store that is rejecting every event still
+ // reads as a healthy writer in `ai-memory status`.
+ if persisted {
+ metrics.record_persisted(now_unix_ms());
+ }
});
(StatusCode::ACCEPTED, "queued")
}
@@ -779,6 +786,7 @@ async fn handle_hook_batch(
let skip_webhooks = admission_skips(level_ext, &headers);
let actor_storage_key = actor.as_ref().map(IdentityKey::storage_key);
let mut accepted_indices = Vec::new();
+ let total_items = items.len();
for (idx, mut item) in items.into_iter().enumerate() {
// Same unconditional assistant-message backstop as `handle_hook`, applied
// per item before the envelope is built (#196).
@@ -793,6 +801,7 @@ async fn handle_hook_batch(
// A protocol-directed drop is committed from the spool's point of
// view, but intentionally spends neither ingress capacity nor a
// source-rate token.
+ state.ingest_metrics.record_dropped_by_policy();
accepted_indices.push(idx);
continue;
};
@@ -800,13 +809,21 @@ async fn handle_hook_batch(
// as committed so the client clears it from its spool, but do not store
// it. Keeps the contiguous-prefix ack contract intact.
if should_drop_subagent(&state, &env).await {
+ state.ingest_metrics.record_dropped_by_policy();
accepted_indices.push(idx);
continue;
}
let Ok(permit) = state.ingest_semaphore.clone().try_acquire_owned() else {
+ // The 429 rejects this item AND every item behind it, so count all
+ // of them: on `/hook` one 429 is one shed event, and a batch that
+ // sheds 200 events must not read as 1.
+ let shed = total_items.saturating_sub(idx);
+ for _ in 0..shed {
+ state.ingest_metrics.record_shed_saturated();
+ }
warn!(
accepted = accepted_indices.len(),
- "hook batch ingest saturated; rejecting with 429"
+ shed, "hook batch ingest saturated; rejecting with 429"
);
return (
StatusCode::TOO_MANY_REQUESTS,
@@ -821,10 +838,12 @@ async fn handle_hook_batch(
.try_take(&rate_key, std::time::Instant::now())
{
drop(permit);
+ state.ingest_metrics.record_shed_rate_limited();
warn!(accepted = accepted_indices.len(), source = %log_rate_key(&rate_key), "hook batch source rate limited; skipping item and continuing");
continue;
}
let _permit = permit;
+ state.ingest_metrics.record_accepted();
if let Err(e) = process_authorized(
&state,
env,
@@ -848,6 +867,9 @@ async fn handle_hook_batch(
Json(HookBatchAck::indexed_failed(accepted_indices, idx)),
);
}
+ // Stamped per item, for the same reason `handle_hook` stamps after
+ // `process_envelope`: this is the point the event cleared the writer.
+ state.ingest_metrics.record_persisted(now_unix_ms());
accepted_indices.push(idx);
}
(
@@ -2190,13 +2212,17 @@ fn sticky_cwd_admits(
&& meaningful_session_anchor(session_cwd, home_dir).is_some())
}
+/// Returns `true` when the event cleared the writer, so the caller can stamp
+/// the ingest "last write" metric. A rejected or failed event returns `false`:
+/// nothing was persisted, and pretending otherwise hides exactly the outage
+/// the metric exists to expose.
async fn process_envelope(
state: Arc<HookState>,
env: HookEnvelope,
actor: Option<IdentityKey>,
level: ai_memory_core::AuthLevel,
skip_webhooks: Vec<String>,
-) {
+) -> bool {
if let Err(e) = process_authorized(&state, env, actor, level, skip_webhooks).await {
if matches!(
e.downcast_ref::<StoreError>(),
@@ -2206,7 +2232,9 @@ async fn process_envelope(
} else {
warn!(error = %e, "hook processing failed");
}
+ return false;
}
+ true
}
async fn enqueue_session_end_consolidation(
@@ -4980,6 +5008,249 @@ mod tests {
assert_eq!(ack["accepted"], 2, "both events committed, oldest-first");
}
+ /// The ingest counters exist to answer "are hooks arriving, is anything
+ /// being shed, is the writer keeping up" (#428). The native drain delivers
+ /// via `/hook/batch`, so a counter only wired into `handle_hook` answers
+ /// that question wrong on the path clients actually use.
+ #[tokio::test]
+ async fn handle_hook_batch_records_accepted_and_persisted() {
+ let tmp = TempDir::new().unwrap();
+ let state = make_state(&tmp).await;
+ let metrics = state.ingest_metrics.clone();
+
+ let response = handle_hook_batch(
+ State(Arc::new(state)),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(vec![
+ HookBatchItem {
+ url: "http://h/hook?event=session-start&agent=claude-code".into(),
+ body: serde_json::json!({ "session_id": "metrics-s1" }),
+ },
+ HookBatchItem {
+ url: "http://h/hook?event=user-prompt-submit&agent=claude-code".into(),
+ body: serde_json::json!({ "session_id": "metrics-s1", "prompt": "hi" }),
+ },
+ ]),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::OK);
+
+ let snap = metrics.snapshot();
+ assert_eq!(snap.accepted, 2, "both batch items were admitted");
+ assert!(
+ snap.last_persisted_ms.is_some(),
+ "a batch that reached the writer must stamp a write time"
+ );
+ }
+
+ #[tokio::test]
+ async fn handle_hook_batch_records_shed_when_saturated() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ state.ingest_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+ let metrics = state.ingest_metrics.clone();
+
+ let response = handle_hook_batch(
+ State(Arc::new(state)),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(vec![HookBatchItem {
+ url: "http://h/hook?event=session-start&agent=claude-code".into(),
+ body: serde_json::json!({}),
+ }]),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
+
+ let snap = metrics.snapshot();
+ assert_eq!(snap.shed_saturated, 1);
+ assert_eq!(snap.accepted, 0, "a shed item was never admitted");
+ }
+
+ #[tokio::test]
+ async fn handle_hook_batch_records_shed_when_rate_limited() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ let mut limiter = IngestRateLimiter::new(0.001, 1.0);
+ assert!(limiter.try_take("u:\ns:flooder", std::time::Instant::now()));
+ state.ingest_rate = Arc::new(tokio::sync::Mutex::new(limiter));
+ let metrics = state.ingest_metrics.clone();
+
+ let response = handle_hook_batch(
+ State(Arc::new(state)),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(vec![HookBatchItem {
+ url: "http://h/hook?event=session-start&agent=claude-code".into(),
+ body: serde_json::json!({ "session_id": "flooder" }),
+ }]),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::OK);
+
+ let snap = metrics.snapshot();
+ assert_eq!(snap.shed_rate_limited, 1);
+ assert_eq!(snap.accepted, 0);
+ }
+
+ /// Both accept-but-drop branches, in one batch: a client-protocol Drop and
+ /// the subagent tail-drop. They are separate call sites in
+ /// `handle_hook_batch`, so a single-branch case would leave the other
+ /// silently uncounted — the exact failure this change exists to fix.
+ #[tokio::test]
+ async fn handle_hook_batch_records_dropped_by_policy() {
+ let tmp = TempDir::new().unwrap();
+ let state = make_state(&tmp).await;
+ let metrics = state.ingest_metrics.clone();
+
+ let response = handle_hook_batch(
+ State(Arc::new(state)),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(vec![
+ HookBatchItem {
+ url: "http://h/hook?event=post-tool-use&agent=claude-code".into(),
+ body: serde_json::json!({
+ "session_id": "drop-s1", "tool_name": "Write",
+ "_ai_memory_capture":
+ capture_protocol("drop", "inactive", "file", 1, "extracted"),
+ }),
+ },
+ HookBatchItem {
+ url: "http://h/hook?event=pre-tool-use&agent=grok&drop_subagent=1".into(),
+ body: serde_json::json!({
+ "sessionId": "sub-s1", "subagentType": "general-purpose", "toolName": "x"
+ }),
+ },
+ ]),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::OK);
+
+ let snap = metrics.snapshot();
+ assert_eq!(snap.dropped_by_policy, 2, "accept-but-drop is still a drop");
+ assert_eq!(
+ snap.accepted, 0,
+ "a dropped item spends no ingress capacity"
+ );
+ }
+
+ /// `last_persisted_ms` is the one signal that separates "hooks are
+ /// arriving but nothing is landing" from "nothing is arriving". Stamping
+ /// it for an event that failed inside the writer collapses those two
+ /// states again: a store rejecting every event would still report a fresh
+ /// last write in `ai-memory status`.
+ #[tokio::test]
+ async fn handle_hook_does_not_stamp_last_write_when_processing_fails() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ // One permit, so re-acquiring it below blocks until the spawned task
+ // has finished (and therefore past the metric stamp).
+ state.ingest_semaphore = Arc::new(tokio::sync::Semaphore::new(1));
+ let metrics = state.ingest_metrics.clone();
+ let semaphore = state.ingest_semaphore.clone();
+
+ // A UserPrompt with no session id fails inside `process_authorized`.
+ let response = handle_hook(
+ State(Arc::new(state)),
+ Query(HookQuery {
+ event: "user-prompt-submit".into(),
+ agent: Some("claude-code".into()),
+ ..Default::default()
+ }),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(serde_json::json!({ "prompt": "missing session fails" })),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::ACCEPTED);
+ drop(semaphore.acquire().await.unwrap());
+
+ let snap = metrics.snapshot();
+ assert_eq!(snap.accepted, 1, "the event was admitted");
+ assert_eq!(
+ snap.last_persisted_ms, None,
+ "an event that failed in the writer never reached durable storage"
+ );
+ }
+
+ #[tokio::test]
+ async fn handle_hook_stamps_last_write_when_processing_succeeds() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ state.ingest_semaphore = Arc::new(tokio::sync::Semaphore::new(1));
+ let metrics = state.ingest_metrics.clone();
+ let semaphore = state.ingest_semaphore.clone();
+
+ let response = handle_hook(
+ State(Arc::new(state)),
+ Query(HookQuery {
+ event: "session-start".into(),
+ agent: Some("claude-code".into()),
+ ..Default::default()
+ }),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(serde_json::json!({ "session_id": "persist-s1" })),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::ACCEPTED);
+ drop(semaphore.acquire().await.unwrap());
+
+ assert!(
+ metrics.snapshot().last_persisted_ms.is_some(),
+ "a stored event must stamp a write time"
+ );
+ }
+
+ /// A saturated batch answers 429 for the item that found no permit AND
+ /// every item behind it. Counting one shed event for a rejection that
+ /// dropped many understates the shed rate by the batch size — on the
+ /// `/hook/batch` path the native drain actually uses.
+ #[tokio::test]
+ async fn handle_hook_batch_counts_every_item_the_429_sheds() {
+ let tmp = TempDir::new().unwrap();
+ let mut state = make_state(&tmp).await;
+ state.ingest_semaphore = Arc::new(tokio::sync::Semaphore::new(0));
+ let metrics = state.ingest_metrics.clone();
+
+ let items = (0..4)
+ .map(|i| HookBatchItem {
+ url: "http://h/hook?event=session-start&agent=claude-code".into(),
+ body: serde_json::json!({ "session_id": format!("shed-{i}") }),
+ })
+ .collect::<Vec<_>>();
+ let response = handle_hook_batch(
+ State(Arc::new(state)),
+ None,
+ None,
+ HeaderMap::new(),
+ Json(items),
+ )
+ .await
+ .into_response();
+ assert_eq!(response.status(), StatusCode::TOO_MANY_REQUESTS);
+
+ assert_eq!(
+ metrics.snapshot().shed_saturated,
+ 4,
+ "the 429 rejected all four items, not just the first"
+ );
+ }
+
/// Recursively scan every file under `dir` for a byte pattern. Used to prove
/// a stripped field left no trace anywhere in the on-disk store (any column,
/// the WAL, etc.), not just in the read-back observation body.