Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(hooks): drop session-less batch items instead of stalling the drain

AkitaOnRails6372392bee1f
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c0229db6..789dcefb 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -12,6 +12,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
provider. (#1026)
### Fixed
+- Fixed hook-spool drains stalling behind an event that has no session id.
+ `/hook/batch` reported such an event (anything but a session start) as a
+ failed item, so the drain retried it up to its attempt budget while every
+ event queued behind it waited. The server now acknowledges and drops it,
+ counted as `dropped_invalid` in the ingest metrics, and keeps processing the
+ rest of the batch. (#1062)
- Fixed interrupted launchers blocking an immediate managed-workstream restart
by adding explicit `ai-memory run --force-unlock` recovery. The server
atomically expires and replaces only a lease attributed to the same
diff --git a/crates/ai-memory-core/src/ingest_metrics.rs b/crates/ai-memory-core/src/ingest_metrics.rs
index 557ced30..2dcf7b0f 100644
--- a/crates/ai-memory-core/src/ingest_metrics.rs
+++ b/crates/ai-memory-core/src/ingest_metrics.rs
@@ -41,6 +41,10 @@ pub struct IngestMetrics {
/// of it is landing. Those want different answers, so they must not share
/// a counter.
dropped_unauthorized: AtomicU64,
+ /// Batch items accepted but not stored because they can never be: an
+ /// event with no session id that is not a SessionStart. Acknowledged so
+ /// the spool drops them instead of retrying them ahead of valid events.
+ dropped_invalid: AtomicU64,
/// Events shed because the global ingest semaphore had no permit (429).
shed_saturated: AtomicU64,
/// Events shed by the per-source rate limiter (429).
@@ -63,6 +67,10 @@ impl IngestMetrics {
pub fn record_dropped_unauthorized(&self) {
self.dropped_unauthorized.fetch_add(1, Ordering::Relaxed);
}
+ /// One batch item dropped because it can never be stored.
+ pub fn record_dropped_invalid(&self) {
+ self.dropped_invalid.fetch_add(1, Ordering::Relaxed);
+ }
/// One event shed because ingest capacity was exhausted.
pub fn record_shed_saturated(&self) {
self.shed_saturated.fetch_add(1, Ordering::Relaxed);
@@ -83,6 +91,7 @@ impl IngestMetrics {
accepted: self.accepted.load(Ordering::Relaxed),
dropped_by_policy: self.dropped_by_policy.load(Ordering::Relaxed),
dropped_unauthorized: self.dropped_unauthorized.load(Ordering::Relaxed),
+ dropped_invalid: self.dropped_invalid.load(Ordering::Relaxed),
shed_saturated: self.shed_saturated.load(Ordering::Relaxed),
shed_rate_limited: self.shed_rate_limited.load(Ordering::Relaxed),
last_persisted_ms: match self.last_persisted_ms.load(Ordering::Relaxed) {
@@ -102,6 +111,8 @@ pub struct IngestMetricsSnapshot {
pub dropped_by_policy: u64,
/// Captures dropped because their author may not write that repository.
pub dropped_unauthorized: u64,
+ /// Batch items dropped because they can never be stored (no session id).
+ pub dropped_invalid: u64,
/// Events shed because ingest capacity was exhausted.
pub shed_saturated: u64,
/// Events shed by the per-source rate limiter.
@@ -120,6 +131,7 @@ mod tests {
assert_eq!(snap.accepted, 0);
assert_eq!(snap.dropped_by_policy, 0);
assert_eq!(snap.dropped_unauthorized, 0);
+ assert_eq!(snap.dropped_invalid, 0);
assert_eq!(snap.shed_saturated, 0);
assert_eq!(snap.shed_rate_limited, 0);
assert_eq!(
@@ -161,6 +173,7 @@ mod tests {
vec![
"accepted",
"dropped_by_policy",
+ "dropped_invalid",
"dropped_unauthorized",
"last_persisted_ms",
"shed_rate_limited",
diff --git a/crates/ai-memory-hooks/src/router.rs b/crates/ai-memory-hooks/src/router.rs
index 659e761c..fc750100 100644
--- a/crates/ai-memory-hooks/src/router.rs
+++ b/crates/ai-memory-hooks/src/router.rs
@@ -853,6 +853,20 @@ async fn handle_hook_batch(
accepted_indices.push(idx);
continue;
}
+ // An event with no session id that is not a SessionStart can never be
+ // stored. Reporting it as failed made the drain retry it until its
+ // attempt budget ran out, stalling every item queued behind it (#1062),
+ // so it is acknowledged and dropped before it spends any capacity.
+ if resolve_session_id(&env).is_err() {
+ state.ingest_metrics.record_dropped_invalid();
+ warn!(
+ agent = %env.agent.as_str(),
+ event = ?env.event,
+ "hook batch item has no session id; dropped"
+ );
+ accepted_indices.push(idx);
+ continue;
+ }
let Ok(permit) = state.ingest_semaphore.clone().try_acquire_owned() else {
// The 429 rejects this item AND every item behind it, so count all
// of them: on `/hook` one 429 is one shed event, and a batch that
@@ -7417,28 +7431,40 @@ mod tests {
assert_eq!(ack["accepted_indices"], serde_json::json!([1]));
}
+ /// The acknowledgement a drain charges after earlier rate-limited skips:
+ /// nothing committed, the empty index list kept explicit, and the failed
+ /// item named so the drain charges it rather than the first unaccepted one.
+ #[test]
+ fn hook_batch_ack_names_the_failed_item_after_skips() {
+ let ack = serde_json::to_value(HookBatchAck::indexed_failed(Vec::new(), 1)).unwrap();
+ assert_eq!(ack["accepted"], 0);
+ assert_eq!(ack["accepted_indices"], serde_json::json!([]));
+ assert_eq!(ack["failed_index"], 1);
+ }
+
+ /// #1062: a session-less event (not a SessionStart) can never be stored.
+ /// Failing it made the drain retry it ahead of every valid event queued
+ /// behind it, so it is acknowledged, counted, and dropped instead.
#[tokio::test]
- async fn handle_hook_batch_reports_failed_index_after_rate_limited_skip() {
+ async fn handle_hook_batch_drops_a_session_less_item_and_keeps_draining() {
let tmp = TempDir::new().unwrap();
- let mut state = make_state(&tmp).await;
- let mut limiter = IngestRateLimiter::new(0.001, 1.0);
- assert!(limiter.try_take("u:\ns:flooder", std::time::Instant::now()));
- state.ingest_rate = Arc::new(tokio::sync::Mutex::new(limiter));
+ let state = Arc::new(make_state(&tmp).await);
+ let metrics = state.ingest_metrics.clone();
let response = handle_hook_batch(
- State(Arc::new(state)),
+ State(state.clone()),
None,
None,
None,
HeaderMap::new(),
Json(vec![
HookBatchItem {
- url: "http://h/hook?event=session-start&agent=claude-code".into(),
- body: serde_json::json!({ "session_id": "flooder" }),
+ url: "http://h/hook?event=post-tool-use&agent=hermes".into(),
+ body: serde_json::json!({ "tool_name": "Read" }),
},
HookBatchItem {
url: "http://h/hook?event=user-prompt-submit&agent=claude-code".into(),
- body: serde_json::json!({ "prompt": "missing session fails" }),
+ body: serde_json::json!({ "session_id": "behind-it", "prompt": "valid" }),
},
]),
)
@@ -7449,9 +7475,56 @@ mod tests {
.await
.unwrap();
let ack: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
- assert_eq!(ack["accepted"], 0);
- assert_eq!(ack["accepted_indices"], serde_json::json!([]));
- assert_eq!(ack["failed_index"], 1);
+ assert_eq!(ack["accepted"], 2, "both items leave the spool");
+ assert!(ack.get("failed_index").is_none() || ack["failed_index"].is_null());
+
+ let snap = metrics.snapshot();
+ assert_eq!(snap.dropped_invalid, 1);
+ assert_eq!(snap.accepted, 1, "only the valid item spends capacity");
+ assert!(
+ state
+ .reader
+ .session_project_ids(SessionId::from_native("behind-it"))
+ .await
+ .unwrap()
+ .is_some(),
+ "the valid item behind the dropped one is stored"
+ );
+ }
+
+ /// Control: a SessionStart without a session id is still processed (the
+ /// server mints one), so it is never counted as an invalid drop.
+ #[tokio::test]
+ async fn handle_hook_batch_still_processes_a_session_less_session_start() {
+ let tmp = TempDir::new().unwrap();
+ let state = make_state(&tmp).await;
+ let metrics = state.ingest_metrics.clone();
+
+ let response = handle_hook_batch(
+ State(Arc::new(state)),
+ None,
+ None,
+ None,
+ HeaderMap::new(),
+ Json(vec![HookBatchItem {
+ url: "http://h/hook?event=session-start&agent=claude-code".into(),
+ body: serde_json::json!({}),
+ }]),
+ )
+ .await
+ .into_response();
+ let bytes = axum::body::to_bytes(response.into_body(), usize::MAX)
+ .await
+ .unwrap();
+ let ack: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
+ assert_eq!(ack["accepted"], 1);
+ let snap = metrics.snapshot();
+ assert_eq!(snap.dropped_invalid, 0);
+ assert_eq!(snap.accepted, 1);
+ assert!(
+ snap.last_persisted_ms.is_some(),
+ "it went through the writer"
+ );
}
#[tokio::test]
diff --git a/docs/external-lifecycle.md b/docs/external-lifecycle.md
index b085d80b..49e988ae 100644
--- a/docs/external-lifecycle.md
+++ b/docs/external-lifecycle.md
@@ -144,7 +144,8 @@ Use the tuple recipe when event IDs have narrower scope.
- A rate-limited source can be skipped while other sources advance. Inspect
acknowledgements even on HTTP 429 or a partial failure; `failed_index` identifies
a processing failure after earlier skips. An acknowledgement can also mean a
- deliberate policy drop, not a new observation.
+ deliberate policy drop, or an event that can never be stored (no session id
+ on anything but a session start), not a new observation.
- On a timeout or lost response, retry unchanged items and keys. Back off on
saturation rather than opening an unbounded number of requests.