Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(hook): give 429 a free retry + add mid-session spool drain

Djalma Júnior6df0612f7d4e
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 403b2de5..87710db3 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -101,6 +101,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`403 user '' not allowed to purge_project`, making purge/move unusable on any
instance running scope-guard. `rename-project` is unaffected (it runs no
admission chain).
+- Hook spool no longer counts a server `429` (saturation / `hook queue full`)
+ against a spooled event's `MAX_ATTEMPTS` retry budget: transient backpressure
+ keeps the event queued without burning an attempt (`MAX_AGE_MS` still bounds
+ it), so a saturation burst no longer silently discards real observations.
+
+### Added
+- Mid-session hook-spool drain: on `post-tool-use`, once the local spool backlog
+ crosses a threshold, the hook runs a tightly time-boxed (~250 ms) catch-up
+ drain so a heavy session keeps the backlog flat instead of waiting for the next
+ session boundary. Tunable via `AI_MEMORY_HOOK_INCREMENTAL_THRESHOLD`
+ (default 32 events).
## [1.0.6] - 2026-06-14
diff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rs
index d28c9832..9095cec6 100644
--- a/crates/ai-memory-cli/src/commands/hook.rs
+++ b/crates/ai-memory-cli/src/commands/hook.rs
@@ -38,6 +38,15 @@ const HANDOFF_TIMEOUT_ENV: &str = "AI_MEMORY_HOOK_HANDOFF_TIMEOUT_MINUTES";
const START_BUDGET_ENV: &str = "AI_MEMORY_HOOK_START_BUDGET_MINUTES";
const END_BUDGET_ENV: &str = "AI_MEMORY_HOOK_END_BUDGET_MINUTES";
+const INCREMENTAL_THRESHOLD_ENV: &str = "AI_MEMORY_HOOK_INCREMENTAL_THRESHOLD";
+/// Backlog size at which `post-tool-use` does a mid-session catch-up drain, so a
+/// light session pays only a `read_dir`. Override via the env var above.
+const DEFAULT_INCREMENTAL_THRESHOLD: usize = 32;
+/// Total budget AND per-event timeout for the mid-session catch-up drain — kept
+/// well under a second so a `post-tool-use` hook never stalls a tool call (one
+/// in-flight POST against a slow server is bounded by this too).
+const INCREMENTAL_DRAIN_BUDGET: Duration = Duration::from_millis(250);
+
/// Per-event POST timeout during a drain. Env: `AI_MEMORY_HOOK_DRAIN_TIMEOUT_MINUTES`.
fn drain_event_timeout() -> Duration {
drain_event_timeout_from(env_lookup)
@@ -74,6 +83,27 @@ fn end_drain_budget_from(lookup: impl FnMut(&str) -> Option<String>) -> Duration
env_minutes(END_BUDGET_ENV, DEFAULT_END_BUDGET, lookup)
}
+/// Backlog size at which `post-tool-use` triggers a mid-session catch-up drain.
+/// Env: `AI_MEMORY_HOOK_INCREMENTAL_THRESHOLD` (positive integer).
+fn incremental_drain_threshold() -> usize {
+ incremental_drain_threshold_from(env_lookup)
+}
+
+fn incremental_drain_threshold_from(mut lookup: impl FnMut(&str) -> Option<String>) -> usize {
+ lookup(INCREMENTAL_THRESHOLD_ENV)
+ .and_then(|s| s.trim().parse::<usize>().ok())
+ .filter(|&n| n > 0)
+ .unwrap_or(DEFAULT_INCREMENTAL_THRESHOLD)
+}
+
+/// Whether to run a mid-session catch-up drain for this event: only
+/// `post-tool-use` (the highest-frequency event) and only once the spool backlog
+/// has crossed `threshold`. Boundaries (`session-start`/`session-end`) remain
+/// the main flush, so a light session never drains mid-session.
+fn should_incremental_drain(event: &str, spool_len: usize, threshold: usize) -> bool {
+ event == "post-tool-use" && spool_len >= threshold
+}
+
fn env_lookup(name: &str) -> Option<String> {
std::env::var(name).ok()
}
@@ -135,6 +165,25 @@ pub async fn run(data_dir: Option<PathBuf>, args: HookArgs) -> anyhow::Result<()
);
let _ = hook_spool::enqueue(&spool, &entry);
+ // Mid-session catch-up: per-event hooks only enqueue, so a heavy session
+ // outpaces the boundary-only drain and the spool grows until the next
+ // boundary. On `post-tool-use`, once the backlog crosses the threshold, do a
+ // tightly time-boxed drain (budget == per-event timeout, sub-second) so the
+ // spool stays flat without ever stalling a tool call.
+ if should_incremental_drain(
+ &args.event,
+ hook_spool::spool_len(&spool),
+ incremental_drain_threshold(),
+ ) {
+ let _ = hook_spool::drain(
+ &spool,
+ &dd,
+ INCREMENTAL_DRAIN_BUDGET,
+ INCREMENTAL_DRAIN_BUDGET,
+ )
+ .await;
+ }
+
// session-start: drain any backlog (e.g. from a previous session that ended
// abruptly), then fetch + inject the pending handoff for the resuming agent.
if args.event == "session-start" {
@@ -191,6 +240,38 @@ fn resolve_data_dir(data_dir: Option<&Path>) -> PathBuf {
mod tests {
use super::*;
+ #[test]
+ fn should_incremental_drain_only_post_tool_use_over_threshold() {
+ assert!(should_incremental_drain("post-tool-use", 32, 32));
+ assert!(should_incremental_drain("post-tool-use", 100, 32));
+ // below threshold: a light session never drains mid-session
+ assert!(!should_incremental_drain("post-tool-use", 31, 32));
+ // other events only enqueue; boundaries do the real flush
+ assert!(!should_incremental_drain("pre-tool-use", 999, 32));
+ assert!(!should_incremental_drain("session-start", 999, 32));
+ assert!(!should_incremental_drain("session-end", 999, 32));
+ assert!(!should_incremental_drain("stop", 999, 32));
+ }
+
+ #[test]
+ fn incremental_threshold_parses_and_falls_back() {
+ assert_eq!(incremental_drain_threshold_from(|_| Some("64".into())), 64);
+ assert_eq!(
+ incremental_drain_threshold_from(|_| None),
+ DEFAULT_INCREMENTAL_THRESHOLD
+ );
+ // zero / non-numeric fall back to the default (a 0 threshold would drain
+ // on every post-tool-use)
+ assert_eq!(
+ incremental_drain_threshold_from(|_| Some("0".into())),
+ DEFAULT_INCREMENTAL_THRESHOLD
+ );
+ assert_eq!(
+ incremental_drain_threshold_from(|_| Some("abc".into())),
+ DEFAULT_INCREMENTAL_THRESHOLD
+ );
+ }
+
#[test]
fn parse_minutes_falls_back_on_invalid() {
assert_eq!(
diff --git a/crates/ai-memory-cli/src/commands/hook_capture.rs b/crates/ai-memory-cli/src/commands/hook_capture.rs
index f18a6037..386f55bc 100644
--- a/crates/ai-memory-cli/src/commands/hook_capture.rs
+++ b/crates/ai-memory-cli/src/commands/hook_capture.rs
@@ -110,19 +110,35 @@ pub fn build_client() -> reqwest::Client {
.unwrap_or_else(|_| reqwest::Client::new())
}
-/// POST the payload as JSON, best-effort. Returns `true` when the server
-/// acknowledged with a 2xx (the engine answers `202 queued`), `false` on any
-/// non-2xx or transport error — never errors, so a hook/flush never fails the
-/// agent. `timeout` is caller-chosen: the per-tool-call hot path no longer
-/// POSTs at all (it spools); only the session-boundary drain calls this, with a
-/// generous budget that tolerates a remote/slow server.
+/// Outcome of one spooled-event POST — enough for the drain loop to decide
+/// whether a miss should cost the entry a retry attempt. Never errors, so a
+/// hook/flush never fails the agent.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum PostOutcome {
+ /// Server acknowledged with a 2xx (the engine answers `202 queued`): the
+ /// entry was delivered and can be removed from the spool.
+ Delivered,
+ /// Server answered `429 Too Many Requests` (`hook queue full`): transient
+ /// backpressure, the event was never processed. Keep it queued WITHOUT
+ /// bumping attempts so saturation never burns the entry's retry budget.
+ Saturated,
+ /// Any other non-2xx, or a transport error: a genuine miss that should
+ /// count against `MAX_ATTEMPTS`.
+ Failed,
+}
+
+/// POST the payload as JSON, best-effort. `timeout` is caller-chosen: the
+/// per-tool-call hot path no longer POSTs at all (it spools); the drain calls
+/// this with a budget that tolerates a remote/slow server. Returns a
+/// [`PostOutcome`] (never errors) so the drain can give a 429 (saturation) a
+/// free retry while still bounding genuine failures by `MAX_ATTEMPTS`.
pub async fn post_hook(
client: &reqwest::Client,
url: &str,
body: &str,
token: Option<&str>,
timeout: Duration,
-) -> bool {
+) -> PostOutcome {
let mut req = client
.post(url)
.header("Content-Type", "application/json")
@@ -132,8 +148,12 @@ pub async fn post_hook(
req = req.bearer_auth(t);
}
match req.send().await {
- Ok(resp) => resp.status().is_success(),
- Err(_) => false,
+ Ok(resp) if resp.status().is_success() => PostOutcome::Delivered,
+ Ok(resp) if resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS => {
+ PostOutcome::Saturated
+ }
+ Ok(_) => PostOutcome::Failed,
+ Err(_) => PostOutcome::Failed,
}
}
@@ -205,11 +225,11 @@ mod tests {
}
#[tokio::test]
- async fn post_hook_returns_false_when_server_unreachable() {
- // Port 1 is unroutable; best-effort means this resolves to `false`
- // (no 2xx) rather than panicking or erroring.
+ async fn post_hook_failed_when_server_unreachable() {
+ // Port 1 is unroutable; best-effort means this resolves to `Failed`
+ // (a genuine miss) rather than panicking or erroring.
let client = build_client();
- let ok = post_hook(
+ let outcome = post_hook(
&client,
"http://127.0.0.1:1/hook?event=pre-tool-use",
"{}",
@@ -217,7 +237,23 @@ mod tests {
Duration::from_millis(500),
)
.await;
- assert!(!ok);
+ assert_eq!(outcome, PostOutcome::Failed);
+ }
+
+ #[tokio::test]
+ async fn post_hook_saturated_on_429() {
+ // 429 = server backpressure; the event was never processed, so the
+ // drain must treat it as a free retry, not a failed attempt.
+ let url = serve_once("429 Too Many Requests", "hook queue full").await;
+ let outcome = post_hook(&build_client(), &url, "{}", None, Duration::from_secs(1)).await;
+ assert_eq!(outcome, PostOutcome::Saturated);
+ }
+
+ #[tokio::test]
+ async fn post_hook_delivered_on_2xx() {
+ let url = serve_once("202 Accepted", "queued").await;
+ let outcome = post_hook(&build_client(), &url, "{}", None, Duration::from_secs(1)).await;
+ assert_eq!(outcome, PostOutcome::Delivered);
}
#[tokio::test]
diff --git a/crates/ai-memory-cli/src/commands/hook_spool.rs b/crates/ai-memory-cli/src/commands/hook_spool.rs
index 940ef4ba..b714aa1e 100644
--- a/crates/ai-memory-cli/src/commands/hook_spool.rs
+++ b/crates/ai-memory-cli/src/commands/hook_spool.rs
@@ -25,7 +25,7 @@ use ai_memory_llm::{OidcToken, refresh_access_token};
use secrecy::ExposeSecret as _;
use serde::{Deserialize, Serialize};
-use super::hook_capture::{build_client, post_hook};
+use super::hook_capture::{PostOutcome, build_client, post_hook};
/// Drop a spooled event after this many failed drain passes — bounds retries of
/// a permanently-undeliverable event (e.g. a server URL that never comes back).
@@ -81,6 +81,14 @@ pub fn spool_dir(data_dir: &Path) -> PathBuf {
data_dir.join("hook-spool")
}
+/// Count queued spool entries (`*.json`), or 0 when the dir is missing/empty.
+/// Cheaper than [`drain`] — a single `read_dir` — so the per-event hot path can
+/// gate a mid-session drain on backlog size without building a client.
+#[must_use]
+pub fn spool_len(spool: &Path) -> usize {
+ list_entries(spool).map_or(0, |f| f.len())
+}
+
fn now_ms() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
@@ -216,7 +224,7 @@ pub async fn drain(
}
};
- if post_hook(
+ match post_hook(
&client,
&entry.url,
&entry.body,
@@ -225,18 +233,28 @@ pub async fn drain(
)
.await
{
- let _ = std::fs::remove_file(&path);
- result.sent += 1;
- } else {
- entry.attempts = entry.attempts.saturating_add(1);
- if entry.attempts >= MAX_ATTEMPTS {
+ PostOutcome::Delivered => {
let _ = std::fs::remove_file(&path);
- result.dropped += 1;
- } else {
- // Persist the bumped attempt count for the next boundary.
- let _ = rewrite_entry(&path, &entry);
+ result.sent += 1;
+ }
+ // Transient server backpressure (429 `hook queue full`): the event
+ // was never processed, so leave it untouched — no attempt bump, no
+ // rewrite — and saturation never burns the entry's retry budget. It
+ // rides the next pass unchanged; `MAX_AGE_MS` still bounds it.
+ PostOutcome::Saturated => {
result.remaining += 1;
}
+ PostOutcome::Failed => {
+ entry.attempts = entry.attempts.saturating_add(1);
+ if entry.attempts >= MAX_ATTEMPTS {
+ let _ = std::fs::remove_file(&path);
+ result.dropped += 1;
+ } else {
+ // Persist the bumped attempt count for the next boundary.
+ let _ = rewrite_entry(&path, &entry);
+ result.remaining += 1;
+ }
+ }
}
}
result
@@ -475,4 +493,80 @@ mod tests {
assert_eq!(r.sent, 0);
assert!(list_entries(&spool).unwrap().is_empty());
}
+
+ #[tokio::test]
+ async fn drain_429_keeps_event_queued_without_bumping_attempts() {
+ // A server that always answers 429 (saturation / `hook queue full`).
+ // The event must ride every pass untouched: never dropped, attempts
+ // never incremented — saturation must not burn the retry budget.
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ tokio::spawn(async move {
+ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+ while let Ok((mut s, _)) = listener.accept().await {
+ tokio::spawn(async move {
+ let mut buf = [0_u8; 1024];
+ let _ = s.read(&mut buf).await;
+ let _ = s
+ .write_all(
+ b"HTTP/1.1 429 Too Many Requests\r\nContent-Length: 4\r\nConnection: close\r\n\r\nfull",
+ )
+ .await;
+ });
+ }
+ });
+
+ let tmp = tempfile::tempdir().unwrap();
+ let spool = spool_dir(tmp.path());
+ let e = entry_for(
+ format!("http://{addr}/hook?event=x"),
+ "{}".into(),
+ None,
+ false,
+ );
+ enqueue(&spool, &e).unwrap();
+
+ // Far more passes than MAX_ATTEMPTS — a 429 must never consume budget.
+ for _ in 0..(MAX_ATTEMPTS + 2) {
+ let r = drain(
+ &spool,
+ tmp.path(),
+ Duration::from_secs(2),
+ Duration::from_millis(500),
+ )
+ .await;
+ assert_eq!(r.sent, 0);
+ assert_eq!(r.dropped, 0, "a 429 must never drop the event");
+ assert_eq!(r.remaining, 1);
+ }
+ let files = list_entries(&spool).unwrap();
+ assert_eq!(files.len(), 1, "event still queued after many 429s");
+ let loaded: SpoolEntry =
+ serde_json::from_slice(&std::fs::read(&files[0]).unwrap()).unwrap();
+ assert_eq!(loaded.attempts, 0, "429 must not consume the retry budget");
+ }
+
+ #[test]
+ fn spool_len_counts_entries() {
+ let tmp = tempfile::tempdir().unwrap();
+ let spool = spool_dir(tmp.path());
+ assert_eq!(spool_len(&spool), 0, "missing dir counts as 0");
+ std::fs::create_dir_all(&spool).unwrap();
+ // Write distinct files directly (enqueue's ms+pid names would collide in
+ // a tight loop, and its prune caps at the test MAX_SPOOL_FILES).
+ for i in 0..3 {
+ let e = entry_for(
+ format!("http://x/hook?event=e{i}"),
+ "{}".into(),
+ None,
+ false,
+ );
+ std::fs::write(
+ spool.join(format!("evt-{i}.json")),
+ serde_json::to_vec(&e).unwrap(),
+ )
+ .unwrap();
+ }
+ assert_eq!(spool_len(&spool), 3);
+ }
}