Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
fix(spool): recover hook events stranded by a rotated auth token (#542) (#551)
Fabio Akita52279607def1
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 953423fb..92f3eb36 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -153,6 +153,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0three destructive commands in one place instead of documenting the limits ofone and overstating the other two.+- The hook-spool drain can now recover events stranded by a rotated auth token+ (#542). A spooled envelope freezes the bearer it was captured with, so+ rotating the token left every already-spooled entry authenticating with the+ old one and failing `401` forever; OIDC already re-resolved per pass, static+ tokens did not.++ The drain now retries a rejected entry once with the token it was handed by+ the hook that spawned it, which is current by construction. Bounded three+ ways: only after the server has already answered `401`, only for a `Static`+ entry, and only when the live token actually differs from the one that just+ failed — so a healthy drain never pays for it and an identical credential is+ not re-sent.++ The token reaches the drain in the child process's **environment**, never on+ its command line: an argument would publish the credential to the process+ table for every local user for as long as the drain runs. A test pins that it+ stays out of argv. `401` also became a distinct `Unauthorized` outcome —+ previously indistinguishable from any other refusal, so the drain could not+ tell a stale credential from a bad event.++ No new credential is stored at rest: the drain is handed a token that already+ exists rather than reading one from a file ai-memory would have to write and+ protect.+### Docs- `docs/windows.md` gains **Scenario E**, a persistent-server story for nativediff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rsindex 0e61e7ff..e9473d18 100644--- a/crates/ai-memory-cli/src/commands/hook.rs+++ b/crates/ai-memory-cli/src/commands/hook.rs@@ -124,8 +124,8 @@ fn should_incremental_drain(event: &str, spool_len: usize, threshold: usize) ->event == "post-tool-use" && spool_len >= threshold}-fn spawn_background_drainer(data_dir: &Path) -> std::io::Result<()> {- hook_drain_process::spawn(data_dir)+fn spawn_background_drainer(data_dir: &Path, live_token: Option<&str>) -> std::io::Result<()> {+ hook_drain_process::spawn(data_dir, live_token)}fn should_spawn_background_drainer(event: &str) -> bool {@@ -299,9 +299,10 @@ fn cwd_query_suffix(fn after_background_drain_event_enqueue(data_dir: &Path,- spawn: impl FnOnce(&Path) -> std::io::Result<()>,+ live_token: Option<&str>,+ spawn: impl FnOnce(&Path, Option<&str>) -> std::io::Result<()>,) -> std::io::Result<()> {- spawn(data_dir)+ spawn(data_dir, live_token)}/// Hidden drain-only fast path. Reads no stdin and writes no stdout.@@ -477,7 +478,7 @@ async fn run_with_payload<W, S>() -> anyhow::Result<()>whereW: std::io::Write,- S: FnOnce(&Path) -> std::io::Result<()>,+ S: FnOnce(&Path, Option<&str>) -> std::io::Result<()>,{let agent_kind = AgentKind::from_wire(&args.agent);let hook_event = HookEvent::parse(&args.event);@@ -794,7 +795,15 @@ where// but `stop` and `pre-compact` also trigger the helper so delivery does not// rely on the single hook most likely to be cancelled during agent shutdown.if should_spawn_background_drainer(&args.event)- && let Err(err) = after_background_drain_event_enqueue(&dd, spawn_background_drainer)+ && let Err(err) = after_background_drain_event_enqueue(+ &dd,+ // The hook's own token: current by definition, since the agent+ // config it came from was rendered by the last `install-hooks`.+ // The drain uses it only to retry an entry the server has already+ // rejected with 401 (#542).+ args.auth_token.as_deref(),+ spawn_background_drainer,+ ){eprintln!("ai-memory hook warning: failed to start background spool drainer; event remains queued: {err}"@@ -1191,7 +1200,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1210,7 +1219,7 @@ mod tests {antigravity_hook_args("pre-tool-use", "http://127.0.0.1:1"),"not-json".into(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1338,7 +1347,7 @@ mod tests {devin_hook_args("session-start"),payload.to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1378,7 +1387,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1430,7 +1439,7 @@ mod tests {devin_hook_args("post-tool-use"),payload.to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1468,7 +1477,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1484,7 +1493,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1627,7 +1636,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1640,7 +1649,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1677,7 +1686,7 @@ mod tests {args,r#"{"session_id":"s","cwd":"/tmp"}"#.into(),&mut stdout,- |path| {+ |path, _token| {assert_eq!(path, data_dir.as_path());assert_eq!(hook_spool::spool_len(&spool), 1, "spawn runs after enqueue");called.set(called.get() + 1);@@ -1720,7 +1729,7 @@ mod tests {args,r#"{"session_id":"s","cwd":"/tmp"}"#.into(),&mut stdout,- |path| {+ |path, _token| {assert_eq!(path, data_dir.as_path());assert_eq!(hook_spool::spool_len(&spool), 1, "spawn runs after enqueue");called.set(called.get() + 1);@@ -1757,9 +1766,13 @@ mod tests {capture_mode: None,};- run_with_payload(Some(data_dir), args, "{}".into(), &mut stdout, |_path| {- Err(std::io::Error::other("spawn failed"))- })+ run_with_payload(+ Some(data_dir),+ args,+ "{}".into(),+ &mut stdout,+ |_path, _token| Err(std::io::Error::other("spawn failed")),+ ).await.unwrap();@@ -1790,7 +1803,7 @@ mod tests {args,r#"{"hook_event_name":"SessionEnd"}"#.into(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -1814,7 +1827,7 @@ mod tests {#[test]fn session_end_spawn_failure_is_returned_for_warning_only() {let tmp = tempfile::tempdir().unwrap();- let err = after_background_drain_event_enqueue(tmp.path(), |_path| {+ let err = after_background_drain_event_enqueue(tmp.path(), None, |_path, _token| {Err(std::io::Error::other("spawn failed"))}).unwrap_err();@@ -1827,7 +1840,7 @@ mod tests {let tmp = tempfile::tempdir().unwrap();let called = std::cell::Cell::new(false);- after_background_drain_event_enqueue(tmp.path(), |path| {+ after_background_drain_event_enqueue(tmp.path(), None, |path, _token| {assert_eq!(path, tmp.path());called.set(true);Ok(())@@ -1877,7 +1890,7 @@ mod tests {let called = std::cell::Cell::new(false);let mut args = devin_hook_args("post-tool-use");args.server_url = "http://127.0.0.1:1".into();- run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"secret/SENTINEL"}}).to_string(), &mut stdout, |_| { called.set(true); Ok(()) }).await.unwrap();+ run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"secret/SENTINEL"}}).to_string(), &mut stdout, |_, _| { called.set(true); Ok(()) }).await.unwrap();assert_eq!(stdout, b"{}\n");assert!(!called.get());assert_eq!(hook_spool::spool_len(&hook_spool::spool_dir(&data_dir)), 0);@@ -1912,7 +1925,7 @@ mod tests {args,raw.to_string(),&mut stdout,- |_| {+ |_, _| {called.set(true);Ok(())},@@ -1953,7 +1966,7 @@ mod tests {args,raw.to_string(),&mut stdout,- |_| {+ |_, _| {called.set(true);Ok(())},@@ -1993,7 +2006,7 @@ mod tests {}).to_string(),&mut stdout,- |_| {+ |_, _| {called.set(true);Ok(())},@@ -2016,7 +2029,7 @@ mod tests {.unwrap();let data_dir = tmp.path().join("data");let mut stdout = Vec::new();- run_with_payload(Some(data_dir.clone()), devin_hook_args("post-tool-use"), serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL_PATH","args":"SENTINEL_ARGS"},"output":"SENTINEL_OUTPUT","error":"SENTINEL_ERROR","nested":{"raw":"SENTINEL_NESTED"}}).to_string(), &mut stdout, |_| Ok(())).await.unwrap();+ run_with_payload(Some(data_dir.clone()), devin_hook_args("post-tool-use"), serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL_PATH","args":"SENTINEL_ARGS"},"output":"SENTINEL_OUTPUT","error":"SENTINEL_ERROR","nested":{"raw":"SENTINEL_NESTED"}}).to_string(), &mut stdout, |_, _| Ok(())).await.unwrap();let entry = read_spooled_entries(&hook_spool::spool_dir(&data_dir)).pop().unwrap();@@ -2039,7 +2052,7 @@ mod tests {let mut args = devin_hook_args("post-tool-use");args.check_capture = true;let mut stdout = Vec::new();- run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL"}}).to_string(), &mut stdout, |_| Err(std::io::Error::other("must not spawn"))).await.unwrap();+ run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL"}}).to_string(), &mut stdout, |_, _| Err(std::io::Error::other("must not spawn"))).await.unwrap();let output: serde_json::Value = serde_json::from_slice(&stdout).unwrap();// Pin the exact key set, not just its size: `--check-capture` is how an// operator verifies an opt-out, so a field silently appearing or@@ -2082,7 +2095,7 @@ mod tests {devin_hook_args("post-tool-use"),inactive_payload.clone(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2103,7 +2116,7 @@ mod tests {serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"public"}}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2134,7 +2147,7 @@ mod tests {args,raw.to_string(),&mut stdout,- |_| Err(std::io::Error::other("must not spawn")),+ |_, _| Err(std::io::Error::other("must not spawn")),).await.unwrap();@@ -2167,7 +2180,7 @@ mod tests {devin_hook_args("post-tool-use"),raw.to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2274,7 +2287,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2322,7 +2335,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Err(std::io::Error::other("must not spawn")),+ |_, _| Err(std::io::Error::other("must not spawn")),).await.unwrap();@@ -2346,7 +2359,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2381,7 +2394,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2412,7 +2425,7 @@ mod tests {serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path(), "prompt": "hi"}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2429,7 +2442,7 @@ mod tests {serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path(), "prompt": "hi"}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2449,7 +2462,7 @@ mod tests {serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path(), "prompt": "hi"}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2475,7 +2488,7 @@ mod tests {kimi_hook_args("session-start", &base),serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path()}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2500,7 +2513,7 @@ mod tests {args,serde_json::json!({"session_id": "claude-session", "cwd": tmp.path()}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2545,7 +2558,7 @@ mod tests {kimi_hook_args("user-prompt-submit", &base),payload.clone(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2566,7 +2579,7 @@ mod tests {kimi_hook_args("user-prompt-submit", &base),payload,&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2598,7 +2611,7 @@ mod tests {kimi_hook_args("user-prompt", &base),payload.clone(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2617,7 +2630,7 @@ mod tests {kimi_hook_args("user-prompt", &base),payload,&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();@@ -2646,7 +2659,7 @@ mod tests {}).to_string(),&mut stdout,- |_| Ok(()),+ |_, _| Ok(()),).await.unwrap();diff --git a/crates/ai-memory-cli/src/commands/hook_capture.rs b/crates/ai-memory-cli/src/commands/hook_capture.rsindex 04eca713..1b84908e 100644--- a/crates/ai-memory-cli/src/commands/hook_capture.rs+++ b/crates/ai-memory-cli/src/commands/hook_capture.rs@@ -344,6 +344,13 @@ pub enum PostOutcome {/// so the drain can skip past them instead of stopping at the first one/// (#493).Unreachable,+ /// `401` — the server rejected the bearer. Distinguished from+ /// [`Self::Failed`] because it says something about the *credential*+ /// rather than the entry: a spooled event carries the token frozen at+ /// capture time, so after a rotation every entry captured under the old+ /// one fails identically and is recoverable by retrying with the current+ /// token (#542).+ Unauthorized,}/// POST the payload as JSON, best-effort. `timeout` is caller-chosen: the@@ -371,6 +378,7 @@ pub async fn post_hook(Ok(resp) if resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS => {PostOutcome::Saturated}+ Ok(resp) if resp.status() == reqwest::StatusCode::UNAUTHORIZED => PostOutcome::Unauthorized,Ok(_) => PostOutcome::Failed,Err(_) => PostOutcome::Unreachable,}@@ -409,6 +417,10 @@ pub enum BatchOutcome {/// `404`/`405` — the server has no `/hook/batch` (a pre-upgrade build). The/// caller falls back to per-event `POST /hook` for the rest of the drain.Unsupported,+ /// `401` — the server rejected the batch's bearer. See+ /// [`PostOutcome::Unauthorized`]; the whole batch shares one identity, so+ /// a rejected credential fails all of it and none of it was processed.+ Unauthorized,/// Transport error or any other non-2xx: the batch outcome is unknown. The/// drain charges conservatively so trailing events that may never have been/// attempted do not burn retry budget.@@ -474,6 +486,8 @@ pub async fn post_batch(|| status == reqwest::StatusCode::METHOD_NOT_ALLOWED{BatchOutcome::Unsupported+ } else if status == reqwest::StatusCode::UNAUTHORIZED {+ BatchOutcome::Unauthorized} else {BatchOutcome::Failed}diff --git a/crates/ai-memory-cli/src/commands/hook_drain_process.rs b/crates/ai-memory-cli/src/commands/hook_drain_process.rsindex 9607c9ba..7f38cc84 100644--- a/crates/ai-memory-cli/src/commands/hook_drain_process.rs+++ b/crates/ai-memory-cli/src/commands/hook_drain_process.rs@@ -36,6 +36,13 @@ pub struct DrainCommandSpec {pub args: Vec<OsString>,/// Stderr log file under `<data_dir>/logs`.pub stderr_log: PathBuf,+ /// Current static bearer to hand the child in its environment, when the+ /// spawning hook has one.+ ///+ /// The environment rather than an argument: a flag would put the+ /// credential in the process table for the lifetime of the drain, readable+ /// by anyone who can run `ps`.+ pub live_token: Option<OsString>,}/// Unit-testable stdio/detach configuration for a drainer process.@@ -67,7 +74,12 @@ pub enum DetachConfig {}/// Build `ai-memory --data-dir <dir> hook-drain`.-pub fn command_spec(data_dir: &Path) -> io::Result<DrainCommandSpec> {+///+/// `live_token` is the bearer the spawning hook is currently configured with.+/// A spooled envelope froze its own credential at capture time, so handing the+/// drain a current one is what lets it recover entries stranded by a rotation+/// (#542).+pub fn command_spec(data_dir: &Path, live_token: Option<&str>) -> io::Result<DrainCommandSpec> {Ok(DrainCommandSpec {exe: std::env::current_exe()?,args: vec![@@ -76,6 +88,7 @@ pub fn command_spec(data_dir: &Path) -> io::Result<DrainCommandSpec> {OsString::from("hook-drain"),],stderr_log: data_dir.join("logs").join("hook-drain.log"),+ live_token: live_token.filter(|t| !t.is_empty()).map(OsString::from),})}@@ -91,11 +104,19 @@ pub fn spawn_config(spec: &DrainCommandSpec, try_breakaway: bool) -> SpawnConfig}/// Spawn the hidden drainer without inheriting hook stdio.-pub fn spawn(data_dir: &Path) -> io::Result<()> {- let spec = command_spec(data_dir)?;+pub fn spawn(data_dir: &Path, live_token: Option<&str>) -> io::Result<()> {+ let spec = command_spec(data_dir, live_token)?;spawn_spec(&spec)}+/// Hand the child the current bearer in its environment, when there is one.+/// Applied to every spawn path so the breakaway-retry cannot silently drop it.+fn apply_live_token(command: &mut Command, spec: &DrainCommandSpec) {+ if let Some(token) = &spec.live_token {+ command.env(crate::commands::hook_spool::LIVE_TOKEN_ENV, token);+ }+}+fn spawn_spec(spec: &DrainCommandSpec) -> io::Result<()> {if let Some(parent) = spec.stderr_log.parent() {std::fs::create_dir_all(parent)?;@@ -108,6 +129,7 @@ fn spawn_spec(spec: &DrainCommandSpec) -> io::Result<()> {.open(&config.stderr_file)?;let mut command = command_for_spec(spec, &config.detach);+ apply_live_token(&mut command, spec);command.stdin(Stdio::null()).stdout(Stdio::null())@@ -130,6 +152,7 @@ fn spawn_spec(spec: &DrainCommandSpec) -> io::Result<()> {.append(true).open(&config.stderr_file)?;let mut retry = command_for_spec(spec, &config.detach);+ apply_live_token(&mut retry, spec);retry.stdin(Stdio::null()).stdout(Stdio::null())@@ -234,7 +257,7 @@ mod tests {#[test]fn command_spec_uses_data_dir_then_hidden_subcommand() {let tmp = tempfile::tempdir().unwrap();- let spec = command_spec(tmp.path()).unwrap();+ let spec = command_spec(tmp.path(), None).unwrap();assert_eq!(spec.args,@@ -277,10 +300,52 @@ mod tests {rotate_oversized_log(&log, 16);}+ /// The credential must reach the drain in its environment and nowhere+ /// near its argv. A command line is world-readable through the process+ /// table for as long as the process lives, so passing the token as a flag+ /// would publish it to every local user each time a session ends.+ #[test]+ fn the_live_token_travels_in_the_environment_not_the_command_line() {+ let tmp = tempfile::tempdir().unwrap();+ let spec = command_spec(tmp.path(), Some("SECRET-BEARER")).unwrap();++ assert_eq!(+ spec.live_token.as_deref(),+ Some(std::ffi::OsStr::new("SECRET-BEARER"))+ );+ let rendered: Vec<String> = spec+ .args+ .iter()+ .map(|a| a.to_string_lossy().into_owned())+ .collect();+ assert!(+ !rendered.iter().any(|a| a.contains("SECRET-BEARER")),+ "token leaked into argv: {rendered:?}"+ );+ assert!(+ rendered.iter().any(|a| a == "hook-drain"),+ "still the drain command: {rendered:?}"+ );+ }++ /// An absent or empty token carries nothing, so a no-auth deployment does+ /// not export an empty bearer into the child.+ #[test]+ fn an_absent_or_empty_token_is_not_carried() {+ let tmp = tempfile::tempdir().unwrap();+ assert!(command_spec(tmp.path(), None).unwrap().live_token.is_none());+ assert!(+ command_spec(tmp.path(), Some(""))+ .unwrap()+ .live_token+ .is_none()+ );+ }+#[test]fn spawn_config_redirects_stdio_and_logs_under_data_dir_logs() {let tmp = tempfile::tempdir().unwrap();- let spec = command_spec(tmp.path()).unwrap();+ let spec = command_spec(tmp.path(), None).unwrap();let config = spawn_config(&spec, true);assert!(config.stdin_null);@@ -295,7 +360,7 @@ mod tests {#[test]fn unix_spawn_config_prefers_true_session_detach_when_available() {let tmp = tempfile::tempdir().unwrap();- let spec = command_spec(tmp.path()).unwrap();+ let spec = command_spec(tmp.path(), None).unwrap();let DetachConfig::UnixNewSession { setsid } = spawn_config(&spec, true).detach;if let Some(path) = setsid {assert!(@@ -309,7 +374,7 @@ mod tests {#[test]fn windows_spawn_config_uses_expected_flags_and_breakaway_toggle() {let tmp = tempfile::tempdir().unwrap();- let spec = command_spec(tmp.path()).unwrap();+ let spec = command_spec(tmp.path(), None).unwrap();let DetachConfig::WindowsCreationFlags(with_breakaway) = spawn_config(&spec, true).detach;let DetachConfig::WindowsCreationFlags(without_breakaway) =spawn_config(&spec, false).detach;diff --git a/crates/ai-memory-cli/src/commands/hook_spool.rs b/crates/ai-memory-cli/src/commands/hook_spool.rsindex 0f91d919..c792f4bc 100644--- a/crates/ai-memory-cli/src/commands/hook_spool.rs+++ b/crates/ai-memory-cli/src/commands/hook_spool.rs@@ -508,6 +508,29 @@ pub async fn drain(data_dir: &Path,total_budget: Duration,per_event_timeout: Duration,+) -> DrainResult {+ drain_with_live_token(+ spool,+ data_dir,+ total_budget,+ per_event_timeout,+ live_static_token().as_deref(),+ )+ .await+}++/// [`drain`] with the current static bearer supplied explicitly.+///+/// The environment is read once, at the boundary in [`drain`], and passed down+/// as data. Tests drive this directly: mutating process-wide environment from a+/// test would race every other test in the binary, and `set_var` is `unsafe` in+/// this edition — which this workspace forbids outright.+pub async fn drain_with_live_token(+ spool: &Path,+ data_dir: &Path,+ total_budget: Duration,+ per_event_timeout: Duration,+ live_token: Option<&str>,) -> DrainResult {let (mut files, unreadable) = match list_entries(spool) {Ok(listed) => listed,@@ -722,6 +745,32 @@ pub async fn drain(bump_or_drop(path, entry, &mut result);unreachable_endpoints.insert(batch_endpoint(&entry.url));}+ PostOutcome::Unauthorized => {+ // Credential rejected, not the entry. Retry once+ // with this drain's live token (#542).+ let retry =+ static_retry_token(entry, item_bearer.as_deref(), live_token);+ let recovered = match retry {+ Some(token) => matches!(+ post_hook(+ &client,+ &entry.url,+ &entry.body,+ Some(token),+ per_event_timeout,+ )+ .await,+ PostOutcome::Delivered+ ),+ None => false,+ };+ if recovered {+ let _ = std::fs::remove_file(path);+ result.sent += 1;+ } else {+ bump_or_drop(path, entry, &mut result);+ }+ }}}}@@ -791,6 +840,49 @@ pub async fn drain(unreachable_endpoints.insert(base.clone());continue;}+ BatchOutcome::Unauthorized => {+ // The envelope froze its credential at capture time, so a+ // rotated token leaves every entry captured under the old+ // one failing identically — the credential half of the+ // same problem #493 fixed for the address (#542).+ //+ // Retry once with the token this drain was handed, which is+ // current by construction. Runs only for a batch that has+ // already been rejected, so a healthy drain never pays for+ // it.+ let retry = static_retry_token(&chunk[0].1, bearer.as_deref(), live_token);+ if let Some(token) = retry {+ match post_batch(&client, &base, &payload, Some(token), batch_timeout).await+ {+ BatchOutcome::Accepted(k) => {+ let k = k.min(chunk.len());+ for (path, _) in &chunk[..k] {+ let _ = std::fs::remove_file(path);+ }+ result.sent += k;+ result.remaining += chunk.len().saturating_sub(k);+ continue;+ }+ BatchOutcome::AcceptedIndices { indices, .. } => {+ let sent = delete_accepted_indices(&chunk, &indices);+ result.sent += sent;+ result.remaining += chunk.len().saturating_sub(sent);+ continue;+ }+ // The current token is no better — the server is+ // rejecting more than a stale credential. Fall+ // through and charge conservatively.+ _ => {}+ }+ }++ // Same conservative accounting as `Failed`: the server+ // refused the whole batch, so nothing in it was processed.+ bump_or_drop(&chunk[0].0, &chunk[0].1, &mut result);+ result.remaining +=+ chunk.len().saturating_sub(1) + files.len().saturating_sub(idx);+ break;+ }BatchOutcome::Failed => {// The batch didn't land (server answered with an unexpected// status, or the body did not parse).@@ -831,6 +923,31 @@ pub async fn drain(bump_or_drop(&path, &entry, &mut result);unreachable_endpoints.insert(batch_endpoint(&entry.url));}+ PostOutcome::Unauthorized => {+ // Credential rejected, not the entry. Retry once with this+ // drain's live token (#542).+ let retry = static_retry_token(&entry, bearer.as_deref(), live_token);+ let recovered = match retry {+ Some(token) => matches!(+ post_hook(+ &client,+ &entry.url,+ &entry.body,+ Some(token),+ per_event_timeout,+ )+ .await,+ PostOutcome::Delivered+ ),+ None => false,+ };+ if recovered {+ let _ = std::fs::remove_file(&path);+ result.sent += 1;+ } else {+ bump_or_drop(&path, &entry, &mut result);+ }+ }}}}@@ -926,6 +1043,47 @@ async fn entry_bearer(}}+/// Environment variable carrying the *current* static bearer into a spawned+/// `hook-drain`.+///+/// Deliberately the environment and not a command-line flag: the drain is a+/// separate process, and an argument would put the credential in the process+/// table for anyone running `ps` while it runs.+pub(crate) const LIVE_TOKEN_ENV: &str = "AI_MEMORY_AUTH_TOKEN";++/// The static bearer this drain process was handed, if any.+///+/// `hook-drain` is spawned by a lifecycle hook that already holds the current+/// token, and passes it down in the child's environment. A drain started by+/// hand, or by a hook running against a server that needs no auth, simply has+/// none — and the re-resolve below is skipped rather than guessed at.+fn live_static_token() -> Option<String> {+ std::env::var(LIVE_TOKEN_ENV).ok().filter(|t| !t.is_empty())+}++/// The token to retry a `401` with, or `None` when retrying is pointless.+///+/// A spooled envelope freezes its credential at capture time, so rotating the+/// token leaves every already-spooled entry authenticating with the old one+/// (#542). The drain process, by contrast, was handed the current token by the+/// hook that spawned it moments ago.+///+/// Two guards keep this from turning into a blind second attempt: only+/// `Static` entries qualify — an `Oidc` bearer is already resolved and+/// refreshed once per pass, so a 401 there is a real rejection — and the live+/// token must actually differ from the one that just failed. Retrying an+/// identical credential would only repeat the failure at double the cost.+fn static_retry_token<'a>(+ entry: &SpoolEntry,+ rejected: Option<&str>,+ live: Option<&'a str>,+) -> Option<&'a str> {+ if entry.auth_mode != AuthMode::Static {+ return None;+ }+ live.filter(|t| Some(*t) != rejected)+}+/// Is this URL's host a loopback address?////// A loopback authority is only meaningful on the machine that recorded it, so@@ -1959,6 +2117,190 @@ mod tests {assert_eq!(attempts, vec![0, 1, 0]);}+ /// A mock hook server that accepts exactly one bearer and answers `401` to+ /// everything else. Models a rotated credential: the token frozen into the+ /// spool is no longer valid, the drain's live one is.+ async fn serve_token_gated_hook(+ accepted_token: &'static str,+ req_count: std::sync::Arc<std::sync::atomic::AtomicUsize>,+ ) -> String {+ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ tokio::spawn(async move {+ while let Ok((mut s, _)) = listener.accept().await {+ let rc = req_count.clone();+ tokio::spawn(async move {+ let mut buf = vec![0_u8; 65536];+ let n = s.read(&mut buf).await.unwrap_or(0);+ rc.fetch_add(1, std::sync::atomic::Ordering::SeqCst);+ let req = String::from_utf8_lossy(&buf[..n]);+ let authorized = req.lines().any(|l| {+ l.trim().eq_ignore_ascii_case(&format!(+ "authorization: Bearer {accepted_token}"+ ))+ });+ let is_batch = req+ .lines()+ .next()+ .is_some_and(|l| l.contains("/hook/batch"));+ let (status, body) = if !authorized {+ ("401 Unauthorized", "{\"error\":\"bad token\"}".to_string())+ } else if is_batch {+ let payload = req.split("\r\n\r\n").nth(1).unwrap_or("");+ let accepted = serde_json::from_str::<serde_json::Value>(payload)+ .ok()+ .and_then(|v| v.as_array().map(Vec::len))+ .unwrap_or(0);+ ("200 OK", format!("{{\"accepted\":{accepted}}}"))+ } else {+ ("202 Accepted", "queued".to_string())+ };+ let resp = format!(+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len()+ );+ let _ = s.write_all(resp.as_bytes()).await;+ });+ }+ });+ addr.to_string()+ }++ /// #542: the spool freezes its credential at capture time, so rotating the+ /// token strands every already-spooled entry on the old one. The drain is+ /// handed the current token by the hook that spawned it, and retries a+ /// rejected entry with it.+ #[tokio::test]+ async fn drain_recovers_an_entry_stranded_by_a_rotated_token() {+ let tmp = tempfile::tempdir().unwrap();+ let spool = spool_dir(tmp.path());+ let count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));+ let addr = serve_token_gated_hook("rotated-new", count.clone()).await;++ let entry = entry_for(+ format!("http://{addr}/hook?event=e0"),+ "{}".into(),+ Some("stale-old"),+ false,+ );+ enqueue(&spool, &entry).unwrap();++ let r = drain_with_live_token(+ &spool,+ tmp.path(),+ Duration::from_secs(5),+ Duration::from_millis(500),+ Some("rotated-new"),+ )+ .await;++ assert_eq!(r.sent, 1, "the stranded entry is recovered");+ assert_eq!(list_entries(&spool).unwrap().0.len(), 0, "and removed");+ }++ /// The control for the test above, and the behaviour before this change:+ /// with no live token the drain has nothing to retry with, so the entry+ /// stays queued and is charged one attempt. Pinned so the recovery test+ /// cannot pass merely because the server was lenient.+ #[tokio::test]+ async fn drain_without_a_live_token_leaves_a_rejected_entry_queued() {+ let tmp = tempfile::tempdir().unwrap();+ let spool = spool_dir(tmp.path());+ let count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));+ let addr = serve_token_gated_hook("rotated-new", count.clone()).await;++ let entry = entry_for(+ format!("http://{addr}/hook?event=e0"),+ "{}".into(),+ Some("stale-old"),+ false,+ );+ enqueue(&spool, &entry).unwrap();++ let r = drain_with_live_token(+ &spool,+ tmp.path(),+ Duration::from_secs(5),+ Duration::from_millis(500),+ None,+ )+ .await;++ assert_eq!(r.sent, 0, "nothing to retry with");+ assert_eq!(list_entries(&spool).unwrap().0.len(), 1, "entry survives");+ }++ /// An identical token is not worth a second request, and a non-static+ /// entry is not this mechanism's business: an OIDC bearer is already+ /// resolved and refreshed once per pass, so a 401 there is a real+ /// rejection rather than a stale freeze.+ #[test]+ fn static_retry_token_only_fires_when_it_can_change_the_outcome() {+ let static_entry = entry_for("http://x/hook".into(), "{}".into(), Some("old"), false);+ let oidc_entry = entry_for("http://x/hook".into(), "{}".into(), None, true);+ let anon_entry = entry_for("http://x/hook".into(), "{}".into(), None, false);++ assert_eq!(+ static_retry_token(&static_entry, Some("old"), Some("new")),+ Some("new"),+ "a rotated token is worth one retry"+ );+ assert_eq!(+ static_retry_token(&static_entry, Some("old"), Some("old")),+ None,+ "an identical token would only repeat the failure"+ );+ assert_eq!(+ static_retry_token(&static_entry, Some("old"), None),+ None,+ "no live token, nothing to retry with"+ );+ assert_eq!(+ static_retry_token(&oidc_entry, Some("resolved"), Some("new")),+ None,+ "OIDC is already re-resolved per pass; a 401 there is genuine"+ );+ assert_eq!(+ static_retry_token(&anon_entry, None, Some("new")),+ None,+ "an anonymous entry was never authenticated"+ );+ }++ /// A `401` must be distinguishable from any other refusal, or the drain+ /// cannot tell "this credential is stale" from "this event is bad".+ #[tokio::test]+ async fn a_401_is_reported_as_unauthorized_not_failed() {+ let count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));+ let addr = serve_token_gated_hook("right", count.clone()).await;+ let client = build_client();++ assert_eq!(+ crate::commands::hook_capture::post_hook(+ &client,+ &format!("http://{addr}/hook?event=e"),+ "{}",+ Some("wrong"),+ Duration::from_millis(500),+ )+ .await,+ crate::commands::hook_capture::PostOutcome::Unauthorized+ );+ assert_eq!(+ crate::commands::hook_capture::post_hook(+ &client,+ &format!("http://{addr}/hook?event=e"),+ "{}",+ Some("right"),+ Duration::from_millis(500),+ )+ .await,+ crate::commands::hook_capture::PostOutcome::Delivered,+ "the gate itself works, so the 401 above is about the token"+ );+ }+/// A mock hook server: answers `200 {"accepted":N}` to `POST /hook/batch`/// (N = array length in the body) and `202 queued` to a per-event/// `POST /hook`. Counts every request so a test can assert batching. Readsdiff --git a/docs/users.md b/docs/users.mdindex 23031855..00174709 100644--- a/docs/users.md+++ b/docs/users.md@@ -430,6 +430,25 @@ XGqsBp...<43-chars>...zRm0VtRotation implicitly revives an expired token — you can recover anoffboarded user without first running `revive`.+#### What rotation does to already-spooled hook events++A lifecycle hook that cannot reach the server writes the event to the local+spool **together with the bearer it was holding at the time**. Rotating a token+therefore strands every spooled event captured under the old one: each is+rejected with `401` on the next drain.++Since #542 the drain recovers them. Re-run `install-hooks --apply` with the new+token, and the next drain retries any `401`ed entry with the current bearer —+the hook that spawns the drain hands it down in the child's environment. The+retry is bounded: it only runs for an entry the server has already rejected,+only for a static bearer (an OIDC token is re-resolved and refreshed every pass+anyway), and only when the current token actually differs from the one that+just failed.++Until you re-run `install-hooks`, the hooks still hold the old token and there+is nothing newer to retry with — those events stay queued and age out on the+normal retry budget rather than blocking the rest of the spool (#493).+## Backward compatibilityIf you're upgrading from a pre-v0.8 ai-memory: