Skip to content

Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors

Rust

fix(spool): recover hook events stranded by a rotated auth token (#542) (#551)

Fabio Akita52279607def1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 953423fb..92f3eb36 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -153,6 +153,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
three destructive commands in one place instead of documenting the limits of
one and overstating the other two.
+- The hook-spool drain can now recover events stranded by a rotated auth token
+ (#542). A spooled envelope freezes the bearer it was captured with, so
+ rotating the token left every already-spooled entry authenticating with the
+ old one and failing `401` forever; OIDC already re-resolved per pass, static
+ tokens did not.
+
+ The drain now retries a rejected entry once with the token it was handed by
+ the hook that spawned it, which is current by construction. Bounded three
+ ways: only after the server has already answered `401`, only for a `Static`
+ entry, and only when the live token actually differs from the one that just
+ failed — so a healthy drain never pays for it and an identical credential is
+ not re-sent.
+
+ The token reaches the drain in the child process's **environment**, never on
+ its command line: an argument would publish the credential to the process
+ table for every local user for as long as the drain runs. A test pins that it
+ stays out of argv. `401` also became a distinct `Unauthorized` outcome —
+ previously indistinguishable from any other refusal, so the drain could not
+ tell a stale credential from a bad event.
+
+ No new credential is stored at rest: the drain is handed a token that already
+ exists rather than reading one from a file ai-memory would have to write and
+ protect.
+
### Docs
- `docs/windows.md` gains **Scenario E**, a persistent-server story for native
diff --git a/crates/ai-memory-cli/src/commands/hook.rs b/crates/ai-memory-cli/src/commands/hook.rs
index 0e61e7ff..e9473d18 100644
--- a/crates/ai-memory-cli/src/commands/hook.rs
+++ b/crates/ai-memory-cli/src/commands/hook.rs
@@ -124,8 +124,8 @@ fn should_incremental_drain(event: &str, spool_len: usize, threshold: usize) ->
event == "post-tool-use" && spool_len >= threshold
}
-fn spawn_background_drainer(data_dir: &Path) -> std::io::Result<()> {
- hook_drain_process::spawn(data_dir)
+fn spawn_background_drainer(data_dir: &Path, live_token: Option<&str>) -> std::io::Result<()> {
+ hook_drain_process::spawn(data_dir, live_token)
}
fn should_spawn_background_drainer(event: &str) -> bool {
@@ -299,9 +299,10 @@ fn cwd_query_suffix(
fn after_background_drain_event_enqueue(
data_dir: &Path,
- spawn: impl FnOnce(&Path) -> std::io::Result<()>,
+ live_token: Option<&str>,
+ spawn: impl FnOnce(&Path, Option<&str>) -> std::io::Result<()>,
) -> std::io::Result<()> {
- spawn(data_dir)
+ spawn(data_dir, live_token)
}
/// Hidden drain-only fast path. Reads no stdin and writes no stdout.
@@ -477,7 +478,7 @@ async fn run_with_payload<W, S>(
) -> anyhow::Result<()>
where
W: std::io::Write,
- S: FnOnce(&Path) -> std::io::Result<()>,
+ S: FnOnce(&Path, Option<&str>) -> std::io::Result<()>,
{
let agent_kind = AgentKind::from_wire(&args.agent);
let hook_event = HookEvent::parse(&args.event);
@@ -794,7 +795,15 @@ where
// but `stop` and `pre-compact` also trigger the helper so delivery does not
// rely on the single hook most likely to be cancelled during agent shutdown.
if should_spawn_background_drainer(&args.event)
- && let Err(err) = after_background_drain_event_enqueue(&dd, spawn_background_drainer)
+ && let Err(err) = after_background_drain_event_enqueue(
+ &dd,
+ // The hook's own token: current by definition, since the agent
+ // config it came from was rendered by the last `install-hooks`.
+ // The drain uses it only to retry an entry the server has already
+ // rejected with 401 (#542).
+ args.auth_token.as_deref(),
+ spawn_background_drainer,
+ )
{
eprintln!(
"ai-memory hook warning: failed to start background spool drainer; event remains queued: {err}"
@@ -1191,7 +1200,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1210,7 +1219,7 @@ mod tests {
antigravity_hook_args("pre-tool-use", "http://127.0.0.1:1"),
"not-json".into(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1338,7 +1347,7 @@ mod tests {
devin_hook_args("session-start"),
payload.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1378,7 +1387,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1430,7 +1439,7 @@ mod tests {
devin_hook_args("post-tool-use"),
payload.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1468,7 +1477,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1484,7 +1493,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1627,7 +1636,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1640,7 +1649,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1677,7 +1686,7 @@ mod tests {
args,
r#"{"session_id":"s","cwd":"/tmp"}"#.into(),
&mut stdout,
- |path| {
+ |path, _token| {
assert_eq!(path, data_dir.as_path());
assert_eq!(hook_spool::spool_len(&spool), 1, "spawn runs after enqueue");
called.set(called.get() + 1);
@@ -1720,7 +1729,7 @@ mod tests {
args,
r#"{"session_id":"s","cwd":"/tmp"}"#.into(),
&mut stdout,
- |path| {
+ |path, _token| {
assert_eq!(path, data_dir.as_path());
assert_eq!(hook_spool::spool_len(&spool), 1, "spawn runs after enqueue");
called.set(called.get() + 1);
@@ -1757,9 +1766,13 @@ mod tests {
capture_mode: None,
};
- run_with_payload(Some(data_dir), args, "{}".into(), &mut stdout, |_path| {
- Err(std::io::Error::other("spawn failed"))
- })
+ run_with_payload(
+ Some(data_dir),
+ args,
+ "{}".into(),
+ &mut stdout,
+ |_path, _token| Err(std::io::Error::other("spawn failed")),
+ )
.await
.unwrap();
@@ -1790,7 +1803,7 @@ mod tests {
args,
r#"{"hook_event_name":"SessionEnd"}"#.into(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -1814,7 +1827,7 @@ mod tests {
#[test]
fn session_end_spawn_failure_is_returned_for_warning_only() {
let tmp = tempfile::tempdir().unwrap();
- let err = after_background_drain_event_enqueue(tmp.path(), |_path| {
+ let err = after_background_drain_event_enqueue(tmp.path(), None, |_path, _token| {
Err(std::io::Error::other("spawn failed"))
})
.unwrap_err();
@@ -1827,7 +1840,7 @@ mod tests {
let tmp = tempfile::tempdir().unwrap();
let called = std::cell::Cell::new(false);
- after_background_drain_event_enqueue(tmp.path(), |path| {
+ after_background_drain_event_enqueue(tmp.path(), None, |path, _token| {
assert_eq!(path, tmp.path());
called.set(true);
Ok(())
@@ -1877,7 +1890,7 @@ mod tests {
let called = std::cell::Cell::new(false);
let mut args = devin_hook_args("post-tool-use");
args.server_url = "http://127.0.0.1:1".into();
- run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"secret/SENTINEL"}}).to_string(), &mut stdout, |_| { called.set(true); Ok(()) }).await.unwrap();
+ run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"secret/SENTINEL"}}).to_string(), &mut stdout, |_, _| { called.set(true); Ok(()) }).await.unwrap();
assert_eq!(stdout, b"{}\n");
assert!(!called.get());
assert_eq!(hook_spool::spool_len(&hook_spool::spool_dir(&data_dir)), 0);
@@ -1912,7 +1925,7 @@ mod tests {
args,
raw.to_string(),
&mut stdout,
- |_| {
+ |_, _| {
called.set(true);
Ok(())
},
@@ -1953,7 +1966,7 @@ mod tests {
args,
raw.to_string(),
&mut stdout,
- |_| {
+ |_, _| {
called.set(true);
Ok(())
},
@@ -1993,7 +2006,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| {
+ |_, _| {
called.set(true);
Ok(())
},
@@ -2016,7 +2029,7 @@ mod tests {
.unwrap();
let data_dir = tmp.path().join("data");
let mut stdout = Vec::new();
- run_with_payload(Some(data_dir.clone()), devin_hook_args("post-tool-use"), serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL_PATH","args":"SENTINEL_ARGS"},"output":"SENTINEL_OUTPUT","error":"SENTINEL_ERROR","nested":{"raw":"SENTINEL_NESTED"}}).to_string(), &mut stdout, |_| Ok(())).await.unwrap();
+ run_with_payload(Some(data_dir.clone()), devin_hook_args("post-tool-use"), serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL_PATH","args":"SENTINEL_ARGS"},"output":"SENTINEL_OUTPUT","error":"SENTINEL_ERROR","nested":{"raw":"SENTINEL_NESTED"}}).to_string(), &mut stdout, |_, _| Ok(())).await.unwrap();
let entry = read_spooled_entries(&hook_spool::spool_dir(&data_dir))
.pop()
.unwrap();
@@ -2039,7 +2052,7 @@ mod tests {
let mut args = devin_hook_args("post-tool-use");
args.check_capture = true;
let mut stdout = Vec::new();
- run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL"}}).to_string(), &mut stdout, |_| Err(std::io::Error::other("must not spawn"))).await.unwrap();
+ run_with_payload(Some(data_dir.clone()), args, serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"SENTINEL"}}).to_string(), &mut stdout, |_, _| Err(std::io::Error::other("must not spawn"))).await.unwrap();
let output: serde_json::Value = serde_json::from_slice(&stdout).unwrap();
// Pin the exact key set, not just its size: `--check-capture` is how an
// operator verifies an opt-out, so a field silently appearing or
@@ -2082,7 +2095,7 @@ mod tests {
devin_hook_args("post-tool-use"),
inactive_payload.clone(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2103,7 +2116,7 @@ mod tests {
serde_json::json!({"cwd":tmp.path(),"tool_name":"Edit","tool_input":{"path":"public"}})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2134,7 +2147,7 @@ mod tests {
args,
raw.to_string(),
&mut stdout,
- |_| Err(std::io::Error::other("must not spawn")),
+ |_, _| Err(std::io::Error::other("must not spawn")),
)
.await
.unwrap();
@@ -2167,7 +2180,7 @@ mod tests {
devin_hook_args("post-tool-use"),
raw.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2274,7 +2287,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2322,7 +2335,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Err(std::io::Error::other("must not spawn")),
+ |_, _| Err(std::io::Error::other("must not spawn")),
)
.await
.unwrap();
@@ -2346,7 +2359,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2381,7 +2394,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2412,7 +2425,7 @@ mod tests {
serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path(), "prompt": "hi"})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2429,7 +2442,7 @@ mod tests {
serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path(), "prompt": "hi"})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2449,7 +2462,7 @@ mod tests {
serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path(), "prompt": "hi"})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2475,7 +2488,7 @@ mod tests {
kimi_hook_args("session-start", &base),
serde_json::json!({"sessionId": "session_abc", "cwd": tmp.path()}).to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2500,7 +2513,7 @@ mod tests {
args,
serde_json::json!({"session_id": "claude-session", "cwd": tmp.path()}).to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2545,7 +2558,7 @@ mod tests {
kimi_hook_args("user-prompt-submit", &base),
payload.clone(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2566,7 +2579,7 @@ mod tests {
kimi_hook_args("user-prompt-submit", &base),
payload,
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2598,7 +2611,7 @@ mod tests {
kimi_hook_args("user-prompt", &base),
payload.clone(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2617,7 +2630,7 @@ mod tests {
kimi_hook_args("user-prompt", &base),
payload,
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
@@ -2646,7 +2659,7 @@ mod tests {
})
.to_string(),
&mut stdout,
- |_| Ok(()),
+ |_, _| Ok(()),
)
.await
.unwrap();
diff --git a/crates/ai-memory-cli/src/commands/hook_capture.rs b/crates/ai-memory-cli/src/commands/hook_capture.rs
index 04eca713..1b84908e 100644
--- a/crates/ai-memory-cli/src/commands/hook_capture.rs
+++ b/crates/ai-memory-cli/src/commands/hook_capture.rs
@@ -344,6 +344,13 @@ pub enum PostOutcome {
/// so the drain can skip past them instead of stopping at the first one
/// (#493).
Unreachable,
+ /// `401` — the server rejected the bearer. Distinguished from
+ /// [`Self::Failed`] because it says something about the *credential*
+ /// rather than the entry: a spooled event carries the token frozen at
+ /// capture time, so after a rotation every entry captured under the old
+ /// one fails identically and is recoverable by retrying with the current
+ /// token (#542).
+ Unauthorized,
}
/// POST the payload as JSON, best-effort. `timeout` is caller-chosen: the
@@ -371,6 +378,7 @@ pub async fn post_hook(
Ok(resp) if resp.status() == reqwest::StatusCode::TOO_MANY_REQUESTS => {
PostOutcome::Saturated
}
+ Ok(resp) if resp.status() == reqwest::StatusCode::UNAUTHORIZED => PostOutcome::Unauthorized,
Ok(_) => PostOutcome::Failed,
Err(_) => PostOutcome::Unreachable,
}
@@ -409,6 +417,10 @@ pub enum BatchOutcome {
/// `404`/`405` — the server has no `/hook/batch` (a pre-upgrade build). The
/// caller falls back to per-event `POST /hook` for the rest of the drain.
Unsupported,
+ /// `401` — the server rejected the batch's bearer. See
+ /// [`PostOutcome::Unauthorized`]; the whole batch shares one identity, so
+ /// a rejected credential fails all of it and none of it was processed.
+ Unauthorized,
/// Transport error or any other non-2xx: the batch outcome is unknown. The
/// drain charges conservatively so trailing events that may never have been
/// attempted do not burn retry budget.
@@ -474,6 +486,8 @@ pub async fn post_batch(
|| status == reqwest::StatusCode::METHOD_NOT_ALLOWED
{
BatchOutcome::Unsupported
+ } else if status == reqwest::StatusCode::UNAUTHORIZED {
+ BatchOutcome::Unauthorized
} else {
BatchOutcome::Failed
}
diff --git a/crates/ai-memory-cli/src/commands/hook_drain_process.rs b/crates/ai-memory-cli/src/commands/hook_drain_process.rs
index 9607c9ba..7f38cc84 100644
--- a/crates/ai-memory-cli/src/commands/hook_drain_process.rs
+++ b/crates/ai-memory-cli/src/commands/hook_drain_process.rs
@@ -36,6 +36,13 @@ pub struct DrainCommandSpec {
pub args: Vec<OsString>,
/// Stderr log file under `<data_dir>/logs`.
pub stderr_log: PathBuf,
+ /// Current static bearer to hand the child in its environment, when the
+ /// spawning hook has one.
+ ///
+ /// The environment rather than an argument: a flag would put the
+ /// credential in the process table for the lifetime of the drain, readable
+ /// by anyone who can run `ps`.
+ pub live_token: Option<OsString>,
}
/// Unit-testable stdio/detach configuration for a drainer process.
@@ -67,7 +74,12 @@ pub enum DetachConfig {
}
/// Build `ai-memory --data-dir <dir> hook-drain`.
-pub fn command_spec(data_dir: &Path) -> io::Result<DrainCommandSpec> {
+///
+/// `live_token` is the bearer the spawning hook is currently configured with.
+/// A spooled envelope froze its own credential at capture time, so handing the
+/// drain a current one is what lets it recover entries stranded by a rotation
+/// (#542).
+pub fn command_spec(data_dir: &Path, live_token: Option<&str>) -> io::Result<DrainCommandSpec> {
Ok(DrainCommandSpec {
exe: std::env::current_exe()?,
args: vec![
@@ -76,6 +88,7 @@ pub fn command_spec(data_dir: &Path) -> io::Result<DrainCommandSpec> {
OsString::from("hook-drain"),
],
stderr_log: data_dir.join("logs").join("hook-drain.log"),
+ live_token: live_token.filter(|t| !t.is_empty()).map(OsString::from),
})
}
@@ -91,11 +104,19 @@ pub fn spawn_config(spec: &DrainCommandSpec, try_breakaway: bool) -> SpawnConfig
}
/// Spawn the hidden drainer without inheriting hook stdio.
-pub fn spawn(data_dir: &Path) -> io::Result<()> {
- let spec = command_spec(data_dir)?;
+pub fn spawn(data_dir: &Path, live_token: Option<&str>) -> io::Result<()> {
+ let spec = command_spec(data_dir, live_token)?;
spawn_spec(&spec)
}
+/// Hand the child the current bearer in its environment, when there is one.
+/// Applied to every spawn path so the breakaway-retry cannot silently drop it.
+fn apply_live_token(command: &mut Command, spec: &DrainCommandSpec) {
+ if let Some(token) = &spec.live_token {
+ command.env(crate::commands::hook_spool::LIVE_TOKEN_ENV, token);
+ }
+}
+
fn spawn_spec(spec: &DrainCommandSpec) -> io::Result<()> {
if let Some(parent) = spec.stderr_log.parent() {
std::fs::create_dir_all(parent)?;
@@ -108,6 +129,7 @@ fn spawn_spec(spec: &DrainCommandSpec) -> io::Result<()> {
.open(&config.stderr_file)?;
let mut command = command_for_spec(spec, &config.detach);
+ apply_live_token(&mut command, spec);
command
.stdin(Stdio::null())
.stdout(Stdio::null())
@@ -130,6 +152,7 @@ fn spawn_spec(spec: &DrainCommandSpec) -> io::Result<()> {
.append(true)
.open(&config.stderr_file)?;
let mut retry = command_for_spec(spec, &config.detach);
+ apply_live_token(&mut retry, spec);
retry
.stdin(Stdio::null())
.stdout(Stdio::null())
@@ -234,7 +257,7 @@ mod tests {
#[test]
fn command_spec_uses_data_dir_then_hidden_subcommand() {
let tmp = tempfile::tempdir().unwrap();
- let spec = command_spec(tmp.path()).unwrap();
+ let spec = command_spec(tmp.path(), None).unwrap();
assert_eq!(
spec.args,
@@ -277,10 +300,52 @@ mod tests {
rotate_oversized_log(&log, 16);
}
+ /// The credential must reach the drain in its environment and nowhere
+ /// near its argv. A command line is world-readable through the process
+ /// table for as long as the process lives, so passing the token as a flag
+ /// would publish it to every local user each time a session ends.
+ #[test]
+ fn the_live_token_travels_in_the_environment_not_the_command_line() {
+ let tmp = tempfile::tempdir().unwrap();
+ let spec = command_spec(tmp.path(), Some("SECRET-BEARER")).unwrap();
+
+ assert_eq!(
+ spec.live_token.as_deref(),
+ Some(std::ffi::OsStr::new("SECRET-BEARER"))
+ );
+ let rendered: Vec<String> = spec
+ .args
+ .iter()
+ .map(|a| a.to_string_lossy().into_owned())
+ .collect();
+ assert!(
+ !rendered.iter().any(|a| a.contains("SECRET-BEARER")),
+ "token leaked into argv: {rendered:?}"
+ );
+ assert!(
+ rendered.iter().any(|a| a == "hook-drain"),
+ "still the drain command: {rendered:?}"
+ );
+ }
+
+ /// An absent or empty token carries nothing, so a no-auth deployment does
+ /// not export an empty bearer into the child.
+ #[test]
+ fn an_absent_or_empty_token_is_not_carried() {
+ let tmp = tempfile::tempdir().unwrap();
+ assert!(command_spec(tmp.path(), None).unwrap().live_token.is_none());
+ assert!(
+ command_spec(tmp.path(), Some(""))
+ .unwrap()
+ .live_token
+ .is_none()
+ );
+ }
+
#[test]
fn spawn_config_redirects_stdio_and_logs_under_data_dir_logs() {
let tmp = tempfile::tempdir().unwrap();
- let spec = command_spec(tmp.path()).unwrap();
+ let spec = command_spec(tmp.path(), None).unwrap();
let config = spawn_config(&spec, true);
assert!(config.stdin_null);
@@ -295,7 +360,7 @@ mod tests {
#[test]
fn unix_spawn_config_prefers_true_session_detach_when_available() {
let tmp = tempfile::tempdir().unwrap();
- let spec = command_spec(tmp.path()).unwrap();
+ let spec = command_spec(tmp.path(), None).unwrap();
let DetachConfig::UnixNewSession { setsid } = spawn_config(&spec, true).detach;
if let Some(path) = setsid {
assert!(
@@ -309,7 +374,7 @@ mod tests {
#[test]
fn windows_spawn_config_uses_expected_flags_and_breakaway_toggle() {
let tmp = tempfile::tempdir().unwrap();
- let spec = command_spec(tmp.path()).unwrap();
+ let spec = command_spec(tmp.path(), None).unwrap();
let DetachConfig::WindowsCreationFlags(with_breakaway) = spawn_config(&spec, true).detach;
let DetachConfig::WindowsCreationFlags(without_breakaway) =
spawn_config(&spec, false).detach;
diff --git a/crates/ai-memory-cli/src/commands/hook_spool.rs b/crates/ai-memory-cli/src/commands/hook_spool.rs
index 0f91d919..c792f4bc 100644
--- a/crates/ai-memory-cli/src/commands/hook_spool.rs
+++ b/crates/ai-memory-cli/src/commands/hook_spool.rs
@@ -508,6 +508,29 @@ pub async fn drain(
data_dir: &Path,
total_budget: Duration,
per_event_timeout: Duration,
+) -> DrainResult {
+ drain_with_live_token(
+ spool,
+ data_dir,
+ total_budget,
+ per_event_timeout,
+ live_static_token().as_deref(),
+ )
+ .await
+}
+
+/// [`drain`] with the current static bearer supplied explicitly.
+///
+/// The environment is read once, at the boundary in [`drain`], and passed down
+/// as data. Tests drive this directly: mutating process-wide environment from a
+/// test would race every other test in the binary, and `set_var` is `unsafe` in
+/// this edition — which this workspace forbids outright.
+pub async fn drain_with_live_token(
+ spool: &Path,
+ data_dir: &Path,
+ total_budget: Duration,
+ per_event_timeout: Duration,
+ live_token: Option<&str>,
) -> DrainResult {
let (mut files, unreadable) = match list_entries(spool) {
Ok(listed) => listed,
@@ -722,6 +745,32 @@ pub async fn drain(
bump_or_drop(path, entry, &mut result);
unreachable_endpoints.insert(batch_endpoint(&entry.url));
}
+ PostOutcome::Unauthorized => {
+ // Credential rejected, not the entry. Retry once
+ // with this drain's live token (#542).
+ let retry =
+ static_retry_token(entry, item_bearer.as_deref(), live_token);
+ let recovered = match retry {
+ Some(token) => matches!(
+ post_hook(
+ &client,
+ &entry.url,
+ &entry.body,
+ Some(token),
+ per_event_timeout,
+ )
+ .await,
+ PostOutcome::Delivered
+ ),
+ None => false,
+ };
+ if recovered {
+ let _ = std::fs::remove_file(path);
+ result.sent += 1;
+ } else {
+ bump_or_drop(path, entry, &mut result);
+ }
+ }
}
}
}
@@ -791,6 +840,49 @@ pub async fn drain(
unreachable_endpoints.insert(base.clone());
continue;
}
+ BatchOutcome::Unauthorized => {
+ // The envelope froze its credential at capture time, so a
+ // rotated token leaves every entry captured under the old
+ // one failing identically — the credential half of the
+ // same problem #493 fixed for the address (#542).
+ //
+ // Retry once with the token this drain was handed, which is
+ // current by construction. Runs only for a batch that has
+ // already been rejected, so a healthy drain never pays for
+ // it.
+ let retry = static_retry_token(&chunk[0].1, bearer.as_deref(), live_token);
+ if let Some(token) = retry {
+ match post_batch(&client, &base, &payload, Some(token), batch_timeout).await
+ {
+ BatchOutcome::Accepted(k) => {
+ let k = k.min(chunk.len());
+ for (path, _) in &chunk[..k] {
+ let _ = std::fs::remove_file(path);
+ }
+ result.sent += k;
+ result.remaining += chunk.len().saturating_sub(k);
+ continue;
+ }
+ BatchOutcome::AcceptedIndices { indices, .. } => {
+ let sent = delete_accepted_indices(&chunk, &indices);
+ result.sent += sent;
+ result.remaining += chunk.len().saturating_sub(sent);
+ continue;
+ }
+ // The current token is no better — the server is
+ // rejecting more than a stale credential. Fall
+ // through and charge conservatively.
+ _ => {}
+ }
+ }
+
+ // Same conservative accounting as `Failed`: the server
+ // refused the whole batch, so nothing in it was processed.
+ bump_or_drop(&chunk[0].0, &chunk[0].1, &mut result);
+ result.remaining +=
+ chunk.len().saturating_sub(1) + files.len().saturating_sub(idx);
+ break;
+ }
BatchOutcome::Failed => {
// The batch didn't land (server answered with an unexpected
// status, or the body did not parse).
@@ -831,6 +923,31 @@ pub async fn drain(
bump_or_drop(&path, &entry, &mut result);
unreachable_endpoints.insert(batch_endpoint(&entry.url));
}
+ PostOutcome::Unauthorized => {
+ // Credential rejected, not the entry. Retry once with this
+ // drain's live token (#542).
+ let retry = static_retry_token(&entry, bearer.as_deref(), live_token);
+ let recovered = match retry {
+ Some(token) => matches!(
+ post_hook(
+ &client,
+ &entry.url,
+ &entry.body,
+ Some(token),
+ per_event_timeout,
+ )
+ .await,
+ PostOutcome::Delivered
+ ),
+ None => false,
+ };
+ if recovered {
+ let _ = std::fs::remove_file(&path);
+ result.sent += 1;
+ } else {
+ bump_or_drop(&path, &entry, &mut result);
+ }
+ }
}
}
}
@@ -926,6 +1043,47 @@ async fn entry_bearer(
}
}
+/// Environment variable carrying the *current* static bearer into a spawned
+/// `hook-drain`.
+///
+/// Deliberately the environment and not a command-line flag: the drain is a
+/// separate process, and an argument would put the credential in the process
+/// table for anyone running `ps` while it runs.
+pub(crate) const LIVE_TOKEN_ENV: &str = "AI_MEMORY_AUTH_TOKEN";
+
+/// The static bearer this drain process was handed, if any.
+///
+/// `hook-drain` is spawned by a lifecycle hook that already holds the current
+/// token, and passes it down in the child's environment. A drain started by
+/// hand, or by a hook running against a server that needs no auth, simply has
+/// none — and the re-resolve below is skipped rather than guessed at.
+fn live_static_token() -> Option<String> {
+ std::env::var(LIVE_TOKEN_ENV).ok().filter(|t| !t.is_empty())
+}
+
+/// The token to retry a `401` with, or `None` when retrying is pointless.
+///
+/// A spooled envelope freezes its credential at capture time, so rotating the
+/// token leaves every already-spooled entry authenticating with the old one
+/// (#542). The drain process, by contrast, was handed the current token by the
+/// hook that spawned it moments ago.
+///
+/// Two guards keep this from turning into a blind second attempt: only
+/// `Static` entries qualify — an `Oidc` bearer is already resolved and
+/// refreshed once per pass, so a 401 there is a real rejection — and the live
+/// token must actually differ from the one that just failed. Retrying an
+/// identical credential would only repeat the failure at double the cost.
+fn static_retry_token<'a>(
+ entry: &SpoolEntry,
+ rejected: Option<&str>,
+ live: Option<&'a str>,
+) -> Option<&'a str> {
+ if entry.auth_mode != AuthMode::Static {
+ return None;
+ }
+ live.filter(|t| Some(*t) != rejected)
+}
+
/// Is this URL's host a loopback address?
///
/// A loopback authority is only meaningful on the machine that recorded it, so
@@ -1959,6 +2117,190 @@ mod tests {
assert_eq!(attempts, vec![0, 1, 0]);
}
+ /// A mock hook server that accepts exactly one bearer and answers `401` to
+ /// everything else. Models a rotated credential: the token frozen into the
+ /// spool is no longer valid, the drain's live one is.
+ async fn serve_token_gated_hook(
+ accepted_token: &'static str,
+ req_count: std::sync::Arc<std::sync::atomic::AtomicUsize>,
+ ) -> String {
+ use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _};
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ tokio::spawn(async move {
+ while let Ok((mut s, _)) = listener.accept().await {
+ let rc = req_count.clone();
+ tokio::spawn(async move {
+ let mut buf = vec![0_u8; 65536];
+ let n = s.read(&mut buf).await.unwrap_or(0);
+ rc.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
+ let req = String::from_utf8_lossy(&buf[..n]);
+ let authorized = req.lines().any(|l| {
+ l.trim().eq_ignore_ascii_case(&format!(
+ "authorization: Bearer {accepted_token}"
+ ))
+ });
+ let is_batch = req
+ .lines()
+ .next()
+ .is_some_and(|l| l.contains("/hook/batch"));
+ let (status, body) = if !authorized {
+ ("401 Unauthorized", "{\"error\":\"bad token\"}".to_string())
+ } else if is_batch {
+ let payload = req.split("\r\n\r\n").nth(1).unwrap_or("");
+ let accepted = serde_json::from_str::<serde_json::Value>(payload)
+ .ok()
+ .and_then(|v| v.as_array().map(Vec::len))
+ .unwrap_or(0);
+ ("200 OK", format!("{{\"accepted\":{accepted}}}"))
+ } else {
+ ("202 Accepted", "queued".to_string())
+ };
+ let resp = format!(
+ "HTTP/1.1 {status}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len()
+ );
+ let _ = s.write_all(resp.as_bytes()).await;
+ });
+ }
+ });
+ addr.to_string()
+ }
+
+ /// #542: the spool freezes its credential at capture time, so rotating the
+ /// token strands every already-spooled entry on the old one. The drain is
+ /// handed the current token by the hook that spawned it, and retries a
+ /// rejected entry with it.
+ #[tokio::test]
+ async fn drain_recovers_an_entry_stranded_by_a_rotated_token() {
+ let tmp = tempfile::tempdir().unwrap();
+ let spool = spool_dir(tmp.path());
+ let count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let addr = serve_token_gated_hook("rotated-new", count.clone()).await;
+
+ let entry = entry_for(
+ format!("http://{addr}/hook?event=e0"),
+ "{}".into(),
+ Some("stale-old"),
+ false,
+ );
+ enqueue(&spool, &entry).unwrap();
+
+ let r = drain_with_live_token(
+ &spool,
+ tmp.path(),
+ Duration::from_secs(5),
+ Duration::from_millis(500),
+ Some("rotated-new"),
+ )
+ .await;
+
+ assert_eq!(r.sent, 1, "the stranded entry is recovered");
+ assert_eq!(list_entries(&spool).unwrap().0.len(), 0, "and removed");
+ }
+
+ /// The control for the test above, and the behaviour before this change:
+ /// with no live token the drain has nothing to retry with, so the entry
+ /// stays queued and is charged one attempt. Pinned so the recovery test
+ /// cannot pass merely because the server was lenient.
+ #[tokio::test]
+ async fn drain_without_a_live_token_leaves_a_rejected_entry_queued() {
+ let tmp = tempfile::tempdir().unwrap();
+ let spool = spool_dir(tmp.path());
+ let count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let addr = serve_token_gated_hook("rotated-new", count.clone()).await;
+
+ let entry = entry_for(
+ format!("http://{addr}/hook?event=e0"),
+ "{}".into(),
+ Some("stale-old"),
+ false,
+ );
+ enqueue(&spool, &entry).unwrap();
+
+ let r = drain_with_live_token(
+ &spool,
+ tmp.path(),
+ Duration::from_secs(5),
+ Duration::from_millis(500),
+ None,
+ )
+ .await;
+
+ assert_eq!(r.sent, 0, "nothing to retry with");
+ assert_eq!(list_entries(&spool).unwrap().0.len(), 1, "entry survives");
+ }
+
+ /// An identical token is not worth a second request, and a non-static
+ /// entry is not this mechanism's business: an OIDC bearer is already
+ /// resolved and refreshed once per pass, so a 401 there is a real
+ /// rejection rather than a stale freeze.
+ #[test]
+ fn static_retry_token_only_fires_when_it_can_change_the_outcome() {
+ let static_entry = entry_for("http://x/hook".into(), "{}".into(), Some("old"), false);
+ let oidc_entry = entry_for("http://x/hook".into(), "{}".into(), None, true);
+ let anon_entry = entry_for("http://x/hook".into(), "{}".into(), None, false);
+
+ assert_eq!(
+ static_retry_token(&static_entry, Some("old"), Some("new")),
+ Some("new"),
+ "a rotated token is worth one retry"
+ );
+ assert_eq!(
+ static_retry_token(&static_entry, Some("old"), Some("old")),
+ None,
+ "an identical token would only repeat the failure"
+ );
+ assert_eq!(
+ static_retry_token(&static_entry, Some("old"), None),
+ None,
+ "no live token, nothing to retry with"
+ );
+ assert_eq!(
+ static_retry_token(&oidc_entry, Some("resolved"), Some("new")),
+ None,
+ "OIDC is already re-resolved per pass; a 401 there is genuine"
+ );
+ assert_eq!(
+ static_retry_token(&anon_entry, None, Some("new")),
+ None,
+ "an anonymous entry was never authenticated"
+ );
+ }
+
+ /// A `401` must be distinguishable from any other refusal, or the drain
+ /// cannot tell "this credential is stale" from "this event is bad".
+ #[tokio::test]
+ async fn a_401_is_reported_as_unauthorized_not_failed() {
+ let count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
+ let addr = serve_token_gated_hook("right", count.clone()).await;
+ let client = build_client();
+
+ assert_eq!(
+ crate::commands::hook_capture::post_hook(
+ &client,
+ &format!("http://{addr}/hook?event=e"),
+ "{}",
+ Some("wrong"),
+ Duration::from_millis(500),
+ )
+ .await,
+ crate::commands::hook_capture::PostOutcome::Unauthorized
+ );
+ assert_eq!(
+ crate::commands::hook_capture::post_hook(
+ &client,
+ &format!("http://{addr}/hook?event=e"),
+ "{}",
+ Some("right"),
+ Duration::from_millis(500),
+ )
+ .await,
+ crate::commands::hook_capture::PostOutcome::Delivered,
+ "the gate itself works, so the 401 above is about the token"
+ );
+ }
+
/// A mock hook server: answers `200 {"accepted":N}` to `POST /hook/batch`
/// (N = array length in the body) and `202 queued` to a per-event
/// `POST /hook`. Counts every request so a test can assert batching. Reads
diff --git a/docs/users.md b/docs/users.md
index 23031855..00174709 100644
--- a/docs/users.md
+++ b/docs/users.md
@@ -430,6 +430,25 @@ XGqsBp...<43-chars>...zRm0Vt
Rotation implicitly revives an expired token — you can recover an
offboarded user without first running `revive`.
+#### What rotation does to already-spooled hook events
+
+A lifecycle hook that cannot reach the server writes the event to the local
+spool **together with the bearer it was holding at the time**. Rotating a token
+therefore strands every spooled event captured under the old one: each is
+rejected with `401` on the next drain.
+
+Since #542 the drain recovers them. Re-run `install-hooks --apply` with the new
+token, and the next drain retries any `401`ed entry with the current bearer —
+the hook that spawns the drain hands it down in the child's environment. The
+retry is bounded: it only runs for an entry the server has already rejected,
+only for a static bearer (an OIDC token is re-resolved and refreshed every pass
+anyway), and only when the current token actually differs from the one that
+just failed.
+
+Until you re-run `install-hooks`, the hooks still hold the old token and there
+is nothing newer to retry with — those events stay queued and age out on the
+normal retry budget rather than blocking the rest of the spool (#493).
+
## Backward compatibility
If you're upgrading from a pre-v0.8 ai-memory: