Solution for long term memory for agent coding CLIs and to facilitate handoff between different agent vendors
Rust
//! Client-side session token-usage extraction (token-cost visibility).
//!
//! Reads a harness's own transcript file to compute cumulative token totals
//! for one session. Live capture reads bounded forward batches on model/tool boundaries.
//! The legacy full-session fallback runs at `session-end`; it stays off
//! the frequent `stop`/`post-tool-use` hot path. A
//! bounded linear scan (Claude Code) or a bounded tail read (Codex, whose
//! rollout files can reach hundreds of MB for a long session) is cheap
//! enough to run synchronously inline in the native hook before it spools
//! the session-end event. Best-effort throughout: any read/parse failure
//! yields `None`, never a wrong number, and a hook must never fail on its
//! account.
//!
//! Field names and the "sum by dedup'd message.id" / "take the last
//! cumulative record" strategies below were verified against real local
//! transcripts (a Claude Code JSONL and Codex rollout files up to 103 MB),
//! not just documentation.
use std::collections::HashSet;
use std::io::{BufRead as _, Read as _, Seek as _, SeekFrom};
use std::path::{Path, PathBuf};
#[path = "session_usage_context.rs"]
mod response_context;
/// One bounded transcript batch. Publish its cursor only after the lifecycle
/// event has been durably spooled; a crash then causes safe response-id replay.
pub struct LiveCallBatch {
/// Usage metadata to attach to this hook event.
pub calls: Vec<ai_memory_core::TokenCall>,
cursor_path: PathBuf,
next: LiveCursor,
}
impl LiveCallBatch {
/// Persist progress after enqueue. Concurrent writers may rewind progress,
/// but can never skip unspooled records; response ids make replay safe.
pub fn commit(self) {
if let Ok(bytes) = serde_json::to_vec(&self.next) {
if ai_memory_wiki::write_atomic(&self.cursor_path, &bytes).is_err() {
return;
}
// Eviction only causes idempotent replay on the next hook.
if let Some(dir) = self.cursor_path.parent()
&& let Ok(entries) = std::fs::read_dir(dir)
{
let mut cursors = entries
.take(1024)
.filter_map(Result::ok)
.filter(|e| {
e.path().extension().is_some_and(|s| s == "json")
&& e.path() != self.cursor_path
})
.filter_map(|e| Some((e.metadata().ok()?.modified().ok()?, e.path())))
.collect::<Vec<_>>();
cursors.sort_by_key(|(modified, _)| std::cmp::Reverse(*modified));
for (_, stale) in cursors.into_iter().skip(511) {
let _ = std::fs::remove_file(stale);
}
}
}
}
}
#[derive(Default, serde::Serialize, serde::Deserialize)]
struct LiveCursor {
offset: u64,
previous: Option<[u64; 4]>,
model: Option<String>,
prefix: String,
anchor: String,
skipping_long_line: bool,
#[serde(default)]
context: response_context::ScanContext,
}
const LIVE_BATCH_CALLS: usize = 256;
const LIVE_CURSOR_BYTES: u64 = 64 * 1024;
fn fingerprint(file: &mut std::fs::File, start: u64, len: u64) -> std::io::Result<String> {
use sha2::{Digest as _, Sha256};
file.seek(SeekFrom::Start(start))?;
let mut bytes = Vec::new();
file.take(len).read_to_end(&mut bytes)?;
Ok(format!("{:x}", Sha256::digest(bytes)))
}
/// Read forward in bounded batches, including after a capture gap. The routing
/// key includes server, project and native session so progress is never shared
/// between independent destinations. No transcript text is stored in the cursor.
pub fn extract_live_calls(
path: &Path,
agent: ai_memory_core::AgentKind,
data_dir: &Path,
routing_key: &str,
) -> Option<LiveCallBatch> {
use sha2::{Digest as _, Sha256};
let key = serde_json::to_vec(&(path, agent.as_str(), routing_key)).ok()?;
let cursor_path = data_dir
.join("hook-state")
.join("token-usage")
.join(format!("{:x}.json", Sha256::digest(key)));
let mut cursor = std::fs::File::open(&cursor_path)
.ok()
.and_then(|file| {
let mut bytes = Vec::new();
file.take(LIVE_CURSOR_BYTES + 1)
.read_to_end(&mut bytes)
.ok()?;
(bytes.len() as u64 <= LIVE_CURSOR_BYTES)
.then(|| serde_json::from_slice::<LiveCursor>(&bytes).ok())
.flatten()
})
.unwrap_or_default();
let mut file = std::fs::File::open(path).ok()?;
let len = file.metadata().ok()?.len();
let prefix = fingerprint(&mut file, 0, len.min(64)).ok()?;
let anchor_start = cursor.offset.saturating_sub(64);
if cursor.offset > len
|| cursor.prefix != prefix
|| cursor.anchor
!= fingerprint(&mut file, anchor_start, cursor.offset - anchor_start).ok()?
{
cursor = LiveCursor::default();
}
file.seek(SeekFrom::Start(cursor.offset)).ok()?;
let mut bytes = Vec::new();
file.by_ref()
.take(CODEX_TAIL_SCAN_BYTES)
.read_to_end(&mut bytes)
.ok()?;
let mut result = std::collections::BTreeMap::<String, ai_memory_core::TokenCall>::new();
let mut order = Vec::new();
let mut previous = if cursor.offset == 0 {
Some([0; 4])
} else {
cursor.previous
};
let mut model = cursor.model.take();
let mut consumed = 0;
for line in bytes.split_inclusive(|b| *b == b'\n') {
if !line.ends_with(b"\n") {
// Retry an incomplete record. An oversized line is skipped in
// bounded chunks, rather than blocking every subsequent record.
if consumed == 0 && bytes.len() as u64 == CODEX_TAIL_SCAN_BYTES {
cursor.skipping_long_line = true;
cursor.context = response_context::ScanContext::default();
previous = None;
model = None;
consumed += line.len();
}
break;
}
consumed += line.len();
if cursor.skipping_long_line {
cursor.skipping_long_line = false;
continue;
}
if result.len() >= LIVE_BATCH_CALLS {
consumed -= line.len();
break;
}
let Ok(v) = serde_json::from_slice::<serde_json::Value>(line) else {
cursor.context = response_context::ScanContext::default();
continue;
};
if let Some(update) = cursor.context.observe(&v, agent) {
if !result.contains_key(&update.call_id) {
order.push(update.call_id.clone());
}
result
.entry(update.call_id.clone())
.and_modify(|old| response_context::merge_context(old, &update))
.or_insert(update);
}
if v.get("type").and_then(|v| v.as_str()) == Some("turn_context") {
model = v
.pointer("/payload/model")
.and_then(|v| v.as_str())
.filter(|s| s.len() <= 512)
.map(str::to_owned);
}
let (id, usage, current_model, codex) = if agent == ai_memory_core::AgentKind::ClaudeCode {
if v.get("type").and_then(|v| v.as_str()) != Some("assistant")
|| v.get("isMeta").and_then(|v| v.as_bool()) == Some(true)
{
continue;
}
let Some(id) = v.pointer("/message/id").and_then(|v| v.as_str()) else {
continue;
};
let Some(u) = v.pointer("/message/usage") else {
continue;
};
(
id.to_owned(),
u.clone(),
v.pointer("/message/model")
.and_then(|v| v.as_str())
.filter(|s| s.len() <= 512)
.map(str::to_owned),
false,
)
} else {
let total = v
.pointer("/payload/info/total_token_usage")
.or_else(|| v.pointer("/payload/thread_token_usage"));
let Some(total) = total else { continue };
let cached = u64_field(total, "cached_input_tokens");
let totals = [
u64_field(total, "input_tokens").saturating_sub(cached),
u64_field(total, "output_tokens"),
u64_field(total, "cache_write_input_tokens"),
cached,
];
let id = format!(
"codex:{}:{}:{}:{}",
totals[0], totals[1], totals[2], totals[3]
);
let usage = if let Some(last) = v
.pointer("/payload/info/last_token_usage")
.or_else(|| v.pointer("/payload/usage"))
{
last.clone()
} else if let Some(old) = previous {
serde_json::json!({"input_tokens":totals[0].saturating_sub(old[0])+totals[3].saturating_sub(old[3]),"output_tokens":totals[1].saturating_sub(old[1]),"cache_write_input_tokens":totals[2].saturating_sub(old[2]),"cached_input_tokens":totals[3].saturating_sub(old[3])})
} else {
previous = Some(totals);
continue;
};
// A duplicate cumulative checkpoint is not a second response.
if previous == Some(totals) {
continue;
}
previous = Some(totals);
(id, usage, model.clone(), true)
};
if id.len() > 128 {
continue;
}
let cached = u64_field(
&usage,
if codex {
"cached_input_tokens"
} else {
"cache_read_input_tokens"
},
);
let input = u64_field(&usage, "input_tokens");
let call = ai_memory_core::TokenCall {
context: Some(cursor.context.for_call(&v, agent)),
call_id: id.clone(),
input_tokens: if codex {
input.saturating_sub(cached)
} else {
input
},
output_tokens: u64_field(&usage, "output_tokens"),
cache_write_tokens: u64_field(
&usage,
if codex {
"cache_write_input_tokens"
} else {
"cache_creation_input_tokens"
},
),
cache_read_tokens: cached,
model: current_model,
};
if call
.input_tokens
.saturating_add(call.output_tokens)
.saturating_add(call.cache_read_tokens)
.saturating_add(call.cache_write_tokens)
== 0
{
continue;
}
if !result.contains_key(&id) {
order.push(id.clone());
}
cursor.context.remember(&call);
result
.entry(id)
.and_modify(|old| {
response_context::merge_context(old, &call);
old.input_tokens = old.input_tokens.max(call.input_tokens);
old.output_tokens = old.output_tokens.max(call.output_tokens);
old.cache_write_tokens = old.cache_write_tokens.max(call.cache_write_tokens);
old.cache_read_tokens = old.cache_read_tokens.max(call.cache_read_tokens);
})
.or_insert(call);
}
cursor.offset += consumed as u64;
cursor.previous = previous;
cursor.model = model;
cursor.prefix = prefix;
let anchor_start = cursor.offset.saturating_sub(64);
cursor.anchor = fingerprint(&mut file, anchor_start, cursor.offset - anchor_start).ok()?;
Some(LiveCallBatch {
calls: order
.into_iter()
.filter_map(|id| result.remove(&id))
.collect(),
cursor_path,
next: cursor,
})
}
/// Cumulative token usage for one session, as reported by the harness's own
/// transcript. An extractor returning `None` means "could not determine",
/// never "zero" — callers must not report a zeroed `SessionUsage`.
#[derive(Debug, Clone, PartialEq, Eq, Default)]
pub struct SessionUsage {
/// Fresh (non-cached) input tokens, summed/read from the transcript.
pub input_tokens: u64,
/// Output tokens, including any reasoning/thinking tokens the harness
/// reports as part of the same total (verified against real data: both
/// Claude Code's `output_tokens_details.thinking_tokens` and Codex's
/// `reasoning_output_tokens` are a breakdown of `output_tokens`, not
/// additive to it).
pub output_tokens: u64,
/// Tokens written to a prompt cache.
pub cache_write_tokens: u64,
/// Tokens served from a prompt cache.
pub cache_read_tokens: u64,
/// Last-seen model name, when the transcript records one.
pub model: Option<String>,
}
/// Refuse to scan a Claude Code transcript past this size. Guards against a
/// pathological file; real sessions are far smaller than this.
const CLAUDE_MAX_BYTES: u64 = 200 * 1024 * 1024;
/// How far from the end of a Codex rollout file to scan for the most recent
/// `token_usage_record`. Codex writes one after nearly every turn, so this
/// window holds one in any session that has had at least one model turn,
/// while staying bounded for the multi-hundred-MB rollout files a long
/// session produces (observed up to 103 MB locally).
const CODEX_TAIL_SCAN_BYTES: u64 = 4 * 1024 * 1024;
/// Sum `message.usage` across every non-meta assistant record in a Claude
/// Code JSONL transcript, deduplicating by `message.id`. Verified against a
/// real 2096-line/531-assistant-record local transcript: Claude Code's own
/// writer emits the same finalized message twice in a row (streaming, then
/// finalized) with byte-identical usage on every duplicate observed —
/// summing without the dedup roughly doubled the true total there.
#[must_use]
pub fn extract_claude_code_usage(transcript_path: &Path) -> Option<SessionUsage> {
let file = std::fs::File::open(transcript_path).ok()?;
if file.metadata().ok()?.len() > CLAUDE_MAX_BYTES {
return None;
}
let reader = std::io::BufReader::new(file);
let mut seen_ids: HashSet<String> = HashSet::new();
let mut totals = SessionUsage::default();
let mut any = false;
for line in reader.lines() {
let Ok(line) = line else { continue };
if line.is_empty() {
continue;
}
let Ok(value) = serde_json::from_str::<serde_json::Value>(&line) else {
continue;
};
if value.get("type").and_then(serde_json::Value::as_str) != Some("assistant") {
continue;
}
if value.get("isMeta").and_then(serde_json::Value::as_bool) == Some(true) {
continue;
}
let Some(message) = value.get("message") else {
continue;
};
let Some(id) = message.get("id").and_then(serde_json::Value::as_str) else {
continue;
};
if !seen_ids.insert(id.to_owned()) {
continue;
}
let Some(usage) = message.get("usage") else {
continue;
};
totals.input_tokens += u64_field(usage, "input_tokens");
totals.output_tokens += u64_field(usage, "output_tokens");
totals.cache_write_tokens += u64_field(usage, "cache_creation_input_tokens");
totals.cache_read_tokens += u64_field(usage, "cache_read_input_tokens");
if let Some(model) = message.get("model").and_then(serde_json::Value::as_str) {
totals.model = Some(model.to_owned());
}
any = true;
}
any.then_some(totals)
}
/// Read the most recent legacy usage record or modern `token_count` event
/// from a Codex rollout JSONL file. Its total usage is already cumulative
/// for this rollout file, so — unlike Claude Code — this needs no summing,
/// only the latest record. Scans backward from a bounded tail window rather
/// than the whole file, since a long Codex session's rollout can reach
/// hundreds of megabytes (observed locally); the window may begin mid-line,
/// but that partial first line simply fails to parse as JSON and is
/// skipped, which is safe because every LATER (more complete) line in an
/// append-only file this process never writes concurrently to is intact.
#[must_use]
pub fn extract_codex_usage(transcript_path: &Path) -> Option<SessionUsage> {
let mut file = std::fs::File::open(transcript_path).ok()?;
let len = file.metadata().ok()?.len();
let start = len.saturating_sub(CODEX_TAIL_SCAN_BYTES);
file.seek(SeekFrom::Start(start)).ok()?;
let mut buf = Vec::with_capacity(usize::try_from(len - start).unwrap_or(0));
file.read_to_end(&mut buf).ok()?;
let text = String::from_utf8_lossy(&buf);
for line in text.lines().rev() {
let Ok(value) = serde_json::from_str::<serde_json::Value>(line) else {
continue;
};
let record_type = value.get("type").and_then(serde_json::Value::as_str);
let modern = record_type == Some("event_msg")
&& value
.pointer("/payload/type")
.and_then(serde_json::Value::as_str)
== Some("token_count");
if record_type != Some("token_usage_record") && !modern {
continue;
}
let Some(usage) = value
.pointer("/payload/thread_token_usage")
.or_else(|| value.pointer("/payload/info/total_token_usage"))
else {
continue;
};
return Some(SessionUsage {
input_tokens: u64_field(usage, "input_tokens")
.saturating_sub(u64_field(usage, "cached_input_tokens")),
// `reasoning_output_tokens` is a breakdown of `output_tokens`
// (verified against real data: input_tokens + output_tokens ==
// total_tokens exactly), not additive to it.
output_tokens: u64_field(usage, "output_tokens"),
cache_write_tokens: u64_field(usage, "cache_write_input_tokens"),
cache_read_tokens: u64_field(usage, "cached_input_tokens"),
// Not carried on this record; left unset rather than guessed.
model: None,
});
}
None
}
fn u64_field(value: &serde_json::Value, key: &str) -> u64 {
value
.get(key)
.and_then(serde_json::Value::as_u64)
.unwrap_or(0)
.min(1_000_000_000)
}
/// Bound how many rollout files a Codex session-id lookup will stat before
/// giving up, mirroring the scan caps used elsewhere in this codebase
/// (`MAX_SCAN_FILES` in `ai-memory-workstream`) — a install with years of
/// rollout history must not turn a session-end hook into an unbounded walk.
const MAX_CODEX_ROLLOUT_SCAN: usize = 20_000;
/// Locate a Codex rollout file by the session id ai-memory's own hooks
/// receive. Codex names each rollout `rollout-<timestamp>-<uuid>.jsonl`
/// under `~/.codex/sessions/<year>/<month>/<day>/`; the trailing UUID is
/// the same id Codex's hook payloads send as `session_id` (verified: the
/// server-visible session id for a real local Codex session matched the
/// UUID suffix of that session's own rollout file, not the different
/// cross-resume id recorded inside `token_usage_record.payload.session_id`
/// on files from an earlier, resumed thread).
#[must_use]
pub fn locate_codex_rollout(codex_home: &Path, session_id: &str) -> Option<PathBuf> {
if session_id.is_empty()
|| !session_id
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-')
{
return None;
}
let suffix = format!("-{session_id}.jsonl");
let root = codex_home.join("sessions");
// UUIDv7 carries the creation date; most current Codex sessions can be
// found in one day directory instead of walking years of rollouts.
if let Ok(id) = uuid::Uuid::parse_str(session_id)
&& id.get_version_num() == 7
{
let mut ms = 0_i64;
for byte in &id.as_bytes()[..6] {
ms = (ms << 8) | i64::from(*byte);
}
if let Ok(time) = jiff::Timestamp::from_millisecond(ms) {
let day = root.join(
time.to_zoned(jiff::tz::TimeZone::UTC)
.strftime("%Y/%m/%d")
.to_string(),
);
if let Ok(entries) = std::fs::read_dir(day) {
for entry in entries.take(MAX_CODEX_ROLLOUT_SCAN).flatten() {
if entry.file_name().to_string_lossy().ends_with(&suffix) {
return Some(entry.path());
}
}
}
}
}
let mut scanned = 0usize;
let mut stack = vec![root];
while let Some(dir) = stack.pop() {
let Ok(entries) = std::fs::read_dir(&dir) else {
continue;
};
for entry in entries.flatten() {
if scanned >= MAX_CODEX_ROLLOUT_SCAN {
return None;
}
scanned += 1;
let path = entry.path();
let Ok(file_type) = entry.file_type() else {
continue;
};
if file_type.is_dir() {
stack.push(path);
continue;
}
if path
.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| name.ends_with(&suffix))
{
return Some(path);
}
}
}
None
}
/// Locate a Claude Code transcript file by session id and cwd, for a
/// session other than the current one (client-side backfill-close of an
/// abandoned session — the current session's own
/// `session-end` payload already carries `transcript_path` directly, so
/// this is only needed for a *different*, earlier session). Claude Code
/// stores each session at `<claude_config_dir>/projects/<flattened
/// cwd>/<session_id>.jsonl`, flattening `cwd` by turning every character
/// outside `[A-Za-z0-9]` into `-`.
#[must_use]
pub fn locate_claude_code_transcript(
claude_config_dir: &Path,
cwd: &str,
session_id: &str,
) -> Option<PathBuf> {
if cwd.is_empty()
|| !session_id
.chars()
.all(|c| c.is_ascii_alphanumeric() || c == '-')
{
return None;
}
let flattened = ai_memory_workstream::claude_project_dir_name(Path::new(cwd));
let path = claude_config_dir
.join("projects")
.join(flattened)
.join(format!("{session_id}.jsonl"));
path.is_file().then_some(path)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn live_context_pairs_tools_across_committed_hooks_and_resets_unknown_turns() {
use ai_memory_core::{AgentKind, TokenToolRelation};
use serde_json::json;
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("context.jsonl");
let rows = [
json!({"type":"user","uuid":"user-turn-1","message":{"content":[{"type":"text","text":"PRIVATE_PROMPT"}]}}),
json!({"type":"assistant","parentUuid":"user-turn-1","timestamp":"2026-10-01T12:00:00Z","message":{"id":"response-1","model":"model-1","usage":{"input_tokens":100,"output_tokens":5},"content":[{"type":"tool_use","id":"tool-1","name":"Read","input":{"path":"PRIVATE_PATH"}}]}}),
];
let data = rows.iter().map(|v| format!("{v}\n")).collect::<String>();
std::fs::write(&path, &data).unwrap();
let first = extract_live_calls(&path, AgentKind::ClaudeCode, tmp.path(), "ctx").unwrap();
let c = first.calls[0].context.as_ref().unwrap();
assert_eq!(c.turn_id.as_deref(), Some("user-turn-1"));
assert_eq!(c.response_id.as_deref(), Some("response-1"));
assert_eq!(c.tools[0].relation, TokenToolRelation::Requested);
assert_eq!(c.recorded_at.as_deref(), Some("2026-10-01T12:00:00Z"));
first.commit();
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&path)
.unwrap();
use std::io::Write as _;
writeln!(file,"{}",json!({"type":"user","uuid":"result-parent","message":{"content":[{"type":"tool_result","tool_use_id":"tool-1","is_error":true,"content":"PRIVATE_OUTPUT"}]}})).unwrap();
writeln!(file,"{}",json!({"type":"assistant","parentUuid":"result-parent","message":{"id":"response-2","usage":{"input_tokens":200,"output_tokens":7}}})).unwrap();
let next = extract_live_calls(&path, AgentKind::ClaudeCode, tmp.path(), "ctx").unwrap();
assert_eq!(next.calls.len(), 2);
let old = next
.calls
.iter()
.find(|c| c.call_id == "response-1")
.unwrap();
assert_eq!(old.context.as_ref().unwrap().tools[0].is_error, Some(true));
let input = next
.calls
.iter()
.find(|c| c.call_id == "response-2")
.unwrap();
assert_eq!(
input.context.as_ref().unwrap().tools[0].relation,
TokenToolRelation::Input
);
let serialized = serde_json::to_string(&next.next).unwrap();
assert!(
!serialized.contains("PRIVATE_"),
"Cursor must never retain prompt, tool arguments or output"
);
next.commit();
writeln!(file, "malformed gap").unwrap();
writeln!(
file,
"{}",
json!({"type":"assistant","message":{"id":"response-3","usage":{"input_tokens":3}}})
)
.unwrap();
let after = extract_live_calls(&path, AgentKind::ClaudeCode, tmp.path(), "ctx").unwrap();
assert!(after.calls[0].context.as_ref().unwrap().turn_id.is_none());
assert!(after.calls[0].context.as_ref().unwrap().tools.is_empty());
}
#[test]
fn live_codex_context_uses_native_coordinates_and_only_turn_association() {
use ai_memory_core::{AgentKind, TokenToolRelation};
use serde_json::json;
use std::io::Write as _;
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("codex-context.jsonl");
let context = json!({"type":"turn_context","payload":{"turn_id":"native-turn","model":"codex-model"}});
let usage = json!({"type":"token_usage","timestamp":"2026-10-01T12:00:00Z","payload":{"turn_id":"native-turn","response_id":"native-response","thread_token_usage":{"input_tokens":50,"output_tokens":10},"usage":{"input_tokens":50,"output_tokens":10}}});
std::fs::write(&path, format!("{context}\n{usage}\n")).unwrap();
let first = extract_live_calls(&path, AgentKind::Codex, tmp.path(), "ctx").unwrap();
assert_eq!(first.calls.len(), 1);
assert_eq!(
first.calls[0]
.context
.as_ref()
.unwrap()
.response_id
.as_deref(),
Some("native-response")
);
first.commit();
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&path)
.unwrap();
writeln!(file,"{}",json!({"type":"response_item","payload":{"type":"function_call","call_id":"exec-1","name":"exec_command","arguments":"PRIVATE_ARGUMENTS"}})).unwrap();
let update = extract_live_calls(&path, AgentKind::Codex, tmp.path(), "ctx").unwrap();
assert_eq!(update.calls.len(), 1);
let tool = &update.calls[0].context.as_ref().unwrap().tools[0];
assert_eq!(tool.relation, TokenToolRelation::Turn);
assert_eq!(tool.name, "exec_command");
update.commit();
writeln!(
file,
"{}",
json!({"type":"turn_context","payload":{"model":"no-turn"}})
)
.unwrap();
let mut unknown = usage;
unknown["payload"]
.as_object_mut()
.unwrap()
.remove("turn_id");
unknown["payload"]["thread_token_usage"]["input_tokens"] = json!(100);
writeln!(file, "{unknown}").unwrap();
let after = extract_live_calls(&path, AgentKind::Codex, tmp.path(), "ctx").unwrap();
assert!(after.calls[0].context.as_ref().unwrap().turn_id.is_none());
assert!(after.calls[0].context.as_ref().unwrap().tools.is_empty());
}
#[test]
fn live_token_calls_merge_streaming_usage_and_skip_partial_lines() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("transcript.jsonl");
std::fs::write(
&path,
format!(
"broken prefix\n{}\n{}\n{}\n{{partial",
claude_line("response-a", 50, 5, 10, 100),
claude_line("response-a", 50, 20, 10, 100),
claude_line("response-b", 70, 30, 0, 200)
),
)
.unwrap();
let calls = extract_live_calls(
&path,
ai_memory_core::AgentKind::ClaudeCode,
tmp.path(),
"fixture",
)
.unwrap()
.calls;
assert_eq!(calls.len(), 2);
assert_eq!(calls[1].call_id, "response-b");
assert_eq!(calls[0].output_tokens, 20);
}
#[test]
fn live_token_codex_reads_modern_usage_without_counting_cached_input_twice() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("rollout.jsonl");
let record=serde_json::json!({"type":"event_msg","payload":{"type":"token_count","info":{
"total_token_usage":{"input_tokens":1000,"output_tokens":40,"cached_input_tokens":800},
"last_token_usage":{"input_tokens":1000,"output_tokens":40,"cached_input_tokens":800,"reasoning_output_tokens":30}
}}}).to_string();
std::fs::write(&path, format!("{record}\n{record}\n")).unwrap();
let calls = extract_live_calls(
&path,
ai_memory_core::AgentKind::Codex,
tmp.path(),
"fixture",
)
.unwrap()
.calls;
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].input_tokens, 200);
assert_eq!(calls[0].output_tokens, 40);
assert_eq!(calls[0].cache_read_tokens, 800);
assert_eq!(extract_codex_usage(&path).unwrap().input_tokens, 200);
}
#[test]
fn live_token_cursor_replays_until_commit_then_recovers_every_response() {
for agent in [
ai_memory_core::AgentKind::ClaudeCode,
ai_memory_core::AgentKind::Codex,
] {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("transcript.jsonl");
let mut text = String::from(
"{\"type\":\"turn_context\",\"payload\":{\"model\":\"cursor-model\"}}\n",
);
for n in 1..=300 {
let line = if agent == ai_memory_core::AgentKind::ClaudeCode {
claude_line(&format!("r-{n}"), 100, 20, 0, 10)
} else {
serde_json::json!({"type":"event_msg","payload":{"type":"token_count","info":{"total_token_usage":{"input_tokens":100*n,"output_tokens":20*n,"cached_input_tokens":80*n}}}}).to_string()
};
text.push_str(&line);
text.push('\n');
}
std::fs::write(&path, text).unwrap();
let read =
|| extract_live_calls(&path, agent, tmp.path(), "server/project/session").unwrap();
let first = read();
assert_eq!(first.calls.len(), 256);
assert_eq!(
read().calls[0].call_id,
first.calls[0].call_id,
"Uncommitted data must replay"
);
first.commit();
let rest = read();
assert_eq!(rest.calls.len(), 44);
if agent == ai_memory_core::AgentKind::Codex {
assert!(rest.calls.iter().all(|c| c.input_tokens == 20
&& c.output_tokens == 20
&& c.cache_read_tokens == 80
&& c.model.as_deref() == Some("cursor-model")));
}
rest.commit();
assert!(read().calls.is_empty());
assert_eq!(
extract_live_calls(&path, agent, tmp.path(), "other/project/session")
.unwrap()
.calls
.len(),
256,
"Routes must not share progress"
);
std::fs::write(
&path,
format!("{}\n", claude_line("replacement", 1, 2, 0, 0)),
)
.unwrap();
if agent == ai_memory_core::AgentKind::ClaudeCode {
assert_eq!(read().calls[0].call_id, "replacement");
}
}
}
#[test]
fn live_token_cursor_crosses_byte_windows_and_retries_partial_usage() {
use std::io::Write as _;
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("transcript.jsonl");
let mut file = std::fs::File::create(&path).unwrap();
// Ordinary large records cross the 4 MiB read window; none may be
// skipped merely because it was outside the latest tail.
for n in 0..3 {
writeln!(file, "{}", claude_line(&format!("before-{n}"), 10, 1, 0, 0)).unwrap();
writeln!(
file,
"{}",
serde_json::json!({"type":"tool","body":"x".repeat(2*1024*1024)})
)
.unwrap();
}
let last = claude_line("after-gap", 20, 5, 0, 0);
write!(file, "{last}").unwrap();
drop(file);
let read = || {
extract_live_calls(
&path,
ai_memory_core::AgentKind::ClaudeCode,
tmp.path(),
"route",
)
.unwrap()
};
let mut ids = std::collections::HashSet::new();
for _ in 0..4 {
let batch = read();
ids.extend(batch.calls.iter().map(|c| c.call_id.clone()));
batch.commit();
}
assert_eq!(ids.len(), 3);
assert!(!ids.contains("after-gap"));
let mut file = std::fs::OpenOptions::new()
.append(true)
.open(&path)
.unwrap();
writeln!(file).unwrap();
let batch = read();
assert_eq!(batch.calls[0].call_id, "after-gap");
batch.commit();
writeln!(file, "{}", claude_line("after-gap", 20, 25, 0, 0)).unwrap();
let batch = read();
assert_eq!(batch.calls[0].output_tokens, 25);
batch.commit();
assert!(read().calls.is_empty());
}
fn claude_line(
msg_id: &str,
input: u64,
output: u64,
cache_write: u64,
cache_read: u64,
) -> String {
serde_json::json!({
"type": "assistant",
"message": {
"id": msg_id,
"model": "claude-sonnet-5",
"usage": {
"input_tokens": input,
"output_tokens": output,
"cache_creation_input_tokens": cache_write,
"cache_read_input_tokens": cache_read,
"output_tokens_details": {"thinking_tokens": output / 2}
}
}
})
.to_string()
}
#[test]
fn claude_code_sums_usage_and_dedups_by_message_id() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("session.jsonl");
// The same finalized message written twice (streaming then final),
// exactly as observed live — the duplicate must not double-count.
let body = format!(
"{}\n{}\n{}\n",
claude_line("msg_1", 2, 298, 67828, 0),
claude_line("msg_1", 2, 298, 67828, 0),
claude_line("msg_2", 2, 188, 371, 67828),
);
std::fs::write(&path, body).unwrap();
let usage = extract_claude_code_usage(&path).unwrap();
assert_eq!(usage.input_tokens, 4);
assert_eq!(usage.output_tokens, 486);
assert_eq!(usage.cache_write_tokens, 68199);
assert_eq!(usage.cache_read_tokens, 67828);
assert_eq!(usage.model.as_deref(), Some("claude-sonnet-5"));
}
#[test]
fn claude_code_ignores_meta_and_malformed_lines() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("session.jsonl");
let body = format!(
"not json\n{}\n{{\"type\":\"assistant\",\"isMeta\":true,\"message\":{{\"id\":\"m\",\"usage\":{{\"input_tokens\":999}}}}}}\n{}\n",
claude_line("real", 5, 5, 0, 0),
"",
);
std::fs::write(&path, body).unwrap();
let usage = extract_claude_code_usage(&path).unwrap();
assert_eq!(
usage.input_tokens, 5,
"isMeta and unparseable lines must be skipped"
);
}
#[test]
fn claude_code_missing_file_is_none_not_zero() {
let tmp = tempfile::tempdir().unwrap();
assert!(extract_claude_code_usage(&tmp.path().join("nope.jsonl")).is_none());
}
fn codex_record(input: u64, output: u64, cache_write: u64, cached: u64) -> String {
serde_json::json!({
"type": "token_usage_record",
"payload": {
"thread_token_usage": {
"input_tokens": input,
"cached_input_tokens": cached,
"cache_write_input_tokens": cache_write,
"output_tokens": output,
"reasoning_output_tokens": output / 3,
"total_tokens": input + output,
}
}
})
.to_string()
}
#[test]
fn codex_takes_the_last_cumulative_record() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("rollout.jsonl");
let body = format!(
"{}\n{{\"type\":\"response_item\",\"payload\":{{}}}}\n{}\n",
codex_record(26852, 145, 0, 4864),
codex_record(2_027_386, 1411, 0, 1_792_256),
);
std::fs::write(&path, body).unwrap();
let usage = extract_codex_usage(&path).unwrap();
assert_eq!(usage.input_tokens, 235_130);
assert_eq!(
usage.output_tokens, 1411,
"reasoning tokens are a breakdown of output_tokens, not additive"
);
assert_eq!(usage.cache_read_tokens, 1_792_256);
}
#[test]
fn codex_scans_only_the_tail_window() {
let tmp = tempfile::tempdir().unwrap();
let path = tmp.path().join("rollout.jsonl");
// Padding well past the tail-scan window, followed by the real
// record: proves the extractor never needed to read the padding.
let padding = "x".repeat((CODEX_TAIL_SCAN_BYTES as usize) * 3);
let body = format!("{padding}\n{}\n", codex_record(100, 50, 0, 10));
std::fs::write(&path, body).unwrap();
let usage = extract_codex_usage(&path).unwrap();
assert_eq!(usage.input_tokens, 90);
}
#[test]
fn codex_missing_file_is_none() {
let tmp = tempfile::tempdir().unwrap();
assert!(extract_codex_usage(&tmp.path().join("nope.jsonl")).is_none());
}
#[test]
fn locate_codex_rollout_finds_nested_file_by_suffix() {
let tmp = tempfile::tempdir().unwrap();
let day_dir = tmp.path().join("sessions/2026/09/22");
std::fs::create_dir_all(&day_dir).unwrap();
let target =
day_dir.join("rollout-2026-09-22T17-39-48-01a0cad8-9932-7433-9c14-97b62a6d5049.jsonl");
std::fs::write(&target, "").unwrap();
// A sibling file with a different id must not match.
std::fs::write(
day_dir.join("rollout-2026-09-22T10-00-00-deadbeef-0000-0000-0000-000000000000.jsonl"),
"",
)
.unwrap();
let found =
locate_codex_rollout(tmp.path(), "01a0cad8-9932-7433-9c14-97b62a6d5049").unwrap();
assert_eq!(found, target);
}
#[test]
fn locate_codex_rollout_rejects_unsafe_session_ids() {
let tmp = tempfile::tempdir().unwrap();
assert!(locate_codex_rollout(tmp.path(), "").is_none());
assert!(locate_codex_rollout(tmp.path(), "../../etc/passwd").is_none());
assert!(locate_codex_rollout(tmp.path(), "has spaces").is_none());
}
#[test]
fn locate_codex_rollout_missing_session_is_none() {
let tmp = tempfile::tempdir().unwrap();
std::fs::create_dir_all(tmp.path().join("sessions/2026/01/01")).unwrap();
assert!(locate_codex_rollout(tmp.path(), "no-such-session").is_none());
}
#[test]
fn locate_claude_code_transcript_uses_claudes_project_folder_rule() {
let tmp = tempfile::tempdir().unwrap();
let session_id = "9fc72c68-53ac-4d50-82aa-135f7ab03b7d";
for (cwd, folder) in [
(
"/home/me/src/github.com/acme/api",
"-home-me-src-github-com-acme-api",
),
("/home/me/my_app", "-home-me-my-app"),
("/home/me/my repo", "-home-me-my-repo"),
("/home/me/café", "-home-me-caf-"),
("/home/me/😀proj", "-home-me---proj"),
(r"C:\Users\me\my.repo", "C--Users-me-my-repo"),
] {
let project_dir = tmp.path().join("projects").join(folder);
std::fs::create_dir_all(&project_dir).unwrap();
let target = project_dir.join(format!("{session_id}.jsonl"));
std::fs::write(&target, "").unwrap();
assert_eq!(
locate_claude_code_transcript(tmp.path(), cwd, session_id),
Some(target),
"{cwd}"
);
}
}
#[test]
fn locate_claude_code_transcript_rejects_unsafe_input_and_missing_file() {
let tmp = tempfile::tempdir().unwrap();
assert!(locate_claude_code_transcript(tmp.path(), "", "sid").is_none());
assert!(locate_claude_code_transcript(tmp.path(), "/proj", "").is_none());
assert!(locate_claude_code_transcript(tmp.path(), "/proj", "../../etc/passwd").is_none());
assert!(locate_claude_code_transcript(tmp.path(), "/no/such/proj", "sid").is_none());
}
}