Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
import { isJsonObject, decodeJsonString } from "../json.ts";
import * as Predicate from "effect/Predicate";
import * as Schema from "effect/Schema";
import type { UsageProviderKind, UsageTokenTotals } from "@akeru/contracts";
/**
* Pure parsers for the provider CLIs' on-disk session transcripts.
*
* Both parsers are line-at-a-time reducers so callers can stream large files
* without materialising them. Neither touches the filesystem.
*
* @module usageTranscripts
*/
export interface UsageRecord {
readonly provider: UsageProviderKind;
readonly timestampMs: number;
readonly model: string;
readonly sessionId: string;
readonly totals: UsageTokenTotals;
readonly reportedCostUsd: number | null;
/**
* Key for cross-file de-duplication, or `null` when the record is inherently
* unique and needs no dedup.
*/
readonly dedupeKey: string | null;
}
const EMPTY_TOTALS: UsageTokenTotals = {
uncachedInputTokens: 0,
cachedInputTokens: 0,
cacheCreationTokens: 0,
outputTokens: 0,
reasoningTokens: 0,
};
function int(value: Schema.Json | undefined): number {
return Predicate.isNumber(value) && Number.isFinite(value) && value > 0 ? Math.trunc(value) : 0;
}
function parseTimestampMs(value: Schema.Json | undefined): number | null {
if (!Predicate.isString(value)) return null;
const parsed = Date.parse(value);
return Number.isNaN(parsed) ? null : parsed;
}
export function addTotals(a: UsageTokenTotals, b: UsageTokenTotals): UsageTokenTotals {
return {
uncachedInputTokens: a.uncachedInputTokens + b.uncachedInputTokens,
cachedInputTokens: a.cachedInputTokens + b.cachedInputTokens,
cacheCreationTokens: a.cacheCreationTokens + b.cacheCreationTokens,
outputTokens: a.outputTokens + b.outputTokens,
reasoningTokens: a.reasoningTokens + b.reasoningTokens,
};
}
export function totalTokens(totals: UsageTokenTotals): number {
// reasoningTokens is a subset of outputTokens and must not be added again.
return (
totals.uncachedInputTokens +
totals.cachedInputTokens +
totals.cacheCreationTokens +
totals.outputTokens
);
}
/**
* Cheap substring gate applied before `JSON.parse`.
*
* Transcripts are mostly tool output; only a minority of lines carry usage. On
* a 30-day window this skips roughly half the lines outright and is worth about
* an order of magnitude.
*/
export function mightCarryUsage(line: string, provider: UsageProviderKind): boolean {
return provider === "claude" ? line.includes('"usage"') : line.includes('"token_count"');
}
/* -------------------------------------------------------------------------- */
/* Claude Code */
/* -------------------------------------------------------------------------- */
/**
* Parses one line of a Claude Code transcript.
*
* Akeru Bot writes one record per assistant *content block*, and every one of
* those records repeats the same complete `usage` object for the parent
* message. Summing them overcounts by roughly 2.4x on a real workload, so the
* caller must drop repeats by `dedupeKey` and keep the first.
*/
export function parseClaudeLine(line: string): UsageRecord | null {
let parsed: Schema.Json;
try {
parsed = decodeJsonString(line);
} catch {
return null;
}
if (!isJsonObject(parsed)) return null;
const record = parsed;
if (record["type"] !== "assistant") return null;
const message = record["message"];
if (!isJsonObject(message)) return null;
const messageRecord = message;
const usage = messageRecord["usage"];
if (!isJsonObject(usage)) return null;
const usageRecord = usage;
const timestampMs = parseTimestampMs(record["timestamp"]);
if (timestampMs === null) return null;
const model = Predicate.isString(messageRecord["model"]) ? messageRecord["model"] : "";
if (model.length === 0) return null;
const messageId = Predicate.isString(messageRecord["id"]) ? messageRecord["id"] : null;
const requestId = Predicate.isString(record["requestId"]) ? record["requestId"] : null;
// Matches ccusage: prefer the message/request pair, fall back to whichever
// half exists. Records with neither cannot be de-duplicated.
const dedupeKey =
messageId === null && requestId === null ? null : `${messageId ?? ""}:${requestId ?? ""}`;
const cost = record["costUSD"];
return {
provider: "claude",
timestampMs,
model,
sessionId: Predicate.isString(record["sessionId"]) ? record["sessionId"] : "",
totals: {
uncachedInputTokens: int(usageRecord["input_tokens"]),
cachedInputTokens: int(usageRecord["cache_read_input_tokens"]),
cacheCreationTokens: int(usageRecord["cache_creation_input_tokens"]),
outputTokens: int(usageRecord["output_tokens"]),
// Anthropic folds thinking tokens into output and does not break them out.
reasoningTokens: 0,
},
reportedCostUsd: Predicate.isNumber(cost) && Number.isFinite(cost) ? cost : null,
dedupeKey,
};
}
/* -------------------------------------------------------------------------- */
/* Codex */
/* -------------------------------------------------------------------------- */
/**
* Rolling state for a single Codex rollout file.
*
* Codex `token_count` events carry no model, so the model is carried forward
* from the most recent `turn_context`. Sessions that switch models mid-run
* attribute correctly from the switch onward.
*/
export interface CodexScanState {
model: string;
sessionId: string;
lastUsageSignature: string | null;
sawSessionMeta: boolean;
/** While true, leading usage events are re-stamped copies of parent history. */
suppressingForkCopies: boolean;
forkCopyAnchorMs: number;
}
export function initialCodexScanState(): CodexScanState {
return {
model: "",
sessionId: "",
lastUsageSignature: null,
sawSessionMeta: false,
suppressingForkCopies: false,
forkCopyAnchorMs: 0,
};
}
/**
* A forked or subagent rollout opens with the parent's full history copied in,
* every line re-stamped to the fork instant. Those copies are written in one
* synchronous burst (observed gaps 0-40ms), while the child's first genuine
* usage event only lands after a real model turn (observed 5s+). One second of
* separation splits the two cleanly; `ccusage` uses the same threshold.
*/
const FORK_COPY_MAX_GAP_MS = 1000;
/** Whether a `session_meta` payload marks the rollout as a fork or subagent. */
function isForkedSessionMeta(payload: Schema.JsonObject): boolean {
if (Predicate.isString(payload["forked_from_id"])) return true;
const source = payload["source"];
if (!isJsonObject(source)) return false;
const subagent = source["subagent"];
if (!isJsonObject(subagent)) return false;
const spawn = subagent["thread_spawn"];
if (!isJsonObject(spawn)) return false;
return Predicate.isString(spawn["parent_thread_id"]);
}
/**
* Feeds one line of a Codex rollout into `state`, returning a record when the
* line was a usage event.
*
* Deltas come from `last_token_usage`. Summing those across a session
* reconciles with the session's final `total_token_usage`, provided
* consecutive duplicate events are dropped, which this does.
*/
export function parseCodexLine(line: string, state: CodexScanState): UsageRecord | null {
let parsed: Schema.Json;
try {
parsed = decodeJsonString(line);
} catch {
return null;
}
if (!isJsonObject(parsed)) return null;
const record = parsed;
const payload = record["payload"];
if (!isJsonObject(payload)) return null;
const payloadRecord = payload;
const payloadType = payloadRecord["type"];
if (record["type"] === "session_meta") {
// Only the first meta describes this file's own session. A forked rollout
// repeats the ancestors' metas right after it; letting those through would
// reassign every subsequent record to an ancestor session.
if (state.sawSessionMeta) return null;
state.sawSessionMeta = true;
const id = payloadRecord["id"] ?? payloadRecord["session_id"];
if (Predicate.isString(id)) state.sessionId = id;
const metaTimestampMs = parseTimestampMs(record["timestamp"]);
if (metaTimestampMs !== null && isForkedSessionMeta(payloadRecord)) {
state.suppressingForkCopies = true;
state.forkCopyAnchorMs = metaTimestampMs;
}
return null;
}
if (record["type"] === "turn_context") {
if (Predicate.isString(payloadRecord["model"])) state.model = payloadRecord["model"];
return null;
}
if (payloadType !== "token_count") return null;
const info = payloadRecord["info"];
if (!isJsonObject(info)) return null;
const last = info["last_token_usage"];
if (!isJsonObject(last)) return null;
const lastRecord = last;
// Only an event that is otherwise eligible may consume the duplicate
// signature. A token_count arriving before its turn_context (no model yet)
// must not poison it, or the re-emitted copy after the model is known would
// be skipped as a duplicate and those tokens never counted.
const timestampMs = parseTimestampMs(record["timestamp"]);
if (timestampMs === null) return null;
if (state.model.length === 0) return null;
// Codex re-emits an unchanged token_count on some stream boundaries. Summing
// those would double count, so identical consecutive payloads are skipped.
const signature = JSON.stringify(lastRecord);
if (signature === state.lastUsageSignature) return null;
state.lastUsageSignature = signature;
// In a forked rollout the copied parent history was already counted from the
// parent's own file. Drop the leading burst; the first usage event separated
// from its predecessor by a real turn's worth of time ends it for good.
if (state.suppressingForkCopies) {
if (timestampMs - state.forkCopyAnchorMs < FORK_COPY_MAX_GAP_MS) {
state.forkCopyAnchorMs = timestampMs;
return null;
}
state.suppressingForkCopies = false;
}
const inputTokens = int(lastRecord["input_tokens"]);
const cachedInputTokens = int(lastRecord["cached_input_tokens"]);
const cacheCreationTokens = int(lastRecord["cache_write_input_tokens"]);
const outputTokens = int(lastRecord["output_tokens"]);
const totals: UsageTokenTotals = {
// Codex reports `input_tokens` inclusive of the cached portion.
uncachedInputTokens: Math.max(0, inputTokens - cachedInputTokens - cacheCreationTokens),
cachedInputTokens,
cacheCreationTokens,
outputTokens,
// Reported inside output_tokens, surfaced separately for the token mix.
reasoningTokens: Math.min(outputTokens, int(lastRecord["reasoning_output_tokens"])),
};
if (totalTokens(totals) === 0) return null;
return {
provider: "codex",
timestampMs,
model: state.model,
sessionId: state.sessionId,
totals,
// Codex does not report cost in the rollout.
reportedCostUsd: null,
// Events surviving the fork-copy suppression above are unique to this
// rollout, so they need no global dedup.
dedupeKey: null,
};
}
export { EMPTY_TOTALS };