Skip to content

armgit / alook

Public Imported from GitHub
0

Multi-Agent Collaboration for Team and Friends

Fix cumulative Claude token usage accounting (#654)

Gustavo Yeba35ffe2074d
diff --git a/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts b/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts
index 560e612b9..f214bbd87 100644
--- a/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts
+++ b/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts
@@ -83,6 +83,287 @@ describe("ClaudeEventNormalizer.normalizeLine", () => {
expect(n.normalizeLine(line).filter((event) => event.kind === "telemetry")).toHaveLength(1);
});
+ it("projects launch-scoped cumulative modelUsage high-water deltas across logical turns", () => {
+ const n = new ClaudeEventNormalizer();
+ const first = J({
+ type: "result",
+ subtype: "success",
+ session_id: "s1",
+ user_message_uuid: "root-1",
+ total_cost_usd: 1,
+ usage: { input_tokens: 1, output_tokens: 1 },
+ modelUsage: {
+ opus: {
+ inputTokens: 10,
+ outputTokens: 2,
+ cacheReadInputTokens: 30,
+ cacheCreationInputTokens: 8,
+ costUSD: 1,
+ },
+ },
+ });
+ expect(n.normalizeLine(first)).toContainEqual({
+ kind: "telemetry",
+ name: "token_usage",
+ source: "claude_result_model_usage",
+ usage: { input: 10, output: 2, cache: 38 },
+ });
+
+ n.beginTurn();
+ const second = J({
+ type: "result",
+ subtype: "success",
+ session_id: "s1",
+ user_message_uuid: "root-2",
+ total_cost_usd: 1.7,
+ usage: { input_tokens: 2, output_tokens: 2 },
+ modelUsage: {
+ opus: {
+ inputTokens: 14,
+ outputTokens: 5,
+ cacheReadInputTokens: 50,
+ cacheCreationInputTokens: 9,
+ costUSD: 1.5,
+ },
+ sonnet: {
+ inputTokens: 2,
+ outputTokens: 1,
+ cacheReadInputTokens: 0,
+ cacheCreationInputTokens: 3,
+ costUSD: 0.2,
+ },
+ },
+ });
+ expect(n.normalizeLine(second)).toContainEqual({
+ kind: "telemetry",
+ name: "token_usage",
+ source: "claude_result_model_usage",
+ usage: { input: 6, output: 4, cache: 24 },
+ });
+ expect(n.normalizeLine(second).filter((event) => event.kind === "telemetry")).toEqual([]);
+
+ n.beginTurn();
+ const unchanged = JSON.parse(second);
+ unchanged.user_message_uuid = "root-3";
+ expect(n.normalizeLine(J(unchanged)).filter((event) => event.kind === "telemetry")).toEqual([]);
+ });
+
+ it("fails cumulative modelUsage closed after session, model-set, component, or cost corruption", () => {
+ const snapshot = (overrides: Record<string, unknown> = {}) => ({
+ type: "result",
+ subtype: "success",
+ session_id: "s1",
+ user_message_uuid: "root-1",
+ total_cost_usd: 1,
+ modelUsage: {
+ opus: {
+ inputTokens: 10,
+ outputTokens: 2,
+ cacheReadInputTokens: 30,
+ cacheCreationInputTokens: 8,
+ costUSD: 1,
+ },
+ },
+ ...overrides,
+ });
+ const telemetry = (normalizer: ClaudeEventNormalizer, event: unknown) => normalizer
+ .normalizeLine(J(event))
+ .filter((item) => item.kind === "telemetry");
+
+ const changedSession = new ClaudeEventNormalizer();
+ expect(telemetry(changedSession, snapshot())).toHaveLength(1);
+ changedSession.beginTurn();
+ expect(telemetry(changedSession, snapshot({ session_id: "s2", user_message_uuid: "root-2" }))).toEqual([]);
+ changedSession.beginTurn();
+ expect(telemetry(changedSession, snapshot({ user_message_uuid: "root-3", usage: { input_tokens: 999 } }))).toEqual([]);
+
+ const droppedModel = new ClaudeEventNormalizer();
+ const withSecondModel = snapshot({
+ total_cost_usd: 1.5,
+ modelUsage: {
+ ...snapshot().modelUsage,
+ sonnet: {
+ inputTokens: 1,
+ outputTokens: 1,
+ cacheReadInputTokens: 1,
+ cacheCreationInputTokens: 1,
+ costUSD: 0.5,
+ },
+ },
+ });
+ expect(telemetry(droppedModel, withSecondModel)).toHaveLength(1);
+ droppedModel.beginTurn();
+ expect(telemetry(droppedModel, snapshot({ user_message_uuid: "root-2" }))).toEqual([]);
+ droppedModel.beginTurn();
+ expect(telemetry(droppedModel, snapshot({
+ user_message_uuid: "root-3",
+ total_cost_usd: 1.7,
+ modelUsage: {
+ opus: {
+ inputTokens: 12,
+ outputTokens: 3,
+ cacheReadInputTokens: 31,
+ cacheCreationInputTokens: 9,
+ costUSD: 1.1,
+ },
+ sonnet: {
+ inputTokens: 2,
+ outputTokens: 2,
+ cacheReadInputTokens: 2,
+ cacheCreationInputTokens: 2,
+ costUSD: 0.6,
+ },
+ },
+ }))).toContainEqual({
+ kind: "telemetry",
+ name: "token_usage",
+ source: "claude_result_model_usage",
+ usage: { input: 3, output: 2, cache: 4 },
+ });
+
+ const regressed = new ClaudeEventNormalizer();
+ expect(telemetry(regressed, snapshot())).toHaveLength(1);
+ regressed.beginTurn();
+ expect(telemetry(regressed, snapshot({
+ user_message_uuid: "root-2",
+ total_cost_usd: 0.9,
+ modelUsage: {
+ opus: {
+ ...snapshot().modelUsage.opus,
+ inputTokens: 9,
+ costUSD: 0.9,
+ },
+ },
+ }))).toEqual([]);
+ regressed.beginTurn();
+ expect(telemetry(regressed, snapshot({
+ user_message_uuid: "root-3",
+ total_cost_usd: 1.2,
+ modelUsage: {
+ opus: {
+ inputTokens: 15,
+ outputTokens: 3,
+ cacheReadInputTokens: 30,
+ cacheCreationInputTokens: 8,
+ costUSD: 1.2,
+ },
+ },
+ }))).toContainEqual({
+ kind: "telemetry",
+ name: "token_usage",
+ source: "claude_result_model_usage",
+ usage: { input: 5, output: 1, cache: 0 },
+ });
+
+ const mismatchedCost = new ClaudeEventNormalizer();
+ expect(telemetry(mismatchedCost, snapshot({ total_cost_usd: 2 }))).toEqual([]);
+
+ const malformedThenValid = new ClaudeEventNormalizer();
+ expect(telemetry(malformedThenValid, snapshot({
+ usage: { input_tokens: 999, output_tokens: 999 },
+ modelUsage: { opus: { ...snapshot().modelUsage.opus, inputTokens: "bad" } },
+ }))).toEqual([]);
+ malformedThenValid.beginTurn();
+ expect(telemetry(malformedThenValid, snapshot({ user_message_uuid: "root-2" }))).toContainEqual({
+ kind: "telemetry",
+ name: "token_usage",
+ source: "claude_result_model_usage",
+ usage: { input: 10, output: 2, cache: 38 },
+ });
+
+ const legacyThenCumulative = new ClaudeEventNormalizer();
+ expect(telemetry(legacyThenCumulative, {
+ type: "result",
+ subtype: "success",
+ session_id: "s1",
+ user_message_uuid: "legacy-1",
+ usage: { input_tokens: 3, output_tokens: 2 },
+ })).toHaveLength(1);
+ legacyThenCumulative.beginTurn();
+ expect(telemetry(legacyThenCumulative, snapshot({ user_message_uuid: "root-2" }))).toEqual([]);
+ });
+
+ it("covers malformed cumulative snapshots and cumulative-to-legacy mode changes", () => {
+ const model = (overrides: Record<string, unknown> = {}) => ({
+ inputTokens: 10,
+ outputTokens: 2,
+ cacheReadInputTokens: 30,
+ cacheCreationInputTokens: 8,
+ costUSD: 1,
+ ...overrides,
+ });
+ const result = (overrides: Record<string, unknown> = {}) => ({
+ type: "result",
+ subtype: "success",
+ session_id: "s1",
+ user_message_uuid: "root-1",
+ total_cost_usd: 1,
+ modelUsage: { opus: model() },
+ ...overrides,
+ });
+ const telemetry = (normalizer: ClaudeEventNormalizer, event: unknown) => normalizer
+ .normalizeLine(J(event))
+ .filter((item) => item.kind === "telemetry");
+
+ for (const malformed of [
+ result({ modelUsage: null }),
+ result({ modelUsage: {} }),
+ result({ modelUsage: { " ": model() } }),
+ ]) {
+ expect(telemetry(new ClaudeEventNormalizer(), malformed)).toEqual([]);
+ }
+
+ const malformedCrossSession = new ClaudeEventNormalizer();
+ expect(telemetry(malformedCrossSession, result({
+ modelUsage: { opus: model({ inputTokens: "bad" }) },
+ }))).toEqual([]);
+ malformedCrossSession.beginTurn();
+ expect(telemetry(malformedCrossSession, result({
+ session_id: "s2",
+ user_message_uuid: "root-2",
+ }))).toEqual([]);
+
+ const countOverflow = new ClaudeEventNormalizer();
+ expect(telemetry(countOverflow, result({
+ total_cost_usd: 0,
+ modelUsage: {
+ opus: model({ inputTokens: Number.MAX_SAFE_INTEGER, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: 0 }),
+ sonnet: model({ inputTokens: 1, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: 0 }),
+ },
+ }))).toEqual([]);
+
+ const costOverflow = new ClaudeEventNormalizer();
+ expect(telemetry(costOverflow, result({
+ total_cost_usd: undefined,
+ modelUsage: {
+ opus: model({ inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: Number.MAX_VALUE }),
+ sonnet: model({ inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: Number.MAX_VALUE }),
+ },
+ }))).toEqual([]);
+
+ const costRegression = new ClaudeEventNormalizer();
+ expect(telemetry(costRegression, result())).toHaveLength(1);
+ costRegression.beginTurn();
+ expect(telemetry(costRegression, result({
+ user_message_uuid: "root-2",
+ total_cost_usd: 0.9,
+ modelUsage: { opus: model({ costUSD: 0.9 }) },
+ }))).toEqual([]);
+
+ const cumulativeThenLegacy = new ClaudeEventNormalizer();
+ expect(telemetry(cumulativeThenLegacy, result())).toHaveLength(1);
+ cumulativeThenLegacy.beginTurn();
+ expect(telemetry(cumulativeThenLegacy, {
+ type: "result",
+ subtype: "success",
+ session_id: "s1",
+ user_message_uuid: "root-2",
+ usage: { input_tokens: 999, output_tokens: 999 },
+ })).toEqual([]);
+ cumulativeThenLegacy.beginTurn();
+ expect(telemetry(cumulativeThenLegacy, result({ user_message_uuid: "root-3" }))).toEqual([]);
+ });
+
it("does not emit usage without a backend session identity", () => {
const out = new ClaudeEventNormalizer().normalizeLine(
J({ type: "result", subtype: "success", usage: { input_tokens: 3, output_tokens: 5 } }),
diff --git a/src/daemon/agent-driver/src/adapters/claude/normalizer.ts b/src/daemon/agent-driver/src/adapters/claude/normalizer.ts
index 1f929b399..a15926727 100644
--- a/src/daemon/agent-driver/src/adapters/claude/normalizer.ts
+++ b/src/daemon/agent-driver/src/adapters/claude/normalizer.ts
@@ -19,10 +19,52 @@ import { tryParseJsonLine } from "../../internal/utils.js";
import type { ClaudeTurnProtocol } from "./turnProtocol.js";
const API_ERROR_RE = /API Error:.*(?:Connection error|\b[45]\d{2}\b)/i;
+const CLAUDE_USAGE_COMPONENTS = [
+ "inputTokens",
+ "outputTokens",
+ "cacheReadInputTokens",
+ "cacheCreationInputTokens",
+] as const;
+
+interface ClaudeModelUsageSnapshot {
+ inputTokens: number;
+ outputTokens: number;
+ cacheReadInputTokens: number;
+ cacheCreationInputTokens: number;
+ costUSD: number;
+}
+
+interface ClaudeModelUsageProjection {
+ highWater: Map<string, ClaudeModelUsageSnapshot>;
+ input: number;
+ output: number;
+ cacheRead: number;
+ cacheWrite: number;
+}
+
+function safeCount(value: unknown): number | null {
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
+}
+
+function safeMoney(value: unknown): number | null {
+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;
+}
+
+function safeAdd(left: number, right: number): number | null {
+ const total = left + right;
+ return Number.isSafeInteger(total) ? total : null;
+}
+
+function equalCost(left: number, right: number): boolean {
+ return Math.abs(left - right) <= Math.max(1e-12, Math.abs(right) * 1e-9);
+}
export class ClaudeEventNormalizer {
private currentSession: string | null = null;
private readonly usageProjector = new SettledUsageProjector();
+ private claudeUsageSession: string | null = null;
+ private claudeModelUsageHighWater = new Map<string, ClaudeModelUsageSnapshot>();
+ private claudeUsageMode: "unknown" | "legacy" | "model_usage" | "invalid" = "unknown";
constructor(private readonly turnProtocol?: ClaudeTurnProtocol) {}
@@ -137,13 +179,40 @@ export class ClaudeEventNormalizer {
}
private buildUsageTelemetry(event: any, rootRequestId: string | null): AdapterEvent | null {
- const u = event?.usage;
- if (!u) return null;
const backendSessionId = event.session_id ?? this.currentSession;
if (typeof backendSessionId !== "string" || !backendSessionId) return null;
const providerRecordId = rootRequestId
?? (typeof event.request_id === "string" ? event.request_id : "invocation-result");
- return this.usageProjector.project({
+ const cumulative = this.prepareClaudeModelUsage(event, backendSessionId);
+ if (cumulative === null) return null;
+ if (cumulative) {
+ const hasDelta = cumulative.input > 0
+ || cumulative.output > 0
+ || cumulative.cacheRead > 0
+ || cumulative.cacheWrite > 0;
+ if (!hasDelta) {
+ this.commitClaudeModelUsage(backendSessionId, cumulative);
+ return null;
+ }
+ const projected = this.usageProjector.project({
+ runtime: "claude",
+ backendSessionId,
+ providerRecordId,
+ source: "claude_result_model_usage",
+ input: cumulative.input,
+ output: cumulative.output,
+ cacheRead: cumulative.cacheRead,
+ cacheWrite: cumulative.cacheWrite,
+ inputIncludesCache: false,
+ outputIncludesReasoning: true,
+ });
+ if (projected) this.commitClaudeModelUsage(backendSessionId, cumulative);
+ return projected;
+ }
+
+ const u = event?.usage;
+ if (!u) return null;
+ const projected = this.usageProjector.project({
runtime: "claude",
backendSessionId,
providerRecordId,
@@ -155,5 +224,112 @@ export class ClaudeEventNormalizer {
inputIncludesCache: false,
outputIncludesReasoning: true,
});
+ if (projected) this.claudeUsageMode = "legacy";
+ return projected;
+ }
+
+ /**
+ * Claude Code reports `result.modelUsage` as a cumulative snapshot for the
+ * lifetime of one physical stream-json process. Keep that high-water mark
+ * across logical turns and project only the component-wise delta.
+ *
+ * `undefined` means this runtime has not exposed modelUsage, so the legacy
+ * per-result `usage` fallback is still safe. `null` skips the snapshot
+ * without committing it. Malformed/regressed snapshots may recover on a
+ * later complete high-water; session or usage-mode switches lock the
+ * normalizer closed because their accounting identities cannot be joined.
+ */
+ private prepareClaudeModelUsage(
+ event: any,
+ backendSessionId: string,
+ ): ClaudeModelUsageProjection | null | undefined {
+ if (this.claudeUsageMode === "invalid") return null;
+ if (!Object.hasOwn(event ?? {}, "modelUsage")) {
+ if (this.claudeUsageMode !== "model_usage") return undefined;
+ return event?.usage ? this.lockClaudeUsageMode() : null;
+ }
+ if (this.claudeUsageMode === "legacy") return this.lockClaudeUsageMode();
+ if (this.claudeUsageMode === "unknown") this.claudeUsageMode = "model_usage";
+ this.claudeUsageSession ??= backendSessionId;
+ if (this.claudeUsageSession !== null && this.claudeUsageSession !== backendSessionId) {
+ return this.lockClaudeUsageMode();
+ }
+ if (!event.modelUsage || typeof event.modelUsage !== "object" || Array.isArray(event.modelUsage)) {
+ return null;
+ }
+ const entries = Object.entries(event.modelUsage);
+ if (entries.length === 0) return null;
+
+ const highWater = new Map<string, ClaudeModelUsageSnapshot>();
+ let input = 0;
+ let output = 0;
+ let cacheRead = 0;
+ let cacheWrite = 0;
+ let snapshotCost = 0;
+ for (const [model, rawValue] of entries) {
+ if (!model.trim() || !rawValue || typeof rawValue !== "object" || Array.isArray(rawValue)) {
+ return null;
+ }
+ const raw = rawValue as Record<string, unknown>;
+ const current = {
+ inputTokens: safeCount(raw.inputTokens),
+ outputTokens: safeCount(raw.outputTokens),
+ cacheReadInputTokens: safeCount(raw.cacheReadInputTokens),
+ cacheCreationInputTokens: safeCount(raw.cacheCreationInputTokens),
+ costUSD: safeMoney(raw.costUSD),
+ };
+ if (Object.values(current).some((value) => value === null)) {
+ return null;
+ }
+ const snapshot = current as ClaudeModelUsageSnapshot;
+ const prior = this.claudeModelUsageHighWater.get(model) ?? {
+ inputTokens: 0,
+ outputTokens: 0,
+ cacheReadInputTokens: 0,
+ cacheCreationInputTokens: 0,
+ costUSD: 0,
+ };
+ for (const component of CLAUDE_USAGE_COMPONENTS) {
+ if (snapshot[component] < prior[component]) return null;
+ }
+ if (snapshot.costUSD < prior.costUSD) return null;
+
+ const nextInput = safeAdd(input, snapshot.inputTokens - prior.inputTokens);
+ const nextOutput = safeAdd(output, snapshot.outputTokens - prior.outputTokens);
+ const nextCacheRead = safeAdd(cacheRead, snapshot.cacheReadInputTokens - prior.cacheReadInputTokens);
+ const nextCacheWrite = safeAdd(cacheWrite, snapshot.cacheCreationInputTokens - prior.cacheCreationInputTokens);
+ if (nextInput === null || nextOutput === null || nextCacheRead === null || nextCacheWrite === null) {
+ return null;
+ }
+ input = nextInput;
+ output = nextOutput;
+ cacheRead = nextCacheRead;
+ cacheWrite = nextCacheWrite;
+ snapshotCost += snapshot.costUSD;
+ if (!Number.isFinite(snapshotCost)) return null;
+ highWater.set(model, snapshot);
+ }
+ for (const model of this.claudeModelUsageHighWater.keys()) {
+ if (!highWater.has(model)) return null;
+ }
+ if (Object.hasOwn(event, "total_cost_usd")) {
+ const totalCost = safeMoney(event.total_cost_usd);
+ if (totalCost === null || !equalCost(snapshotCost, totalCost)) return null;
+ }
+ return { highWater, input, output, cacheRead, cacheWrite };
+ }
+
+ private commitClaudeModelUsage(
+ backendSessionId: string,
+ projection: ClaudeModelUsageProjection,
+ ): void {
+ this.claudeUsageSession ??= backendSessionId;
+ this.claudeModelUsageHighWater = projection.highWater;
+ this.claudeUsageMode = "model_usage";
+ }
+
+ private lockClaudeUsageMode(): null {
+ this.claudeUsageMode = "invalid";
+ return null;
}
}