Multi-Agent Collaboration for Team and Friends
TypeScript agent-automationagent-orchestrationagent-workflowai-agentsai-collaborationai-memoryai-workforceautonomous-agentsclaude-codeclaude-code-skillscodexcoding-agentmulti-agentmultiple-agentone-person-businessone-person-companyone-person-teamopencodesolopreneur
Fix cumulative Claude token usage accounting (#654)
Gustavo Yeba35ffe2074d
diff --git a/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts b/src/daemon/agent-driver/src/adapters/claude/normalizer.test.tsindex 560e612b9..f214bbd87 100644--- a/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts+++ b/src/daemon/agent-driver/src/adapters/claude/normalizer.test.ts@@ -83,6 +83,287 @@ describe("ClaudeEventNormalizer.normalizeLine", () => {expect(n.normalizeLine(line).filter((event) => event.kind === "telemetry")).toHaveLength(1);});+ it("projects launch-scoped cumulative modelUsage high-water deltas across logical turns", () => {+ const n = new ClaudeEventNormalizer();+ const first = J({+ type: "result",+ subtype: "success",+ session_id: "s1",+ user_message_uuid: "root-1",+ total_cost_usd: 1,+ usage: { input_tokens: 1, output_tokens: 1 },+ modelUsage: {+ opus: {+ inputTokens: 10,+ outputTokens: 2,+ cacheReadInputTokens: 30,+ cacheCreationInputTokens: 8,+ costUSD: 1,+ },+ },+ });+ expect(n.normalizeLine(first)).toContainEqual({+ kind: "telemetry",+ name: "token_usage",+ source: "claude_result_model_usage",+ usage: { input: 10, output: 2, cache: 38 },+ });++ n.beginTurn();+ const second = J({+ type: "result",+ subtype: "success",+ session_id: "s1",+ user_message_uuid: "root-2",+ total_cost_usd: 1.7,+ usage: { input_tokens: 2, output_tokens: 2 },+ modelUsage: {+ opus: {+ inputTokens: 14,+ outputTokens: 5,+ cacheReadInputTokens: 50,+ cacheCreationInputTokens: 9,+ costUSD: 1.5,+ },+ sonnet: {+ inputTokens: 2,+ outputTokens: 1,+ cacheReadInputTokens: 0,+ cacheCreationInputTokens: 3,+ costUSD: 0.2,+ },+ },+ });+ expect(n.normalizeLine(second)).toContainEqual({+ kind: "telemetry",+ name: "token_usage",+ source: "claude_result_model_usage",+ usage: { input: 6, output: 4, cache: 24 },+ });+ expect(n.normalizeLine(second).filter((event) => event.kind === "telemetry")).toEqual([]);++ n.beginTurn();+ const unchanged = JSON.parse(second);+ unchanged.user_message_uuid = "root-3";+ expect(n.normalizeLine(J(unchanged)).filter((event) => event.kind === "telemetry")).toEqual([]);+ });++ it("fails cumulative modelUsage closed after session, model-set, component, or cost corruption", () => {+ const snapshot = (overrides: Record<string, unknown> = {}) => ({+ type: "result",+ subtype: "success",+ session_id: "s1",+ user_message_uuid: "root-1",+ total_cost_usd: 1,+ modelUsage: {+ opus: {+ inputTokens: 10,+ outputTokens: 2,+ cacheReadInputTokens: 30,+ cacheCreationInputTokens: 8,+ costUSD: 1,+ },+ },+ ...overrides,+ });+ const telemetry = (normalizer: ClaudeEventNormalizer, event: unknown) => normalizer+ .normalizeLine(J(event))+ .filter((item) => item.kind === "telemetry");++ const changedSession = new ClaudeEventNormalizer();+ expect(telemetry(changedSession, snapshot())).toHaveLength(1);+ changedSession.beginTurn();+ expect(telemetry(changedSession, snapshot({ session_id: "s2", user_message_uuid: "root-2" }))).toEqual([]);+ changedSession.beginTurn();+ expect(telemetry(changedSession, snapshot({ user_message_uuid: "root-3", usage: { input_tokens: 999 } }))).toEqual([]);++ const droppedModel = new ClaudeEventNormalizer();+ const withSecondModel = snapshot({+ total_cost_usd: 1.5,+ modelUsage: {+ ...snapshot().modelUsage,+ sonnet: {+ inputTokens: 1,+ outputTokens: 1,+ cacheReadInputTokens: 1,+ cacheCreationInputTokens: 1,+ costUSD: 0.5,+ },+ },+ });+ expect(telemetry(droppedModel, withSecondModel)).toHaveLength(1);+ droppedModel.beginTurn();+ expect(telemetry(droppedModel, snapshot({ user_message_uuid: "root-2" }))).toEqual([]);+ droppedModel.beginTurn();+ expect(telemetry(droppedModel, snapshot({+ user_message_uuid: "root-3",+ total_cost_usd: 1.7,+ modelUsage: {+ opus: {+ inputTokens: 12,+ outputTokens: 3,+ cacheReadInputTokens: 31,+ cacheCreationInputTokens: 9,+ costUSD: 1.1,+ },+ sonnet: {+ inputTokens: 2,+ outputTokens: 2,+ cacheReadInputTokens: 2,+ cacheCreationInputTokens: 2,+ costUSD: 0.6,+ },+ },+ }))).toContainEqual({+ kind: "telemetry",+ name: "token_usage",+ source: "claude_result_model_usage",+ usage: { input: 3, output: 2, cache: 4 },+ });++ const regressed = new ClaudeEventNormalizer();+ expect(telemetry(regressed, snapshot())).toHaveLength(1);+ regressed.beginTurn();+ expect(telemetry(regressed, snapshot({+ user_message_uuid: "root-2",+ total_cost_usd: 0.9,+ modelUsage: {+ opus: {+ ...snapshot().modelUsage.opus,+ inputTokens: 9,+ costUSD: 0.9,+ },+ },+ }))).toEqual([]);+ regressed.beginTurn();+ expect(telemetry(regressed, snapshot({+ user_message_uuid: "root-3",+ total_cost_usd: 1.2,+ modelUsage: {+ opus: {+ inputTokens: 15,+ outputTokens: 3,+ cacheReadInputTokens: 30,+ cacheCreationInputTokens: 8,+ costUSD: 1.2,+ },+ },+ }))).toContainEqual({+ kind: "telemetry",+ name: "token_usage",+ source: "claude_result_model_usage",+ usage: { input: 5, output: 1, cache: 0 },+ });++ const mismatchedCost = new ClaudeEventNormalizer();+ expect(telemetry(mismatchedCost, snapshot({ total_cost_usd: 2 }))).toEqual([]);++ const malformedThenValid = new ClaudeEventNormalizer();+ expect(telemetry(malformedThenValid, snapshot({+ usage: { input_tokens: 999, output_tokens: 999 },+ modelUsage: { opus: { ...snapshot().modelUsage.opus, inputTokens: "bad" } },+ }))).toEqual([]);+ malformedThenValid.beginTurn();+ expect(telemetry(malformedThenValid, snapshot({ user_message_uuid: "root-2" }))).toContainEqual({+ kind: "telemetry",+ name: "token_usage",+ source: "claude_result_model_usage",+ usage: { input: 10, output: 2, cache: 38 },+ });++ const legacyThenCumulative = new ClaudeEventNormalizer();+ expect(telemetry(legacyThenCumulative, {+ type: "result",+ subtype: "success",+ session_id: "s1",+ user_message_uuid: "legacy-1",+ usage: { input_tokens: 3, output_tokens: 2 },+ })).toHaveLength(1);+ legacyThenCumulative.beginTurn();+ expect(telemetry(legacyThenCumulative, snapshot({ user_message_uuid: "root-2" }))).toEqual([]);+ });++ it("covers malformed cumulative snapshots and cumulative-to-legacy mode changes", () => {+ const model = (overrides: Record<string, unknown> = {}) => ({+ inputTokens: 10,+ outputTokens: 2,+ cacheReadInputTokens: 30,+ cacheCreationInputTokens: 8,+ costUSD: 1,+ ...overrides,+ });+ const result = (overrides: Record<string, unknown> = {}) => ({+ type: "result",+ subtype: "success",+ session_id: "s1",+ user_message_uuid: "root-1",+ total_cost_usd: 1,+ modelUsage: { opus: model() },+ ...overrides,+ });+ const telemetry = (normalizer: ClaudeEventNormalizer, event: unknown) => normalizer+ .normalizeLine(J(event))+ .filter((item) => item.kind === "telemetry");++ for (const malformed of [+ result({ modelUsage: null }),+ result({ modelUsage: {} }),+ result({ modelUsage: { " ": model() } }),+ ]) {+ expect(telemetry(new ClaudeEventNormalizer(), malformed)).toEqual([]);+ }++ const malformedCrossSession = new ClaudeEventNormalizer();+ expect(telemetry(malformedCrossSession, result({+ modelUsage: { opus: model({ inputTokens: "bad" }) },+ }))).toEqual([]);+ malformedCrossSession.beginTurn();+ expect(telemetry(malformedCrossSession, result({+ session_id: "s2",+ user_message_uuid: "root-2",+ }))).toEqual([]);++ const countOverflow = new ClaudeEventNormalizer();+ expect(telemetry(countOverflow, result({+ total_cost_usd: 0,+ modelUsage: {+ opus: model({ inputTokens: Number.MAX_SAFE_INTEGER, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: 0 }),+ sonnet: model({ inputTokens: 1, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: 0 }),+ },+ }))).toEqual([]);++ const costOverflow = new ClaudeEventNormalizer();+ expect(telemetry(costOverflow, result({+ total_cost_usd: undefined,+ modelUsage: {+ opus: model({ inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: Number.MAX_VALUE }),+ sonnet: model({ inputTokens: 0, outputTokens: 0, cacheReadInputTokens: 0, cacheCreationInputTokens: 0, costUSD: Number.MAX_VALUE }),+ },+ }))).toEqual([]);++ const costRegression = new ClaudeEventNormalizer();+ expect(telemetry(costRegression, result())).toHaveLength(1);+ costRegression.beginTurn();+ expect(telemetry(costRegression, result({+ user_message_uuid: "root-2",+ total_cost_usd: 0.9,+ modelUsage: { opus: model({ costUSD: 0.9 }) },+ }))).toEqual([]);++ const cumulativeThenLegacy = new ClaudeEventNormalizer();+ expect(telemetry(cumulativeThenLegacy, result())).toHaveLength(1);+ cumulativeThenLegacy.beginTurn();+ expect(telemetry(cumulativeThenLegacy, {+ type: "result",+ subtype: "success",+ session_id: "s1",+ user_message_uuid: "root-2",+ usage: { input_tokens: 999, output_tokens: 999 },+ })).toEqual([]);+ cumulativeThenLegacy.beginTurn();+ expect(telemetry(cumulativeThenLegacy, result({ user_message_uuid: "root-3" }))).toEqual([]);+ });+it("does not emit usage without a backend session identity", () => {const out = new ClaudeEventNormalizer().normalizeLine(J({ type: "result", subtype: "success", usage: { input_tokens: 3, output_tokens: 5 } }),diff --git a/src/daemon/agent-driver/src/adapters/claude/normalizer.ts b/src/daemon/agent-driver/src/adapters/claude/normalizer.tsindex 1f929b399..a15926727 100644--- a/src/daemon/agent-driver/src/adapters/claude/normalizer.ts+++ b/src/daemon/agent-driver/src/adapters/claude/normalizer.ts@@ -19,10 +19,52 @@ import { tryParseJsonLine } from "../../internal/utils.js";import type { ClaudeTurnProtocol } from "./turnProtocol.js";const API_ERROR_RE = /API Error:.*(?:Connection error|\b[45]\d{2}\b)/i;+const CLAUDE_USAGE_COMPONENTS = [+ "inputTokens",+ "outputTokens",+ "cacheReadInputTokens",+ "cacheCreationInputTokens",+] as const;++interface ClaudeModelUsageSnapshot {+ inputTokens: number;+ outputTokens: number;+ cacheReadInputTokens: number;+ cacheCreationInputTokens: number;+ costUSD: number;+}++interface ClaudeModelUsageProjection {+ highWater: Map<string, ClaudeModelUsageSnapshot>;+ input: number;+ output: number;+ cacheRead: number;+ cacheWrite: number;+}++function safeCount(value: unknown): number | null {+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;+}++function safeMoney(value: unknown): number | null {+ return typeof value === "number" && Number.isFinite(value) && value >= 0 ? value : null;+}++function safeAdd(left: number, right: number): number | null {+ const total = left + right;+ return Number.isSafeInteger(total) ? total : null;+}++function equalCost(left: number, right: number): boolean {+ return Math.abs(left - right) <= Math.max(1e-12, Math.abs(right) * 1e-9);+}export class ClaudeEventNormalizer {private currentSession: string | null = null;private readonly usageProjector = new SettledUsageProjector();+ private claudeUsageSession: string | null = null;+ private claudeModelUsageHighWater = new Map<string, ClaudeModelUsageSnapshot>();+ private claudeUsageMode: "unknown" | "legacy" | "model_usage" | "invalid" = "unknown";constructor(private readonly turnProtocol?: ClaudeTurnProtocol) {}@@ -137,13 +179,40 @@ export class ClaudeEventNormalizer {}private buildUsageTelemetry(event: any, rootRequestId: string | null): AdapterEvent | null {- const u = event?.usage;- if (!u) return null;const backendSessionId = event.session_id ?? this.currentSession;if (typeof backendSessionId !== "string" || !backendSessionId) return null;const providerRecordId = rootRequestId?? (typeof event.request_id === "string" ? event.request_id : "invocation-result");- return this.usageProjector.project({+ const cumulative = this.prepareClaudeModelUsage(event, backendSessionId);+ if (cumulative === null) return null;+ if (cumulative) {+ const hasDelta = cumulative.input > 0+ || cumulative.output > 0+ || cumulative.cacheRead > 0+ || cumulative.cacheWrite > 0;+ if (!hasDelta) {+ this.commitClaudeModelUsage(backendSessionId, cumulative);+ return null;+ }+ const projected = this.usageProjector.project({+ runtime: "claude",+ backendSessionId,+ providerRecordId,+ source: "claude_result_model_usage",+ input: cumulative.input,+ output: cumulative.output,+ cacheRead: cumulative.cacheRead,+ cacheWrite: cumulative.cacheWrite,+ inputIncludesCache: false,+ outputIncludesReasoning: true,+ });+ if (projected) this.commitClaudeModelUsage(backendSessionId, cumulative);+ return projected;+ }++ const u = event?.usage;+ if (!u) return null;+ const projected = this.usageProjector.project({runtime: "claude",backendSessionId,providerRecordId,@@ -155,5 +224,112 @@ export class ClaudeEventNormalizer {inputIncludesCache: false,outputIncludesReasoning: true,});+ if (projected) this.claudeUsageMode = "legacy";+ return projected;+ }++ /**+ * Claude Code reports `result.modelUsage` as a cumulative snapshot for the+ * lifetime of one physical stream-json process. Keep that high-water mark+ * across logical turns and project only the component-wise delta.+ *+ * `undefined` means this runtime has not exposed modelUsage, so the legacy+ * per-result `usage` fallback is still safe. `null` skips the snapshot+ * without committing it. Malformed/regressed snapshots may recover on a+ * later complete high-water; session or usage-mode switches lock the+ * normalizer closed because their accounting identities cannot be joined.+ */+ private prepareClaudeModelUsage(+ event: any,+ backendSessionId: string,+ ): ClaudeModelUsageProjection | null | undefined {+ if (this.claudeUsageMode === "invalid") return null;+ if (!Object.hasOwn(event ?? {}, "modelUsage")) {+ if (this.claudeUsageMode !== "model_usage") return undefined;+ return event?.usage ? this.lockClaudeUsageMode() : null;+ }+ if (this.claudeUsageMode === "legacy") return this.lockClaudeUsageMode();+ if (this.claudeUsageMode === "unknown") this.claudeUsageMode = "model_usage";+ this.claudeUsageSession ??= backendSessionId;+ if (this.claudeUsageSession !== null && this.claudeUsageSession !== backendSessionId) {+ return this.lockClaudeUsageMode();+ }+ if (!event.modelUsage || typeof event.modelUsage !== "object" || Array.isArray(event.modelUsage)) {+ return null;+ }+ const entries = Object.entries(event.modelUsage);+ if (entries.length === 0) return null;++ const highWater = new Map<string, ClaudeModelUsageSnapshot>();+ let input = 0;+ let output = 0;+ let cacheRead = 0;+ let cacheWrite = 0;+ let snapshotCost = 0;+ for (const [model, rawValue] of entries) {+ if (!model.trim() || !rawValue || typeof rawValue !== "object" || Array.isArray(rawValue)) {+ return null;+ }+ const raw = rawValue as Record<string, unknown>;+ const current = {+ inputTokens: safeCount(raw.inputTokens),+ outputTokens: safeCount(raw.outputTokens),+ cacheReadInputTokens: safeCount(raw.cacheReadInputTokens),+ cacheCreationInputTokens: safeCount(raw.cacheCreationInputTokens),+ costUSD: safeMoney(raw.costUSD),+ };+ if (Object.values(current).some((value) => value === null)) {+ return null;+ }+ const snapshot = current as ClaudeModelUsageSnapshot;+ const prior = this.claudeModelUsageHighWater.get(model) ?? {+ inputTokens: 0,+ outputTokens: 0,+ cacheReadInputTokens: 0,+ cacheCreationInputTokens: 0,+ costUSD: 0,+ };+ for (const component of CLAUDE_USAGE_COMPONENTS) {+ if (snapshot[component] < prior[component]) return null;+ }+ if (snapshot.costUSD < prior.costUSD) return null;++ const nextInput = safeAdd(input, snapshot.inputTokens - prior.inputTokens);+ const nextOutput = safeAdd(output, snapshot.outputTokens - prior.outputTokens);+ const nextCacheRead = safeAdd(cacheRead, snapshot.cacheReadInputTokens - prior.cacheReadInputTokens);+ const nextCacheWrite = safeAdd(cacheWrite, snapshot.cacheCreationInputTokens - prior.cacheCreationInputTokens);+ if (nextInput === null || nextOutput === null || nextCacheRead === null || nextCacheWrite === null) {+ return null;+ }+ input = nextInput;+ output = nextOutput;+ cacheRead = nextCacheRead;+ cacheWrite = nextCacheWrite;+ snapshotCost += snapshot.costUSD;+ if (!Number.isFinite(snapshotCost)) return null;+ highWater.set(model, snapshot);+ }+ for (const model of this.claudeModelUsageHighWater.keys()) {+ if (!highWater.has(model)) return null;+ }+ if (Object.hasOwn(event, "total_cost_usd")) {+ const totalCost = safeMoney(event.total_cost_usd);+ if (totalCost === null || !equalCost(snapshotCost, totalCost)) return null;+ }+ return { highWater, input, output, cacheRead, cacheWrite };+ }++ private commitClaudeModelUsage(+ backendSessionId: string,+ projection: ClaudeModelUsageProjection,+ ): void {+ this.claudeUsageSession ??= backendSessionId;+ this.claudeModelUsageHighWater = projection.highWater;+ this.claudeUsageMode = "model_usage";+ }++ private lockClaudeUsageMode(): null {+ this.claudeUsageMode = "invalid";+ return null;}}