Skip to content

The open-source agent harness - the runtime layer that turns an LLM into a working agent.

AGE-1368: Expose aggregated turn metrics (tokens/cost) across parent and sub-agents (#7)

Bhavesh Patele5158254a066
diff --git a/fern/openapi/openapi.json b/fern/openapi/openapi.json
index 52d3e7527..cecbe8586 100644
--- a/fern/openapi/openapi.json
+++ b/fern/openapi/openapi.json
@@ -2130,6 +2130,39 @@
}
]
},
+ "TurnMetrics": {
+ "properties": {
+ "total_cache_read_tokens": {
+ "minimum": 0,
+ "type": "integer"
+ },
+ "total_cache_write_tokens": {
+ "minimum": 0,
+ "type": "integer"
+ },
+ "total_cost_in_usd": {
+ "minimum": 0,
+ "type": "number"
+ },
+ "total_input_tokens": {
+ "minimum": 0,
+ "type": "integer"
+ },
+ "total_output_tokens": {
+ "minimum": 0,
+ "type": "integer"
+ },
+ "total_reasoning_tokens": {
+ "minimum": 0,
+ "type": "integer"
+ },
+ "total_tokens": {
+ "minimum": 0,
+ "type": "integer"
+ }
+ },
+ "type": "object"
+ },
"TurnState": {
"discriminator": {
"mapping": {
@@ -2160,6 +2193,9 @@
"completed_at": {
"type": "string"
},
+ "metrics": {
+ "$ref": "#/components/schemas/TurnMetrics"
+ },
"reason": {
"$ref": "#/components/schemas/TurnStateCancelledReason"
},
@@ -2192,6 +2228,9 @@
"completed_at": {
"type": "string"
},
+ "metrics": {
+ "$ref": "#/components/schemas/TurnMetrics"
+ },
"output": {
"$ref": "#/components/schemas/ModelMessageEvent"
},
@@ -2223,6 +2262,9 @@
"message": {
"type": "string"
},
+ "metrics": {
+ "$ref": "#/components/schemas/TurnMetrics"
+ },
"status": {
"enum": [
"error"
diff --git a/packages/harness/src/agent-session/TurnHandle.ts b/packages/harness/src/agent-session/TurnHandle.ts
index c90733480..f051583be 100644
--- a/packages/harness/src/agent-session/TurnHandle.ts
+++ b/packages/harness/src/agent-session/TurnHandle.ts
@@ -4,7 +4,6 @@
import { AgentHarnessError } from '../core/errors';
import type { MCPAuthRequiredEvent, ModelMessageDeltaEvent, ThreadDoneEvent } from '../core/events/schema';
import { EventType as HarnessEventType, newEventId } from '../core/events/schema';
-import { getEmptyUsage } from '../core/llm/LLMTypes';
import {
InternalEventType,
type AgentThreadExecutionEvent,
@@ -13,12 +12,19 @@ import {
type InternalThreadDoneEvent,
} from '../core/runtime/AgentThread.types';
import type { AgentThreadOrchestrator } from '../core/runtime/AgentThreadOrchestrator';
-import { createEmptyAgentThreadMetrics } from '../core/runtime/metrics';
+import { getEmptyCurrentContextUsage } from '../core/runtime/contextUsage';
+import type { AgentThreadMetrics } from '../core/runtime/metrics';
import type { ITurnResourceResolver } from './ITurnResourceResolver';
import type { TurnRecord } from './models/TurnRecord';
import { EventType, type PersistedTurnEvent, type TurnCreatedEvent, type TurnDoneEvent } from './schemas/events';
import type { TokenPagination } from './schemas/pagination';
-import { CancellationReason, type TerminalTurnState, type TurnInputItem, type TurnState } from './schemas/turn';
+import {
+ CancellationReason,
+ type TerminalTurnState,
+ type TurnInputItem,
+ type TurnMetrics,
+ type TurnState,
+} from './schemas/turn';
import type { ISessionStore } from './store/ISessionStore';
import { TurnNotRunningError } from './store/SessionStoreErrors';
@@ -67,6 +73,18 @@ function toMCPAuthRequiredEvent(event: InternalMCPAuthRequiredEvent): MCPAuthReq
};
}
+function turnMetricsFromAgentThreadMetrics(metrics: AgentThreadMetrics): TurnMetrics {
+ return {
+ total_input_tokens: metrics.total_input_tokens,
+ total_output_tokens: metrics.total_output_tokens,
+ total_tokens: metrics.total_tokens,
+ total_cache_read_tokens: metrics.total_cache_read_tokens,
+ total_cache_write_tokens: metrics.total_cache_write_tokens,
+ total_reasoning_tokens: metrics.total_reasoning_tokens,
+ total_cost_in_usd: metrics.total_cost_in_usd,
+ };
+}
+
export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
private readonly store: ISessionStore<object, TTurnCustom>;
private readonly tenantId: string;
@@ -214,17 +232,21 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
const emptyResult: AgentThreadExecutionResult = {
output: null,
required_actions: [],
- metrics: { total: createEmptyAgentThreadMetrics() },
};
await generator.return(emptyResult);
// Cleared so the finally block does not close the generator a second time.
generator = undefined;
- this.turn = { ...this.turn, state: error.state, updated_at: new Date().toISOString() };
+ const createdAt = new Date().toISOString();
+ const state: TerminalTurnState = {
+ ...error.state,
+ metrics: turnMetricsFromAgentThreadMetrics(orchestrator.getMetrics()),
+ };
+ this.turn = { ...this.turn, state, updated_at: createdAt };
yield {
type: EventType.TURN_DONE,
id: newEventId(),
- created_at: new Date().toISOString(),
- state: error.state,
+ created_at: createdAt,
+ state,
thread_id: null,
};
return;
@@ -241,30 +263,33 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
const emptyResult: AgentThreadExecutionResult = {
output: null,
required_actions: [],
- metrics: { total: createEmptyAgentThreadMetrics() },
};
await generator.return(emptyResult);
}
const createdAt = new Date().toISOString();
+ const metrics = turnMetricsFromAgentThreadMetrics(orchestrator.getMetrics());
let terminalState: TerminalTurnState;
if (signal.aborted) {
terminalState = {
status: 'cancelled',
reason: cancellationReasonFromAbortReason(signal.reason),
completed_at: createdAt,
+ metrics,
};
} else if (caughtError) {
terminalState = {
status: 'error',
message: caughtError.message,
completed_at: createdAt,
+ metrics,
};
} else if (executeResult?.root_agent_error) {
terminalState = {
status: 'error',
message: executeResult.root_agent_error.error,
completed_at: createdAt,
+ metrics,
};
} else if (executeResult === undefined) {
// Consumer abandoned the generator (break/return) without aborting —
@@ -273,6 +298,7 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
status: 'cancelled',
reason: CancellationReason.ClientCancelled,
completed_at: createdAt,
+ metrics,
};
} else {
terminalState = {
@@ -281,6 +307,7 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
...(executeResult.output && { output: executeResult.output }),
required_actions: executeResult.required_actions,
completed_at: createdAt,
+ metrics,
};
}
@@ -304,7 +331,8 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
this.turn = { ...this.turn, state: terminalState, updated_at: createdAt };
} catch (persistError) {
if (persistError instanceof TurnNotRunningError) {
- this.turn = { ...this.turn, state: persistError.state, updated_at: createdAt };
+ const state: TerminalTurnState = { ...persistError.state, metrics };
+ this.turn = { ...this.turn, state, updated_at: createdAt };
await resolver.close().catch(() => {
/* no-op */
});
@@ -312,7 +340,7 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
type: EventType.TURN_DONE,
id: newEventId(),
created_at: createdAt,
- state: persistError.state,
+ state,
thread_id: null,
};
// eslint-disable-next-line no-unsafe-finally -- deliberate: a concurrent freeze during the terminal write makes the store's state authoritative, so the stream ends here
@@ -448,7 +476,7 @@ export class TurnHandle<TTurnCustom extends object = Record<string, never>> {
parent: event.parent,
agent_info: event.agent_info,
context: [],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
completion: null,
capability_state: null,
},
diff --git a/packages/harness/src/agent-session/index.ts b/packages/harness/src/agent-session/index.ts
index 40fc05194..c326d5faf 100644
--- a/packages/harness/src/agent-session/index.ts
+++ b/packages/harness/src/agent-session/index.ts
@@ -11,6 +11,7 @@ export {
CreateTurnRequestSchema,
PreviousTurnIdInputSchema,
TurnInputItemSchema,
+ TurnMetricsSchema,
TurnSchema,
TurnStateCancelledReasonSchema,
TurnStateCancelledSchema,
@@ -19,7 +20,7 @@ export {
TurnStateRunningSchema,
TurnStateSchema,
} from './schemas/turn';
-export type { TerminalTurnState, Turn, TurnInputItem, TurnState } from './schemas/turn';
+export type { TerminalTurnState, Turn, TurnInputItem, TurnMetrics, TurnState } from './schemas/turn';
export { CreateSessionRequestSchema, SessionSchema, UpdateSessionRequestSchema } from './schemas/session';
export type { Session } from './schemas/session';
diff --git a/packages/harness/src/agent-session/schemas/turn.ts b/packages/harness/src/agent-session/schemas/turn.ts
index 8a275c823..c9b1cf608 100644
--- a/packages/harness/src/agent-session/schemas/turn.ts
+++ b/packages/harness/src/agent-session/schemas/turn.ts
@@ -30,11 +30,25 @@ export const TurnStateCancelledReasonSchema = z
.describe('Reason for the cancellation.')
.openapi('TurnStateCancelledReason');
+/** Billable aggregate for one turn. */
+export const TurnMetricsSchema = z
+ .object({
+ total_input_tokens: z.number().int().nonnegative().optional(),
+ total_output_tokens: z.number().int().nonnegative().optional(),
+ total_tokens: z.number().int().nonnegative().optional(),
+ total_cache_read_tokens: z.number().int().nonnegative().optional(),
+ total_cache_write_tokens: z.number().int().nonnegative().optional(),
+ total_reasoning_tokens: z.number().int().nonnegative().optional(),
+ total_cost_in_usd: z.number().nonnegative().optional(),
+ })
+ .openapi('TurnMetrics');
+
export const TurnStateCancelledSchema = z
.object({
status: z.literal('cancelled'),
reason: TurnStateCancelledReasonSchema,
completed_at: z.string(),
+ metrics: TurnMetricsSchema.optional(),
})
.openapi('TurnStateCancelled');
@@ -43,6 +57,7 @@ export const TurnStateErrorSchema = z
status: z.literal('error'),
message: z.string(),
completed_at: z.string(),
+ metrics: TurnMetricsSchema.optional(),
})
.openapi('TurnStateError');
@@ -53,6 +68,7 @@ export const TurnStateDoneSchema = z
output: ModelMessageEventSchema.optional(),
required_actions: z.array(ActionRequiredEventSchema),
completed_at: z.string(),
+ metrics: TurnMetricsSchema.optional(),
})
.openapi('TurnStateDone');
@@ -115,3 +131,4 @@ export type Turn = z.infer<typeof TurnSchema>;
export type TurnInputItem = z.infer<typeof TurnInputItemSchema>;
export type TurnState = z.infer<typeof TurnStateSchema>;
export type TerminalTurnState = Exclude<TurnState, { status: 'running' }>;
+export type TurnMetrics = z.infer<typeof TurnMetricsSchema>;
diff --git a/packages/harness/src/agent-session/store/ISessionStore.ts b/packages/harness/src/agent-session/store/ISessionStore.ts
index 93fb383cd..5ba533060 100644
--- a/packages/harness/src/agent-session/store/ISessionStore.ts
+++ b/packages/harness/src/agent-session/store/ISessionStore.ts
@@ -1,11 +1,11 @@
import type { JsonValue } from '../../core/capabilities/AgentCapability';
import type { MCPServerInitInfo, ThreadOverwriteContextEvent } from '../../core/events/schema';
-import type { CompletionUsage } from '../../core/llm/LLMTypes';
import type {
AgentThreadSnapshot,
ContextMessage,
SubAgentCompletionMarker,
} from '../../core/runtime/AgentThread.types';
+import type { CurrentContextUsage } from '../../core/runtime/contextUsage';
import type { SandboxInfo } from '../../core/sandbox/Sandbox';
import type { SessionRecord } from '../models/SessionRecord';
import type { TurnRecord } from '../models/TurnRecord';
@@ -63,7 +63,7 @@ export type ThreadCapabilityStateInit = Pick<AgentThreadSnapshot, 'thread_id' |
export interface TurnContextAppend {
thread_id: string;
context: ContextMessage[];
- current_context_usage: CompletionUsage | null;
+ current_context_usage: CurrentContextUsage | null;
}
export interface CreateTurnInput<TTurnCustom extends object = Record<string, never>> {
@@ -145,7 +145,7 @@ export interface AppendToThreadContextInput {
turn_id: string;
thread_id: string;
context: ContextMessage[];
- current_context_usage: CompletionUsage | null;
+ current_context_usage: CurrentContextUsage | null;
completion: SubAgentCompletionMarker | null;
}
diff --git a/packages/harness/src/agent-session/store/InMemorySessionStore.ts b/packages/harness/src/agent-session/store/InMemorySessionStore.ts
index 10a5d440f..b3b808fb1 100644
--- a/packages/harness/src/agent-session/store/InMemorySessionStore.ts
+++ b/packages/harness/src/agent-session/store/InMemorySessionStore.ts
@@ -1,5 +1,5 @@
-import { getEmptyUsage } from '../../core/llm/LLMTypes';
import type { AgentThreadSnapshot } from '../../core/runtime/AgentThread.types';
+import { getEmptyCurrentContextUsage } from '../../core/runtime/contextUsage';
import type { SessionRecord } from '../models/SessionRecord';
import type { TurnRecord, TurnSnapshot } from '../models/TurnRecord';
import type { PersistedTurnEvent, SessionEventItem } from '../schemas/events';
@@ -74,7 +74,7 @@ function newThreadSnapshot(thread: NewThreadInit): AgentThreadSnapshot {
agent_info: thread.agent_info ?? null,
capability_state: null,
context: [],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
completion: null,
};
}
diff --git a/packages/harness/src/core/capabilities/AgentContextProcessor.ts b/packages/harness/src/core/capabilities/AgentContextProcessor.ts
index bef387230..24b37bf01 100644
--- a/packages/harness/src/core/capabilities/AgentContextProcessor.ts
+++ b/packages/harness/src/core/capabilities/AgentContextProcessor.ts
@@ -1,19 +1,19 @@
import type { ChatCompletionMessageParam } from 'openai/resources/chat';
import type { ThreadOverwriteContextEvent } from '../events/schema';
-import type { CompletionUsage } from '../llm/LLMTypes';
import type {
AgentThreadAppendContext,
ContextMessage,
InternalCapabilityStateEvent,
InternalPassthroughEvent,
} from '../runtime/AgentThread.types';
+import type { CurrentContextUsage } from '../runtime/contextUsage';
import type { Sandbox } from '../sandbox/Sandbox';
/** Ambient thread snapshot passed Readonly to every capability processor hook. */
export interface AgentThreadExecutionContext {
threadId: string;
sandbox?: Sandbox | undefined;
- currentContextUsage: CompletionUsage;
+ currentContextUsage: CurrentContextUsage;
context: ContextMessage[];
}
diff --git a/packages/harness/src/core/capabilities/builtins/ContextCompaction.ts b/packages/harness/src/core/capabilities/builtins/ContextCompaction.ts
index c69f08829..9ba342596 100644
--- a/packages/harness/src/core/capabilities/builtins/ContextCompaction.ts
+++ b/packages/harness/src/core/capabilities/builtins/ContextCompaction.ts
@@ -183,7 +183,8 @@ export class ContextCompaction implements PreLLMAgentContextProcessor {
async *processPreLLM(
execution: Readonly<AgentThreadExecutionContext>,
): AsyncGenerator<AgentContextProcessorOutput, void, unknown> {
- if (execution.currentContextUsage.total_tokens + PROMPT_TOKENS < this.compactionThresholdTokens) {
+ const contextTokens = execution.currentContextUsage.prompt_tokens + execution.currentContextUsage.completion_tokens;
+ if (contextTokens + PROMPT_TOKENS < this.compactionThresholdTokens) {
return;
}
@@ -206,15 +207,14 @@ export class ContextCompaction implements PreLLMAgentContextProcessor {
reason: 'compaction',
context: context,
current_context_usage: {
- total_tokens: response.usage.total_tokens + CONTINUATION_MESSAGE_TOKENS,
// NOTE(agent): This is not really correct.
// This is not taking into account that the original request had
// tool definition in them.
// This will get refreshed in the next LLM call.
- prompt_tokens: response.usage.prompt_tokens + CONTINUATION_MESSAGE_TOKENS,
+ prompt_tokens: response.usage.input_tokens + CONTINUATION_MESSAGE_TOKENS,
completion_tokens: 0,
},
- compaction_llm_usage: response.usage,
+ usage: response.usage,
};
}
}
diff --git a/packages/harness/src/core/events/schema.ts b/packages/harness/src/core/events/schema.ts
index 6a836fc85..94fb17223 100644
--- a/packages/harness/src/core/events/schema.ts
+++ b/packages/harness/src/core/events/schema.ts
@@ -10,6 +10,7 @@ import {
LLMToolMessageSchema,
LLMUserMessageSchema,
} from '../llm/LLMTypes';
+import { CurrentContextUsageSchema } from '../runtime/contextUsage';
/**
* Process-local monotonic ULIDs preserve event creation order, including
@@ -244,10 +245,8 @@ export const ThreadOverwriteContextEventSchema = z.object({
// NOTE: add other reasons here.
reason: z.literal('compaction'),
context: z.array(ContextMessageSchema),
- // Estimated token usage of the new (compacted) context for the next LLM call's budget
- current_context_usage: CompletionUsageSchema,
- // Actual token usage consumed by the compaction LLM call itself
- compaction_llm_usage: CompletionUsageSchema,
+ current_context_usage: CurrentContextUsageSchema,
+ usage: CompletionUsageSchema,
});
export const MCPServerAuthInfoSchema = z
diff --git a/packages/harness/src/core/index.ts b/packages/harness/src/core/index.ts
index f3631b248..3e59525a1 100644
--- a/packages/harness/src/core/index.ts
+++ b/packages/harness/src/core/index.ts
@@ -115,8 +115,11 @@ export type {
ThreadDoneEvent,
ThreadOverwriteContextEvent,
} from './events/schema';
+export { CompletionUsageSchema } from './llm/LLMTypes';
+export type { CompletionUsage } from './llm/LLMTypes';
export { InternalEventType } from './runtime/AgentThread.types';
export type { AgentThreadSendBatch, ContextMessage } from './runtime/AgentThread.types';
+export { AgentThreadMetricsSchema } from './runtime/metrics';
export type { AgentThreadMetrics } from './runtime/metrics';
// Tracing
diff --git a/packages/harness/src/core/llm/LLMTypes.ts b/packages/harness/src/core/llm/LLMTypes.ts
index c534b1c35..038953fe6 100644
--- a/packages/harness/src/core/llm/LLMTypes.ts
+++ b/packages/harness/src/core/llm/LLMTypes.ts
@@ -136,21 +136,13 @@ export const LLMToolMessageSchema = ChatCompletionToolMessageParamSchema.omit({
export const CompletionUsageSchema = z
.object({
- completion_tokens: z.number().int().nonnegative(),
- prompt_tokens: z.number().int().nonnegative(),
+ input_tokens: z.number().int().nonnegative(),
+ output_tokens: z.number().int().nonnegative(),
total_tokens: z.number().int().nonnegative(),
- completion_tokens_details: z
- .object({
- reasoning_tokens: z.number().int().nonnegative().optional(),
- })
- .optional(),
- prompt_tokens_details: z
- .object({
- cached_tokens: z.number().int().nonnegative().optional(),
- })
- .optional(),
- cache_read_input_tokens: z.number().int().nonnegative().optional(),
- cache_creation_input_tokens: z.number().int().nonnegative().optional(),
+ cache_read_tokens: z.number().int().nonnegative().optional(),
+ cache_write_tokens: z.number().int().nonnegative().optional(),
+ reasoning_tokens: z.number().int().nonnegative().optional(),
+ cost_in_usd: z.number().nonnegative().optional(),
})
.openapi('CompletionUsage');
@@ -175,7 +167,7 @@ export type CompletionUsage = z.infer<typeof CompletionUsageSchema>;
export type FinishReason = z.infer<typeof FinishReasonSchema>;
export function getEmptyUsage(): CompletionUsage {
- return { prompt_tokens: 0, total_tokens: 0, completion_tokens: 0 };
+ return { input_tokens: 0, output_tokens: 0, total_tokens: 0 };
}
/**
@@ -190,12 +182,14 @@ interface ExtendedChoice extends Omit<ChatCompletionChunk.Choice, 'delta'> {
/**
* Extended chat completion chunk that includes thought_signature in tool calls.
+ * `usage` is harness-normalized; OpenAILLM maps the gateway wire `usage` payload at the adapter boundary.
*/
-export interface ExtendedChatCompletionChunk extends Omit<ChatCompletionChunk, 'choices'> {
+export interface ExtendedChatCompletionChunk extends Omit<ChatCompletionChunk, 'choices' | 'usage'> {
/**
* A list of chat completion choices, with thought_signature support in deltas.
*/
choices: ExtendedChoice[];
+ usage?: CompletionUsage | null;
}
export interface RawAssistantMessageWithUsage {
diff --git a/packages/harness/src/core/llm/OpenAILLM.ts b/packages/harness/src/core/llm/OpenAILLM.ts
index 23ffd4258..e9f8b1263 100644
--- a/packages/harness/src/core/llm/OpenAILLM.ts
+++ b/packages/harness/src/core/llm/OpenAILLM.ts
@@ -1,17 +1,75 @@
+import { z } from '@hono/zod-openapi';
import OpenAI from 'openai';
import type { RequestOptions } from 'openai/core';
-import type { ChatCompletionCreateParams, ChatCompletionCreateParamsStreaming } from 'openai/resources/chat';
+import type {
+ ChatCompletionChunk,
+ ChatCompletionCreateParams,
+ ChatCompletionCreateParamsStreaming,
+} from 'openai/resources/chat';
import type { Logger } from 'winston';
import { extractErrorLogFields } from '../util/errorLogFields';
import type { ILLM } from './ILLM';
import {
- getEmptyUsage,
type CompletionUsage,
type ExtendedChatCompletionChunk,
+ getEmptyUsage,
type RawAssistantMessage,
type RawAssistantMessageWithUsage,
} from './LLMTypes';
+/** Gateway wire usage. Private; normalized once into CompletionUsage. */
+const OptionalGatewayTokenCountSchema = z.number().int().nonnegative().optional().catch(undefined);
+const OptionalGatewayCostSchema = z.number().nonnegative().optional().catch(undefined);
+
+const GatewayChatCompletionUsageSchema = z.object({
+ completion_tokens: OptionalGatewayTokenCountSchema,
+ prompt_tokens: OptionalGatewayTokenCountSchema,
+ total_tokens: OptionalGatewayTokenCountSchema,
+ completion_tokens_details: z
+ .object({
+ reasoning_tokens: OptionalGatewayTokenCountSchema,
+ })
+ .optional()
+ .catch(undefined),
+ prompt_tokens_details: z
+ .object({
+ cached_tokens: OptionalGatewayTokenCountSchema,
+ })
+ .optional()
+ .catch(undefined),
+ cache_read_input_tokens: OptionalGatewayTokenCountSchema,
+ cache_creation_input_tokens: OptionalGatewayTokenCountSchema,
+ costInUSD: OptionalGatewayCostSchema,
+});
+
+type GatewayChatCompletionUsage = z.infer<typeof GatewayChatCompletionUsageSchema>;
+
+/** One-shot translation: gateway wire → canonical harness CompletionUsage. */
+function normalizeGatewayUsage(usage: unknown): CompletionUsage {
+ const result = GatewayChatCompletionUsageSchema.safeParse(usage);
+ const parsed: GatewayChatCompletionUsage = result.success ? result.data : {};
+ return {
+ input_tokens: parsed.prompt_tokens ?? 0,
+ output_tokens: parsed.completion_tokens ?? 0,
+ total_tokens: parsed.total_tokens ?? 0,
+ cache_read_tokens: parsed.cache_read_input_tokens ?? parsed.prompt_tokens_details?.cached_tokens,
+ cache_write_tokens: parsed.cache_creation_input_tokens,
+ reasoning_tokens: parsed.completion_tokens_details?.reasoning_tokens,
+ cost_in_usd: parsed.costInUSD,
+ };
+}
+
+function toHarnessChunk(event: ChatCompletionChunk): ExtendedChatCompletionChunk {
+ const { usage, ...rest } = event;
+ if (usage == null) {
+ return rest;
+ }
+ return {
+ ...rest,
+ usage: normalizeGatewayUsage(usage),
+ };
+}
+
/** Static headers, or a resolver invoked on every request. */
export type LLMHeaders = Record<string, string> | (() => Record<string, string>);
@@ -87,8 +145,9 @@ export class OpenAILLM implements ILLM {
}
for await (const event of response) {
- yield event;
- accumulateTokensFromChunk(event, newMessage, this.logger);
+ const chunk = toHarnessChunk(event);
+ yield chunk;
+ accumulateTokensFromChunk(chunk, newMessage, this.logger);
}
// Record after the stream drains so the last completed LLM call wins.
const servedModelContextLength = Number(servedModelContextLengthHeader);
diff --git a/packages/harness/src/core/llm/usage.ts b/packages/harness/src/core/llm/usage.ts
index 4258a5739..ae20e4d84 100644
--- a/packages/harness/src/core/llm/usage.ts
+++ b/packages/harness/src/core/llm/usage.ts
@@ -4,19 +4,21 @@ export function estimateTokensForString(s: string): number {
return s.length / 4;
}
+function sumOptional(a: number | undefined, b: number | undefined): number | undefined {
+ if (a === undefined && b === undefined) {
+ return undefined;
+ }
+ return (a ?? 0) + (b ?? 0);
+}
+
export function mergeUsage(a: CompletionUsage, b: CompletionUsage): CompletionUsage {
return {
- prompt_tokens: a.prompt_tokens + b.prompt_tokens,
- completion_tokens: a.completion_tokens + b.completion_tokens,
+ input_tokens: a.input_tokens + b.input_tokens,
+ output_tokens: a.output_tokens + b.output_tokens,
total_tokens: a.total_tokens + b.total_tokens,
- cache_read_input_tokens: (a.cache_read_input_tokens ?? 0) + (b.cache_read_input_tokens ?? 0),
- cache_creation_input_tokens: (a.cache_creation_input_tokens ?? 0) + (b.cache_creation_input_tokens ?? 0),
- prompt_tokens_details: {
- cached_tokens: (a.prompt_tokens_details?.cached_tokens ?? 0) + (b.prompt_tokens_details?.cached_tokens ?? 0),
- },
- completion_tokens_details: {
- reasoning_tokens:
- (a.completion_tokens_details?.reasoning_tokens ?? 0) + (b.completion_tokens_details?.reasoning_tokens ?? 0),
- },
+ cache_read_tokens: sumOptional(a.cache_read_tokens, b.cache_read_tokens),
+ cache_write_tokens: sumOptional(a.cache_write_tokens, b.cache_write_tokens),
+ reasoning_tokens: sumOptional(a.reasoning_tokens, b.reasoning_tokens),
+ cost_in_usd: sumOptional(a.cost_in_usd, b.cost_in_usd),
};
}
diff --git a/packages/harness/src/core/runtime/AgentThread.ts b/packages/harness/src/core/runtime/AgentThread.ts
index cf77f4e5f..0dd300816 100644
--- a/packages/harness/src/core/runtime/AgentThread.ts
+++ b/packages/harness/src/core/runtime/AgentThread.ts
@@ -40,7 +40,6 @@ import {
} from '../events/schema';
import { InstructionBuilder, ROOT_AGENT_IDENTITY } from '../InstructionBuilder';
import {
- getEmptyUsage,
type CompletionUsage,
type ExtendedChatCompletionChunk,
type FinishReason,
@@ -52,7 +51,7 @@ import {
} from '../llm/LLMTypes';
import { toOpenAIResponseFormat } from '../llm/responseFormat';
import { toOpenAIChatMessage } from '../llm/toOpenAIChatMessage';
-import { estimateTokensForString, mergeUsage } from '../llm/usage';
+import { estimateTokensForString } from '../llm/usage';
import { convertMCPServersToTools, type ConvertToolsResult, type MappedMCPTool } from '../mcp/convertMCPServers';
import { executeToolCalls } from '../mcp/executeToolCalls';
import type { IToolSet, MCPAuthRequired } from '../mcp/IMCPServer';
@@ -75,6 +74,12 @@ import {
type InternalThreadDoneEvent,
type SubAgentCompletionMarker,
} from './AgentThread.types';
+import {
+ currentContextUsageFromCompletion,
+ getEmptyCurrentContextUsage,
+ mergeCurrentContextUsage,
+ type CurrentContextUsage,
+} from './contextUsage';
import {
assistantMessageContentToStringForSubAgent,
estimateTokensForContextMessages,
@@ -90,7 +95,7 @@ import {
toToolCallInfo,
} from './contextUtils';
import { DeferredTool } from './DeferredTool';
-import type { AgentThreadMetrics } from './metrics';
+import { createEmptyAgentThreadMetrics, updateMetricsFromUsage, type AgentThreadMetrics } from './metrics';
import { getClosableOpenToolCallIds, OpenToolCallCloser } from './OpenToolCallCloser';
import { isEmptyMessageContent, processAgentUserInput, type AgentInputUserMessage } from './UserInputMessage';
@@ -465,7 +470,7 @@ export class AgentThread {
readonly agentInfo?: AgentInfo | undefined;
private context: ContextMessage[];
- private currentContextUsage: CompletionUsage;
+ private currentContextUsage: CurrentContextUsage;
private preSendContextProcessors: PreSendContextProcessor[];
private preLLMContextProcessors: PreLLMAgentContextProcessor[];
private preEphemeralLLMContextProcessors: PreLLMEphemeralAgentContextProcessor[];
@@ -482,11 +487,7 @@ export class AgentThread {
private readonly tracing: AgentTracing;
private readonly logger: Logger;
- private _iterations = 0;
- private _totalToolCalls = 0;
- private _totalSummarizations = 0;
- private _totalSubAgents = 0;
- private _cumulativeUsage: CompletionUsage = getEmptyUsage();
+ private metrics: AgentThreadMetrics = createEmptyAgentThreadMetrics();
private tfyManagedServerNames = new Set<string>();
private contextBusy = false;
@@ -505,7 +506,7 @@ export class AgentThread {
this.context = input.context ? [...input.context] : [];
this.title = input.title;
this.parent = input.parent;
- this.currentContextUsage = input.currentContextUsage ?? getEmptyUsage();
+ this.currentContextUsage = input.currentContextUsage ?? getEmptyCurrentContextUsage();
this.agentInfo = input.agentInfo;
this.preComputedCompletion = input.preComputedCompletion;
this.sandbox = input.sandbox;
@@ -624,12 +625,16 @@ export class AgentThread {
yield* this.appendToContext({
context: [...approvalContext, ...toolResponseContext],
output: [],
+ currentContextUsage: undefined,
+ usage: undefined,
});
}
if (contextMessages.length > 0) {
yield* this.appendToContext({
context: contextMessages,
output: [],
+ currentContextUsage: undefined,
+ usage: undefined,
});
}
} finally {
@@ -655,14 +660,15 @@ export class AgentThread {
private *appendToContext(opts: {
context: ContextMessage[];
output: AgentOutputEvent[];
- overwriteUsage?: CompletionUsage | undefined;
- updateCumulativeUsage?: boolean | undefined;
+ currentContextUsage: CurrentContextUsage | undefined;
+ usage: CompletionUsage | undefined;
completion?: SubAgentCompletionMarker | undefined;
}): Generator<AgentThreadAppendContext, void, unknown> {
- const { context, output, overwriteUsage, updateCumulativeUsage, completion } = opts;
+ const { context, output, currentContextUsage, usage, completion } = opts;
const newCurrentContextUsage =
- overwriteUsage ?? mergeUsage(this.currentContextUsage, estimateTokensForContextMessages(context));
+ currentContextUsage ??
+ mergeCurrentContextUsage(this.currentContextUsage, estimateTokensForContextMessages(context));
const event: AgentThreadAppendContext = {
type: InternalEventType.AGENT_CONTEXT_APPEND,
@@ -673,13 +679,15 @@ export class AgentThread {
...(completion && { completion }),
};
+ // Update metrics before yield so we still count it if the stream stops here.
+ if (usage) {
+ updateMetricsFromUsage(this.metrics, usage);
+ }
+
yield event;
this.context = this.context.concat(context);
this.currentContextUsage = newCurrentContextUsage;
- if (updateCumulativeUsage && overwriteUsage) {
- this._cumulativeUsage = mergeUsage(this._cumulativeUsage, overwriteUsage);
- }
}
private *overwriteContext(
@@ -689,11 +697,13 @@ export class AgentThread {
thread_id: this.threadId,
...payload,
};
+ // Update metrics before yield so we still count it if the stream stops here.
+ updateMetricsFromUsage(this.metrics, payload.usage);
+
yield event;
this.context = payload.context;
- this._cumulativeUsage = mergeUsage(this._cumulativeUsage, payload.compaction_llm_usage);
this.currentContextUsage = payload.current_context_usage;
- this._totalSummarizations++;
+ this.metrics.total_summarizations++;
}
private generateErrorEvent(message: string, output?: ModelMessageEvent): InternalThreadDoneEvent {
@@ -747,13 +757,7 @@ export class AgentThread {
}
public getAgentThreadMetrics(): AgentThreadMetrics {
- return {
- iterations: this._iterations,
- total_tool_calls: this._totalToolCalls,
- total_summarizations: this._totalSummarizations,
- usage: { ...this._cumulativeUsage },
- total_sub_agents: this._totalSubAgents,
- };
+ return { ...this.metrics };
}
private transformToLLMRequest(tools: ChatCompletionTool[]): ChatCompletionCreateParamsStreaming {
@@ -837,13 +841,13 @@ export class AgentThread {
const harnessEstimate = harnessInstructionTokens + tfyManagedToolTokens;
const estimatedComponentsTotal = harnessEstimate + skillsEstimate + instructionsEstimate + userToolTokens;
- const messagesEstimate = Math.max(0, usage.prompt_tokens - estimatedComponentsTotal);
+ const messagesEstimate = Math.max(0, usage.input_tokens - estimatedComponentsTotal);
return {
- input_tokens: usage.prompt_tokens,
- output_tokens: usage.completion_tokens,
- cache_read_tokens: usage.cache_read_input_tokens ?? usage.prompt_tokens_details?.cached_tokens,
- cache_write_tokens: usage.cache_creation_input_tokens,
+ input_tokens: usage.input_tokens,
+ output_tokens: usage.output_tokens,
+ cache_read_tokens: usage.cache_read_tokens,
+ cache_write_tokens: usage.cache_write_tokens,
input_tokens_breakdown: {
harness: harnessEstimate,
skills: skillsEstimate,
@@ -906,7 +910,8 @@ export class AgentThread {
yield* this.appendToContext({
context: response.context,
output: response.output,
- overwriteUsage: response.current_context_usage ?? this.currentContextUsage,
+ currentContextUsage: response.current_context_usage ?? this.currentContextUsage,
+ usage: undefined,
});
break;
case EventType.AGENT_CONTEXT_OVERWRITE:
@@ -1005,17 +1010,21 @@ export class AgentThread {
let firstDeltaTimestamped = false;
let result = await llmStream.next();
- let usage: ModelMessageUsage | undefined = undefined;
+ let modelMessageUsage: ModelMessageUsage | undefined = undefined;
while (!result.done) {
const chunk = result.value;
if (chunk.usage) {
- usage = this.computeAssistantMessageUsage({ requestBody, usage: chunk.usage, toolMapping });
+ modelMessageUsage = this.computeAssistantMessageUsage({
+ requestBody,
+ usage: chunk.usage,
+ toolMapping,
+ });
}
const event = await buildModelMessageDeltaEvent({
chunk,
threadId: this.threadId,
toolMapping,
- usage,
+ usage: modelMessageUsage,
id: modelMessageEventId,
logger: this.logger,
});
@@ -1037,8 +1046,12 @@ export class AgentThread {
return { outcome: 'exit', modelMessageEventId };
}
- // should be computed by this point but in case provider never gives out any delta with usage key, compute it now.
- usage ??= this.computeAssistantMessageUsage({ requestBody, usage: result.value.usage, toolMapping });
+ // should be computed by this point but in case provider never gives out any delta with usage, compute it now.
+ modelMessageUsage ??= this.computeAssistantMessageUsage({
+ requestBody,
+ usage: result.value.usage,
+ toolMapping,
+ });
const assistantMessage: InternalEnrichedAssistantMessage = await buildContextAssistantMessage(
result.value.output,
toolMapping,
@@ -1048,7 +1061,7 @@ export class AgentThread {
assistantMessage,
threadId: this.threadId,
finishReason,
- usage,
+ usage: modelMessageUsage,
id: modelMessageEventId,
});
@@ -1078,8 +1091,8 @@ export class AgentThread {
yield* this.appendToContext({
context: [assistantMessage],
output: [agentAssistantMessage],
- overwriteUsage: result.value.usage,
- updateCumulativeUsage: true,
+ currentContextUsage: currentContextUsageFromCompletion(result.value.usage),
+ usage: result.value.usage,
completion,
});
@@ -1140,8 +1153,8 @@ export class AgentThread {
if (approvalRequiredToolCalls.length > 0) {
throw new Error('Unreachable');
}
- this._totalToolCalls += toolCallResults.length;
- this._totalSubAgents += createThreadEvents.length;
+ this.metrics.total_tool_calls += toolCallResults.length;
+ this.metrics.total_sub_agents += createThreadEvents.length;
if (initializationInfo.length > 0) {
yield buildMCPInitializeEvent(initializationInfo, this.threadId);
@@ -1198,6 +1211,8 @@ export class AgentThread {
yield* this.appendToContext({
context: toolCallResults.map(t => t.message),
output: [],
+ currentContextUsage: undefined,
+ usage: undefined,
});
if (authRequirementInfo.length > 0) {
@@ -1314,13 +1329,13 @@ export class AgentThread {
case 'llm-call-required': {
// Cancel only before network/side-effecting steps (LLM call, tool execution), never before 'user-input-required', so the required event is always emitted once the assistant message is committed.
if (signal?.aborted) return;
- if (this._iterations >= iterationLimit) {
+ if (this.metrics.iterations >= iterationLimit) {
yield this.generateErrorEvent(
`You have reached iteration limit of ${String(iterationLimit)}, please request again`,
);
return;
}
- this._iterations++;
+ this.metrics.iterations++;
const llmResult = yield* this.stepLLMCall(tools, toolMapping, signal);
outcome = llmResult.outcome;
diff --git a/packages/harness/src/core/runtime/AgentThread.types.ts b/packages/harness/src/core/runtime/AgentThread.types.ts
index 1aaa84256..5a5cf79f1 100644
--- a/packages/harness/src/core/runtime/AgentThread.types.ts
+++ b/packages/harness/src/core/runtime/AgentThread.types.ts
@@ -25,16 +25,11 @@ import type {
UserToolApprovalMessage,
UserToolResponseMessage,
} from '../events/schema';
-import type {
- CompletionUsage,
- InternalEnrichedAssistantMessage,
- LLMToolMessage,
- LLMUserMessage,
-} from '../llm/LLMTypes';
+import type { InternalEnrichedAssistantMessage, LLMToolMessage, LLMUserMessage } from '../llm/LLMTypes';
import type { Sandbox } from '../sandbox/Sandbox';
import type { AgentTracing } from '../tracing/AgentTracing';
import type { AgentDefinition } from './AgentDefinition';
-import type { AgentThreadMetrics } from './metrics';
+import type { CurrentContextUsage } from './contextUsage';
export type { AgentInfo, AgentParent };
@@ -107,7 +102,7 @@ export interface AgentThreadAppendContext {
thread_id: string;
context: ContextMessage[];
output: AgentOutputEvent[];
- current_context_usage?: CompletionUsage | undefined;
+ current_context_usage?: CurrentContextUsage | undefined;
completion?: SubAgentCompletionMarker | undefined;
}
@@ -144,7 +139,6 @@ export interface AgentThreadExecutionResult {
output: ModelMessageEvent | null;
required_actions: ActionRequiredEvent[];
root_agent_error?: Pick<ThreadStateError, 'error' | 'output'> | undefined;
- metrics: { total: AgentThreadMetrics };
}
/** Public send items plus internal LLM tool messages (child→parent delivery). */
@@ -156,7 +150,7 @@ export type AgentThreadRuntimeSendBatch = AgentThreadSendBatch | LLMToolMessage[
export interface AgentThreadSnapshot {
thread_id: string;
context: ContextMessage[];
- current_context_usage: CompletionUsage;
+ current_context_usage: CurrentContextUsage;
parent: AgentParent | null;
agent_info: AgentInfo | null;
completion: SubAgentCompletionMarker | null;
@@ -171,7 +165,7 @@ export interface AgentThreadConstructorInput {
parent?: AgentParent | undefined;
agentInfo?: AgentInfo | undefined;
context?: ContextMessage[] | undefined;
- currentContextUsage?: CompletionUsage | undefined;
+ currentContextUsage?: CurrentContextUsage | undefined;
preComputedCompletion?: SubAgentCompletionMarker | undefined;
sandbox?: Sandbox | undefined;
capabilities?: readonly AgentCapability[] | undefined;
diff --git a/packages/harness/src/core/runtime/AgentThreadOrchestrator.ts b/packages/harness/src/core/runtime/AgentThreadOrchestrator.ts
index 4df2ca105..a277f1bf0 100644
--- a/packages/harness/src/core/runtime/AgentThreadOrchestrator.ts
+++ b/packages/harness/src/core/runtime/AgentThreadOrchestrator.ts
@@ -32,7 +32,7 @@ import {
isInternalThreadDoneError,
} from './contextUtils';
import type { CreateDynamicSubAgentThread } from './CreateDynamicSubAgentThread';
-import { addAgentThreadMetrics, createEmptyAgentThreadMetrics } from './metrics';
+import { addAgentThreadMetrics, createEmptyAgentThreadMetrics, type AgentThreadMetrics } from './metrics';
const MAX_PARALLEL_SUB_AGENTS = 5;
@@ -229,6 +229,8 @@ export class AgentThreadOrchestrator {
private readonly createDynamicSubAgentThread: CreateDynamicSubAgentThread;
private readonly tracing: AgentTracing;
private readonly logger: Logger;
+ // Finished sub-agents removed from `agentThreads`; kept so totals still include them.
+ private finishedSubAgentMetrics: AgentThreadMetrics = createEmptyAgentThreadMetrics();
constructor(params: AgentThreadOrchestratorInput) {
this.agentThreads = params.agentThreads;
@@ -237,6 +239,20 @@ export class AgentThreadOrchestrator {
this.logger = params.logger.child({ module: 'AgentThreadOrchestrator' });
}
+ /**
+ * Turn-wide metrics so far: live threads plus finished sub-agents.
+ * Safe mid-flight and after cancel/error (`execute()` may not return).
+ * Each thread is counted once — a sub-agent moves from live → finished atomically.
+ */
+ public getMetrics(): AgentThreadMetrics {
+ const total = createEmptyAgentThreadMetrics();
+ addAgentThreadMetrics(total, this.finishedSubAgentMetrics);
+ for (const thread of this.agentThreads.values()) {
+ addAgentThreadMetrics(total, thread.getAgentThreadMetrics());
+ }
+ return total;
+ }
+
public async *send(messages: AgentThreadSendBatch): AsyncGenerator<AgentThreadAppendContext, void, unknown> {
const byThread = new Map<string, AgentThreadRuntimeSendBatch>();
for (const thread of this.agentThreads.values()) {
@@ -345,6 +361,9 @@ export class AgentThreadOrchestrator {
yield* this.sendToThread(chunk.parent.thread_id, [chunk.send_to_parent]);
}
yield chunk;
+ // Move metrics to the finished bucket and drop the live entry with no `yield` between,
+ // so getMetrics() counts this sub-agent once. After `yield chunk` per durable-state.
+ addAgentThreadMetrics(this.finishedSubAgentMetrics, currentThread.getAgentThreadMetrics());
this.agentThreads.delete(chunk.thread_id);
return { shouldStopExecution: false };
}
@@ -394,13 +413,13 @@ export class AgentThreadOrchestrator {
signal: AbortSignal;
}): AsyncGenerator<AgentThreadExecutionEvent, AgentThreadExecutionResult, unknown> {
const { agentThreads } = this;
+
let shouldStopExecution = false;
let caughtError: unknown;
let output: ModelMessageEvent | null = null;
const requiredActions: ActionRequiredEvent[] = [];
let rootAgentError: AgentThreadExecutionResult['root_agent_error'];
const pendingAuthEvents: InternalMCPAuthRequiredEvent[] = [];
- const totalMetrics = createEmptyAgentThreadMetrics();
const mainThread = [...agentThreads.values()].find(e => !e.parent);
if (!mainThread) {
throw new Error('Unreachable: no root thread found');
@@ -432,17 +451,10 @@ export class AgentThreadOrchestrator {
shouldStopExecution = true;
continue;
}
- if (chunk.type === InternalEventType.AGENT_DONE) {
- if (chunk.parent) {
- const completedThread = agentThreads.get(chunk.thread_id);
- if (completedThread) {
- addAgentThreadMetrics(totalMetrics, completedThread.getAgentThreadMetrics());
- }
- } else {
- // Root-thread only — sub-agent AGENT_DONE would overwrite root trace output /
- // leave a stale error; sub-agent spans finalize in wrapWithSubAgentSpan.
- rootSpan.setOutputFromEvent(chunk);
- }
+ if (chunk.type === InternalEventType.AGENT_DONE && !chunk.parent) {
+ // Root-thread only — sub-agent AGENT_DONE would overwrite root trace output /
+ // leave a stale error; sub-agent spans finalize in wrapWithSubAgentSpan.
+ rootSpan.setOutputFromEvent(chunk);
}
const result = yield* this.processAgentStreamChunk(chunk, signal);
@@ -469,7 +481,6 @@ export class AgentThreadOrchestrator {
caughtError = error;
throw error;
} finally {
- addAgentThreadMetrics(totalMetrics, mainThread.getAgentThreadMetrics());
rootSpan.finalize(mainThread, caughtError);
rootSpan.end();
}
@@ -494,7 +505,6 @@ export class AgentThreadOrchestrator {
output,
required_actions: requiredActions,
root_agent_error: rootAgentError,
- metrics: { total: totalMetrics },
};
}
}
diff --git a/packages/harness/src/core/runtime/OpenToolCallCloser.ts b/packages/harness/src/core/runtime/OpenToolCallCloser.ts
index c5d921a57..972717912 100644
--- a/packages/harness/src/core/runtime/OpenToolCallCloser.ts
+++ b/packages/harness/src/core/runtime/OpenToolCallCloser.ts
@@ -18,9 +18,9 @@ import type {
PreSendContextProcessor,
} from '../capabilities/AgentContextProcessor';
import type { InternalEnrichedAssistantMessage, LLMToolMessage } from '../llm/LLMTypes';
-import { mergeUsage } from '../llm/usage';
import type { ContextMessage } from './AgentThread.types';
import { InternalEventType } from './AgentThread.types';
+import { mergeCurrentContextUsage } from './contextUsage';
import { estimateTokensForContextMessages, isLLMContextMessage } from './contextUtils';
const DUMMY_TOOL_MESSAGE_CONTENT = JSON.stringify({
@@ -82,7 +82,7 @@ export class OpenToolCallCloser implements PreSendContextProcessor {
content: DUMMY_TOOL_MESSAGE_CONTENT,
}));
- const currentContextUsage = mergeUsage(
+ const currentContextUsage = mergeCurrentContextUsage(
execution.currentContextUsage,
estimateTokensForContextMessages(dummyToolMessages),
);
diff --git a/packages/harness/src/core/runtime/contextUsage.ts b/packages/harness/src/core/runtime/contextUsage.ts
new file mode 100644
index 000000000..fdd0ecaf7
--- /dev/null
+++ b/packages/harness/src/core/runtime/contextUsage.ts
@@ -0,0 +1,37 @@
+import { z } from '@hono/zod-openapi';
+import type { CompletionUsage } from '../llm/LLMTypes';
+
+/**
+ * Live context budget for the next LLM call (not billable). The legacy
+ * `prompt_tokens`/`completion_tokens` names are load-bearing: this shape is persisted per
+ * thread, so renaming them would make already-stored rows load as missing fields.
+ *
+ * We Need Zod mainly for ThreadOverwriteContextEventSchema
+ */
+export const CurrentContextUsageSchema = z
+ .object({
+ prompt_tokens: z.number().int().nonnegative(),
+ completion_tokens: z.number().int().nonnegative(),
+ })
+ .openapi('CurrentContextUsage');
+
+export type CurrentContextUsage = z.infer<typeof CurrentContextUsageSchema>;
+
+export function getEmptyCurrentContextUsage(): CurrentContextUsage {
+ return { prompt_tokens: 0, completion_tokens: 0 };
+}
+
+export function mergeCurrentContextUsage(a: CurrentContextUsage, b: CurrentContextUsage): CurrentContextUsage {
+ return {
+ prompt_tokens: a.prompt_tokens + b.prompt_tokens,
+ completion_tokens: a.completion_tokens + b.completion_tokens,
+ };
+}
+
+/** Project per-call billable usage onto the live context budget fields. */
+export function currentContextUsageFromCompletion(usage: CompletionUsage): CurrentContextUsage {
+ return {
+ prompt_tokens: usage.input_tokens,
+ completion_tokens: usage.output_tokens,
+ };
+}
diff --git a/packages/harness/src/core/runtime/contextUtils.ts b/packages/harness/src/core/runtime/contextUtils.ts
index 26167f2ad..bd3ee744f 100644
--- a/packages/harness/src/core/runtime/contextUtils.ts
+++ b/packages/harness/src/core/runtime/contextUtils.ts
@@ -7,7 +7,6 @@ import {
type UserToolResponseMessage,
} from '../events/schema';
import type {
- CompletionUsage,
EnrichedToolCall,
InternalEnrichedToolCall,
InternalToolCallInfo,
@@ -23,6 +22,7 @@ import type {
InternalThreadDoneEvent,
LLMContextMessage,
} from './AgentThread.types';
+import type { CurrentContextUsage } from './contextUsage';
import type { AgentInputUserMessage } from './UserInputMessage';
export const SYSTEM_TAG_START = '<tfy-internal>';
@@ -139,7 +139,7 @@ export function getThreadId(): string {
return crypto.randomUUID();
}
-export function estimateTokensForContextMessages(messages: ContextMessage[]): CompletionUsage {
+export function estimateTokensForContextMessages(messages: ContextMessage[]): CurrentContextUsage {
let tokenCount = 0;
for (const b of messages) {
@@ -170,6 +170,5 @@ export function estimateTokensForContextMessages(messages: ContextMessage[]): Co
return {
prompt_tokens: tokenCount,
completion_tokens: 0,
- total_tokens: tokenCount,
};
}
diff --git a/packages/harness/src/core/runtime/metrics.ts b/packages/harness/src/core/runtime/metrics.ts
index 08c509398..0c10a3ed8 100644
--- a/packages/harness/src/core/runtime/metrics.ts
+++ b/packages/harness/src/core/runtime/metrics.ts
@@ -1,29 +1,65 @@
+import { z } from '@hono/zod-openapi';
import type { CompletionUsage } from '../llm/LLMTypes';
-import { getEmptyUsage } from '../llm/LLMTypes';
-import { mergeUsage } from '../llm/usage';
-export interface AgentThreadMetrics {
- iterations: number;
- total_tool_calls: number;
- total_summarizations: number;
- usage: CompletionUsage;
- total_sub_agents: number;
+/** Billable aggregate — thread/orchestrator totals (`total_*`). */
+export const AgentThreadMetricsSchema = z
+ .object({
+ total_input_tokens: z.number().int().nonnegative(),
+ total_output_tokens: z.number().int().nonnegative(),
+ total_tokens: z.number().int().nonnegative(),
+ total_cache_read_tokens: z.number().int().nonnegative().optional(),
+ total_cache_write_tokens: z.number().int().nonnegative().optional(),
+ total_reasoning_tokens: z.number().int().nonnegative().optional(),
+ total_cost_in_usd: z.number().nonnegative().optional(),
+ iterations: z.number().int().nonnegative(),
+ total_tool_calls: z.number().int().nonnegative(),
+ total_summarizations: z.number().int().nonnegative(),
+ total_sub_agents: z.number().int().nonnegative(),
+ })
+ .openapi('AgentThreadMetrics');
+
+export type AgentThreadMetrics = z.infer<typeof AgentThreadMetricsSchema>;
+
+function sumOptional(a: number | undefined, b: number | undefined): number | undefined {
+ if (a === undefined && b === undefined) {
+ return undefined;
+ }
+ return (a ?? 0) + (b ?? 0);
}
export function createEmptyAgentThreadMetrics(): AgentThreadMetrics {
return {
+ total_input_tokens: 0,
+ total_output_tokens: 0,
+ total_tokens: 0,
iterations: 0,
total_tool_calls: 0,
total_summarizations: 0,
- usage: getEmptyUsage(),
total_sub_agents: 0,
};
}
+/** Fold one per-call completion usage into aggregated thread metrics. */
+export function updateMetricsFromUsage(target: AgentThreadMetrics, usage: CompletionUsage): void {
+ target.total_input_tokens += usage.input_tokens;
+ target.total_output_tokens += usage.output_tokens;
+ target.total_tokens += usage.total_tokens;
+ target.total_cache_read_tokens = sumOptional(target.total_cache_read_tokens, usage.cache_read_tokens);
+ target.total_cache_write_tokens = sumOptional(target.total_cache_write_tokens, usage.cache_write_tokens);
+ target.total_reasoning_tokens = sumOptional(target.total_reasoning_tokens, usage.reasoning_tokens);
+ target.total_cost_in_usd = sumOptional(target.total_cost_in_usd, usage.cost_in_usd);
+}
+
export function addAgentThreadMetrics(target: AgentThreadMetrics, source: AgentThreadMetrics): void {
+ target.total_input_tokens += source.total_input_tokens;
+ target.total_output_tokens += source.total_output_tokens;
+ target.total_tokens += source.total_tokens;
+ target.total_cache_read_tokens = sumOptional(target.total_cache_read_tokens, source.total_cache_read_tokens);
+ target.total_cache_write_tokens = sumOptional(target.total_cache_write_tokens, source.total_cache_write_tokens);
+ target.total_reasoning_tokens = sumOptional(target.total_reasoning_tokens, source.total_reasoning_tokens);
+ target.total_cost_in_usd = sumOptional(target.total_cost_in_usd, source.total_cost_in_usd);
target.iterations += source.iterations;
target.total_tool_calls += source.total_tool_calls;
target.total_summarizations += source.total_summarizations;
target.total_sub_agents += source.total_sub_agents;
- target.usage = mergeUsage(target.usage, source.usage);
}
diff --git a/packages/harness/tests/agent-session/store/storeContractSuite.ts b/packages/harness/tests/agent-session/store/storeContractSuite.ts
index 7eed065f6..2c457c469 100644
--- a/packages/harness/tests/agent-session/store/storeContractSuite.ts
+++ b/packages/harness/tests/agent-session/store/storeContractSuite.ts
@@ -15,6 +15,7 @@ import {
import { newEventId } from '../../../src/core/events/schema';
import { getEmptyUsage } from '../../../src/core/llm/LLMTypes';
import type { ContextMessage } from '../../../src/core/runtime/AgentThread.types';
+import { getEmptyCurrentContextUsage } from '../../../src/core/runtime/contextUsage';
import {
makeAgentSpec,
makeCancelledTurnState,
@@ -279,7 +280,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('shared')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -296,7 +297,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('a-only')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -311,7 +312,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('b-only')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -381,7 +382,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('L1a'), userMessage('L1b')],
- current_context_usage: { ...getEmptyUsage(), total_tokens: 2 },
+ current_context_usage: { ...getEmptyCurrentContextUsage(), prompt_tokens: 2 },
},
],
}),
@@ -407,7 +408,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('L2')],
- current_context_usage: { ...getEmptyUsage(), total_tokens: 3 },
+ current_context_usage: { ...getEmptyCurrentContextUsage(), prompt_tokens: 3 },
},
],
}),
@@ -416,7 +417,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
const t2 = await store.getTurn({ tenant_id: tenant, session_id: sessionId, turn_id: 't2' });
const main = mustGet(t2).snapshot.threads[MAIN_THREAD_ID];
expect(contextContents(main?.context)).toEqual(['L1a', 'L1b', 'L2']);
- expect(main?.current_context_usage.total_tokens).toBe(3);
+ expect(main?.current_context_usage.prompt_tokens).toBe(3);
expect(main?.capability_state).toEqual({ 'tfy.plan': { step: 2 } });
});
@@ -469,7 +470,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('from-t1')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -486,7 +487,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('from-t2')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -502,8 +503,8 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
thread_id: MAIN_THREAD_ID,
reason: 'compaction',
context: [userMessage('t2-summary-only')],
- current_context_usage: getEmptyUsage(),
- compaction_llm_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
+ usage: getEmptyUsage(),
},
});
await finishTurn(store, 't2');
@@ -518,7 +519,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('fork-append')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -565,17 +566,17 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('main-msg')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
{
thread_id: 'child-a',
context: [userMessage('a-msg')],
- current_context_usage: { ...getEmptyUsage(), total_tokens: 1 },
+ current_context_usage: { ...getEmptyCurrentContextUsage(), prompt_tokens: 1 },
},
{
thread_id: 'child-b',
context: [userMessage('b-msg-1'), userMessage('b-msg-2')],
- current_context_usage: { ...getEmptyUsage(), total_tokens: 2 },
+ current_context_usage: { ...getEmptyCurrentContextUsage(), prompt_tokens: 2 },
},
],
}),
@@ -601,7 +602,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
input: 'task-a',
});
expect(threads['child-a']?.capability_state).toEqual({ 'tfy.plan': { step: 1 } });
- expect(threads['child-b']?.current_context_usage.total_tokens).toBe(2);
+ expect(threads['child-b']?.current_context_usage.prompt_tokens).toBe(2);
});
});
@@ -653,8 +654,8 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
thread_id: MAIN_THREAD_ID,
reason: 'compaction',
context: [userMessage('late-overwrite')],
- current_context_usage: getEmptyUsage(),
- compaction_llm_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
+ usage: getEmptyUsage(),
},
}),
() =>
@@ -664,7 +665,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: 'child',
context: [],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
parent: { thread_id: MAIN_THREAD_ID, tool_call_id: 'tc1' },
agent_info: { type: 'dynamic', name: 'child', input: 'do work' },
completion: null,
@@ -895,7 +896,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: 'child',
context: [],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
parent: { thread_id: MAIN_THREAD_ID, tool_call_id: 'tc1' },
agent_info: { type: 'dynamic', name: 'child', input: 'do work' },
completion: null,
@@ -926,8 +927,8 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
thread_id: 'child',
reason: 'compaction',
context: [{ role: 'user', content: 'replaced' }],
- current_context_usage: getEmptyUsage(),
- compaction_llm_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
+ usage: getEmptyUsage(),
},
});
turn = await store.getTurn({ tenant_id: tenant, session_id: sessionId, turn_id: 'turn-1' });
@@ -958,7 +959,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: firstBatch,
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
@@ -1065,7 +1066,7 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {
{
thread_id: MAIN_THREAD_ID,
context: [userMessage('hello')],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
},
],
}),
diff --git a/packages/harness/tests/agent-session/testHelpers.ts b/packages/harness/tests/agent-session/testHelpers.ts
index c1e869d33..d8a50d36d 100644
--- a/packages/harness/tests/agent-session/testHelpers.ts
+++ b/packages/harness/tests/agent-session/testHelpers.ts
@@ -8,8 +8,13 @@ import type { CreateTurnInput, NewThreadInit, TurnContextAppend } from '../../sr
import { TurnResourceResolver } from '../../src/agent-session/TurnResourceResolver';
import type { AgentCapability } from '../../src/core/capabilities/AgentCapability';
import { newEventId } from '../../src/core/events/schema';
-import type { ExtendedChatCompletionChunk, RawAssistantMessageWithUsage } from '../../src/core/llm/LLMTypes';
+import type {
+ CompletionUsage,
+ ExtendedChatCompletionChunk,
+ RawAssistantMessageWithUsage,
+} from '../../src/core/llm/LLMTypes';
import { getEmptyUsage } from '../../src/core/llm/LLMTypes';
+import { getEmptyCurrentContextUsage } from '../../src/core/runtime/contextUsage';
import type { Sandbox } from '../../src/core/sandbox/Sandbox';
import { makeMockILLM, makeSilentLogger } from '../core/harnessMocks';
@@ -55,11 +60,9 @@ export function makeAgentSpec(
}
// eslint-disable-next-line @typescript-eslint/require-await -- async generator fixture, not awaiting I/O
-export async function* emptyLlmStream(): AsyncGenerator<
- ExtendedChatCompletionChunk,
- RawAssistantMessageWithUsage,
- unknown
-> {
+export async function* emptyLlmStream(
+ usage: CompletionUsage = getEmptyUsage(),
+): AsyncGenerator<ExtendedChatCompletionChunk, RawAssistantMessageWithUsage, unknown> {
yield {
id: 'chunk-1',
object: 'chat.completion.chunk',
@@ -69,7 +72,7 @@ export async function* emptyLlmStream(): AsyncGenerator<
};
return {
output: { role: 'assistant', content: 'ok' },
- usage: getEmptyUsage(),
+ usage,
finish_reason: 'stop',
};
}
@@ -78,9 +81,10 @@ export function makeTestResolver<TTurnCustom extends object = Record<string, nev
extraCapabilities?: AgentCapability[];
sandbox?: Sandbox;
close?: () => Promise<void>;
+ usage?: CompletionUsage;
}): ITurnResourceResolver<TTurnCustom> {
const llm = makeMockILLM({
- create: jest.fn().mockImplementation(() => emptyLlmStream()),
+ create: jest.fn().mockImplementation(() => emptyLlmStream(options?.usage)),
});
const base = new TurnResourceResolver<TTurnCustom>({
llm: () => llm,
@@ -214,7 +218,7 @@ export function makeRunningTurnRecord(input: {
[MAIN_THREAD_ID]: {
thread_id: MAIN_THREAD_ID,
context: [],
- current_context_usage: getEmptyUsage(),
+ current_context_usage: getEmptyCurrentContextUsage(),
parent: null,
agent_info: null,
completion: null,
diff --git a/packages/harness/tests/agent-session/turnStream.test.ts b/packages/harness/tests/agent-session/turnStream.test.ts
index 6d4bd9e7c..a33bdb454 100644
--- a/packages/harness/tests/agent-session/turnStream.test.ts
+++ b/packages/harness/tests/agent-session/turnStream.test.ts
@@ -49,7 +49,19 @@ describe('TurnHandle.stream()', () => {
}
expect(types[0]).toBe(EventType.TURN_CREATED);
expect(types[types.length - 1]).toBe(EventType.TURN_DONE);
- expect(turn.state.status).toBe('done');
+ expect(turn.state).toMatchObject({
+ status: 'done',
+ metrics: {},
+ });
+ if (turn.state.status === 'done') {
+ // Token counts are always reported, so an unbilled turn aggregates to 0. Cost and the
+ // cache counts stay undefined until a provider actually reports them.
+ expect(turn.state.metrics?.total_input_tokens).toBe(0);
+ expect(turn.state.metrics?.total_output_tokens).toBe(0);
+ expect(turn.state.metrics?.total_tokens).toBe(0);
+ expect(turn.state.metrics?.total_cache_read_tokens).toBeUndefined();
+ expect(turn.state.metrics?.total_cost_in_usd).toBeUndefined();
+ }
const { data } = await turn.listEvents({ limit: 50 });
expect(data.some(e => e.type === EventType.TURN_CREATED)).toBe(true);
@@ -63,6 +75,132 @@ describe('TurnHandle.stream()', () => {
expect(stored?.state.status).toBe('done');
});
+ it('persists final turn usage from orchestrator metrics', async () => {
+ const { session } = await createSession();
+ const turn = await session.createTurn({
+ turn_id: 'turn-usage',
+ input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ previous_turn_id: null,
+ signal: new AbortController().signal,
+ resolver: makeTestResolver({
+ usage: {
+ input_tokens: 12,
+ output_tokens: 5,
+ total_tokens: 17,
+ cache_read_tokens: 4,
+ reasoning_tokens: 3,
+ cost_in_usd: 0.42,
+ },
+ }),
+ });
+
+ for await (const event of turn.stream()) void event;
+
+ expect(turn.state).toMatchObject({
+ status: 'done',
+ metrics: {
+ total_input_tokens: 12,
+ total_output_tokens: 5,
+ total_tokens: 17,
+ total_cache_read_tokens: 4,
+ total_reasoning_tokens: 3,
+ total_cost_in_usd: 0.42,
+ },
+ });
+ });
+
+ it('isolates billable usage across turns (Turn 2 does not include Turn 1)', async () => {
+ const { session } = await createSession();
+
+ const turn1 = await session.createTurn({
+ turn_id: 'turn-isolation-1',
+ input: [{ type: EventType.USER_MESSAGE, content: 'turn one' }],
+ previous_turn_id: null,
+ signal: new AbortController().signal,
+ resolver: makeTestResolver({
+ usage: {
+ input_tokens: 100,
+ output_tokens: 50,
+ total_tokens: 150,
+ cache_read_tokens: 20,
+ cost_in_usd: 1.5,
+ },
+ }),
+ });
+ for await (const event of turn1.stream()) void event;
+ expect(turn1.state).toMatchObject({
+ status: 'done',
+ metrics: {
+ total_input_tokens: 100,
+ total_output_tokens: 50,
+ total_tokens: 150,
+ total_cache_read_tokens: 20,
+ total_cost_in_usd: 1.5,
+ },
+ });
+
+ const turn2 = await session.createTurn({
+ turn_id: 'turn-isolation-2',
+ input: [{ type: EventType.USER_MESSAGE, content: 'turn two' }],
+ previous_turn_id: 'auto',
+ signal: new AbortController().signal,
+ resolver: makeTestResolver({
+ usage: {
+ input_tokens: 7,
+ output_tokens: 3,
+ total_tokens: 10,
+ cache_read_tokens: 1,
+ cost_in_usd: 0.05,
+ },
+ }),
+ });
+ for await (const event of turn2.stream()) void event;
+
+ expect(turn2.state).toMatchObject({
+ status: 'done',
+ metrics: {
+ total_input_tokens: 7,
+ total_output_tokens: 3,
+ total_tokens: 10,
+ total_cache_read_tokens: 1,
+ total_cost_in_usd: 0.05,
+ },
+ });
+ // Explicitly not a sum with turn 1.
+ expect(turn2.state.status === 'done' && turn2.state.metrics).not.toMatchObject({
+ total_input_tokens: 107,
+ total_output_tokens: 53,
+ total_tokens: 160,
+ total_cost_in_usd: 1.55,
+ });
+ });
+
+ it('persists cache-read tokens on turn usage', async () => {
+ const { session } = await createSession();
+ const turn = await session.createTurn({
+ turn_id: 'turn-cache-read',
+ input: [{ type: EventType.USER_MESSAGE, content: 'hello' }],
+ previous_turn_id: null,
+ signal: new AbortController().signal,
+ resolver: makeTestResolver({
+ usage: {
+ input_tokens: 12,
+ output_tokens: 5,
+ total_tokens: 17,
+ cache_read_tokens: 4,
+ cost_in_usd: 0.42,
+ },
+ }),
+ });
+
+ for await (const event of turn.stream()) void event;
+
+ expect(turn.state).toMatchObject({
+ status: 'done',
+ metrics: { total_cache_read_tokens: 4 },
+ });
+ });
+
it('background drain reaches terminal done', async () => {
const { session } = await createSession();
const turn = await session.createTurn({
diff --git a/packages/harness/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts b/packages/harness/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts
index 12e8a1f99..c670d3d32 100644
--- a/packages/harness/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts
+++ b/packages/harness/tests/core/capabilities/builtins/sandboxObjectIdentity.test.ts
@@ -1,7 +1,7 @@
import { largeToolResponse } from '../../../../src/core/capabilities/builtins/LargeToolResponse';
-import { getEmptyUsage } from '../../../../src/core/llm/LLMTypes';
import type { IToolSet } from '../../../../src/core/mcp/IMCPServer';
import type { ToolCallResult } from '../../../../src/core/mcp/executeToolCalls';
+import { getEmptyCurrentContextUsage } from '../../../../src/core/runtime/contextUsage';
import { makeMockIMCPServer, makeSilentLogger, makeStubPublicSandbox } from '../../harnessMocks';
const silentLogger = makeSilentLogger();
@@ -44,7 +44,7 @@ describe('LargeToolResponse sandbox object identity (ISSUE-035)', () => {
await processor.process([sandboxResult, mcpResult], {
threadId: 'main',
sandbox,
- currentContextUsage: getEmptyUsage(),
+ currentContextUsage: getEmptyCurrentContextUsage(),
context: [],
});
diff --git a/packages/harness/tests/core/usage.test.ts b/packages/harness/tests/core/usage.test.ts
new file mode 100644
index 000000000..39e911637
--- /dev/null
+++ b/packages/harness/tests/core/usage.test.ts
@@ -0,0 +1,123 @@
+import { CompletionUsageSchema, getEmptyUsage } from '../../src/core/llm/LLMTypes';
+import { mergeUsage } from '../../src/core/llm/usage';
+import {
+ addAgentThreadMetrics,
+ createEmptyAgentThreadMetrics,
+ updateMetricsFromUsage,
+} from '../../src/core/runtime/metrics';
+
+describe('CompletionUsage', () => {
+ it('accepts the canonical harness usage shape', () => {
+ const usage = {
+ input_tokens: 10,
+ output_tokens: 5,
+ total_tokens: 15,
+ cache_read_tokens: 4,
+ cache_write_tokens: 2,
+ reasoning_tokens: 3,
+ cost_in_usd: 0.12,
+ };
+ expect(CompletionUsageSchema.parse(usage)).toEqual(usage);
+ });
+
+ it('requires input, output, and total token counts', () => {
+ expect(CompletionUsageSchema.safeParse({}).success).toBe(false);
+ });
+
+ it('uses zero for required token counts in empty usage', () => {
+ expect(getEmptyUsage()).toEqual({
+ input_tokens: 0,
+ output_tokens: 0,
+ total_tokens: 0,
+ });
+ });
+});
+
+describe('mergeUsage', () => {
+ it('sums cost across model calls', () => {
+ const merged = mergeUsage(
+ {
+ ...getEmptyUsage(),
+ input_tokens: 10,
+ output_tokens: 5,
+ total_tokens: 15,
+ cost_in_usd: 0.12,
+ },
+ {
+ ...getEmptyUsage(),
+ input_tokens: 20,
+ output_tokens: 8,
+ total_tokens: 28,
+ cost_in_usd: 0.23,
+ },
+ );
+
+ expect(merged).toMatchObject({
+ input_tokens: 30,
+ output_tokens: 13,
+ total_tokens: 43,
+ });
+ expect(merged.cost_in_usd).toBe(0.12 + 0.23);
+ });
+
+ it('retains full precision and never rounds cost to cents', () => {
+ // Sub-cent inputs must survive verbatim; any rounding to 2 dp would collapse these to 0.
+ const merged = mergeUsage({ ...getEmptyUsage(), cost_in_usd: 0.0001 }, { ...getEmptyUsage(), cost_in_usd: 0.0002 });
+
+ expect(merged.cost_in_usd).toBe(0.0001 + 0.0002);
+ expect(merged.cost_in_usd).toBeGreaterThan(0);
+ });
+
+ it('preserves a missing cost as absent until a call reports one', () => {
+ const merged = mergeUsage(getEmptyUsage(), { ...getEmptyUsage(), cost_in_usd: 0.5 });
+
+ expect(merged.cost_in_usd).toBe(0.5);
+ });
+
+ it('keeps optional fields undefined when neither side reports them', () => {
+ const merged = mergeUsage(getEmptyUsage(), getEmptyUsage());
+
+ expect(merged.cache_read_tokens).toBeUndefined();
+ expect(merged.cache_write_tokens).toBeUndefined();
+ expect(merged.reasoning_tokens).toBeUndefined();
+ expect(merged.cost_in_usd).toBeUndefined();
+ });
+
+ it('sums flat cache, reasoning, and cost fields', () => {
+ const merged = mergeUsage(
+ { ...getEmptyUsage(), cache_read_tokens: 9, cache_write_tokens: 2, reasoning_tokens: 3, cost_in_usd: 0.1 },
+ { ...getEmptyUsage(), cache_read_tokens: 4, cache_write_tokens: 3, reasoning_tokens: 5, cost_in_usd: 0.2 },
+ );
+ expect(merged.cache_read_tokens).toBe(13);
+ expect(merged.cache_write_tokens).toBe(5);
+ expect(merged.reasoning_tokens).toBe(8);
+ expect(merged.cost_in_usd).toBe(0.1 + 0.2);
+ });
+});
+
+describe('AgentThreadMetrics', () => {
+ it('initializes required token totals to zero and leaves optional totals undefined', () => {
+ const target = createEmptyAgentThreadMetrics();
+ addAgentThreadMetrics(target, createEmptyAgentThreadMetrics());
+
+ expect(target.total_input_tokens).toBe(0);
+ expect(target.total_output_tokens).toBe(0);
+ expect(target.total_tokens).toBe(0);
+ expect(target.total_cache_read_tokens).toBeUndefined();
+ expect(target.total_cost_in_usd).toBeUndefined();
+ });
+
+ it('materializes required token aggregates after folding usage', () => {
+ const target = createEmptyAgentThreadMetrics();
+
+ updateMetricsFromUsage(target, getEmptyUsage());
+
+ expect(target.total_input_tokens).toBe(0);
+ expect(target.total_output_tokens).toBe(0);
+ expect(target.total_tokens).toBe(0);
+ expect(target.total_cache_read_tokens).toBeUndefined();
+ expect(target.total_cache_write_tokens).toBeUndefined();
+ expect(target.total_reasoning_tokens).toBeUndefined();
+ expect(target.total_cost_in_usd).toBeUndefined();
+ });
+});
diff --git a/packages/sdk/.fern/metadata.json b/packages/sdk/.fern/metadata.json
index b4dab6b60..afe6487a1 100644
--- a/packages/sdk/.fern/metadata.json
+++ b/packages/sdk/.fern/metadata.json
@@ -26,8 +26,8 @@
"streamType": "web",
"useDefaultRequestParameterValues": true
},
- "originGitCommit": "70854f9f99677aeca430caf131a2d13df2996205",
- "originGitCommitIsDirty": false,
+ "originGitCommit": "5360bb423eae4f38106c9faeba6ca272904bc605",
+ "originGitCommitIsDirty": true,
"invokedBy": "ci",
"requestedVersion": "0.0.0",
"ciProvider": "github",
diff --git a/packages/sdk/src/api/types/TurnMetrics.ts b/packages/sdk/src/api/types/TurnMetrics.ts
new file mode 100644
index 000000000..a00ae81a6
--- /dev/null
+++ b/packages/sdk/src/api/types/TurnMetrics.ts
@@ -0,0 +1,11 @@
+// This file was auto-generated by Fern from our API Definition.
+
+export interface TurnMetrics {
+ totalCacheReadTokens?: number;
+ totalCacheWriteTokens?: number;
+ totalCostInUsd?: number;
+ totalInputTokens?: number;
+ totalOutputTokens?: number;
+ totalReasoningTokens?: number;
+ totalTokens?: number;
+}
diff --git a/packages/sdk/src/api/types/TurnStateCancelled.ts b/packages/sdk/src/api/types/TurnStateCancelled.ts
index 47e2c8a97..55f23c668 100644
--- a/packages/sdk/src/api/types/TurnStateCancelled.ts
+++ b/packages/sdk/src/api/types/TurnStateCancelled.ts
@@ -4,6 +4,7 @@ import type * as TrueHarness from "../index.js";
export interface TurnStateCancelled {
completedAt: string;
+ metrics?: TrueHarness.TurnMetrics;
reason: TrueHarness.TurnStateCancelledReason;
status: "cancelled";
}
diff --git a/packages/sdk/src/api/types/TurnStateDone.ts b/packages/sdk/src/api/types/TurnStateDone.ts
index 202a84fad..dddabae41 100644
--- a/packages/sdk/src/api/types/TurnStateDone.ts
+++ b/packages/sdk/src/api/types/TurnStateDone.ts
@@ -4,6 +4,7 @@ import type * as TrueHarness from "../index.js";
export interface TurnStateDone {
completedAt: string;
+ metrics?: TrueHarness.TurnMetrics;
output?: TrueHarness.ModelMessageEvent;
requiredActions: TrueHarness.ActionRequiredEvent[];
status: "done";
diff --git a/packages/sdk/src/api/types/TurnStateError.ts b/packages/sdk/src/api/types/TurnStateError.ts
index 1b771ab0d..cd340fe02 100644
--- a/packages/sdk/src/api/types/TurnStateError.ts
+++ b/packages/sdk/src/api/types/TurnStateError.ts
@@ -1,7 +1,10 @@
// This file was auto-generated by Fern from our API Definition.
+import type * as TrueHarness from "../index.js";
+
export interface TurnStateError {
completedAt: string;
message: string;
+ metrics?: TrueHarness.TurnMetrics;
status: "error";
}
diff --git a/packages/sdk/src/api/types/index.ts b/packages/sdk/src/api/types/index.ts
index 2706372a6..5361e4cfc 100644
--- a/packages/sdk/src/api/types/index.ts
+++ b/packages/sdk/src/api/types/index.ts
@@ -100,6 +100,7 @@ export * from "./TurnCreatedEvent.js";
export * from "./TurnDoneEvent.js";
export * from "./TurnDoneEventState.js";
export * from "./TurnInputItem.js";
+export * from "./TurnMetrics.js";
export * from "./TurnState.js";
export * from "./TurnStateCancelled.js";
export * from "./TurnStateCancelledReason.js";
diff --git a/packages/sdk/src/serialization/types/TurnMetrics.ts b/packages/sdk/src/serialization/types/TurnMetrics.ts
new file mode 100644
index 000000000..a49e979ae
--- /dev/null
+++ b/packages/sdk/src/serialization/types/TurnMetrics.ts
@@ -0,0 +1,37 @@
+// This file was auto-generated by Fern from our API Definition.
+
+import type * as TrueHarness from "../../api/index.js";
+import * as core from "../../core/index.js";
+import type * as serializers from "../index.js";
+
+export const TurnMetrics: core.serialization.ObjectSchema<serializers.TurnMetrics.Raw, TrueHarness.TurnMetrics> =
+ core.serialization.object({
+ totalCacheReadTokens: core.serialization.property(
+ "total_cache_read_tokens",
+ core.serialization.number().optional(),
+ ),
+ totalCacheWriteTokens: core.serialization.property(
+ "total_cache_write_tokens",
+ core.serialization.number().optional(),
+ ),
+ totalCostInUsd: core.serialization.property("total_cost_in_usd", core.serialization.number().optional()),
+ totalInputTokens: core.serialization.property("total_input_tokens", core.serialization.number().optional()),
+ totalOutputTokens: core.serialization.property("total_output_tokens", core.serialization.number().optional()),
+ totalReasoningTokens: core.serialization.property(
+ "total_reasoning_tokens",
+ core.serialization.number().optional(),
+ ),
+ totalTokens: core.serialization.property("total_tokens", core.serialization.number().optional()),
+ });
+
+export declare namespace TurnMetrics {
+ export interface Raw {
+ total_cache_read_tokens?: number | null;
+ total_cache_write_tokens?: number | null;
+ total_cost_in_usd?: number | null;
+ total_input_tokens?: number | null;
+ total_output_tokens?: number | null;
+ total_reasoning_tokens?: number | null;
+ total_tokens?: number | null;
+ }
+}
diff --git a/packages/sdk/src/serialization/types/TurnStateCancelled.ts b/packages/sdk/src/serialization/types/TurnStateCancelled.ts
index 70999a8c7..93e2b8793 100644
--- a/packages/sdk/src/serialization/types/TurnStateCancelled.ts
+++ b/packages/sdk/src/serialization/types/TurnStateCancelled.ts
@@ -3,6 +3,7 @@
import type * as TrueHarness from "../../api/index.js";
import * as core from "../../core/index.js";
import type * as serializers from "../index.js";
+import { TurnMetrics } from "./TurnMetrics.js";
import { TurnStateCancelledReason } from "./TurnStateCancelledReason.js";
export const TurnStateCancelled: core.serialization.ObjectSchema<
@@ -10,6 +11,7 @@ export const TurnStateCancelled: core.serialization.ObjectSchema<
TrueHarness.TurnStateCancelled
> = core.serialization.object({
completedAt: core.serialization.property("completed_at", core.serialization.string()),
+ metrics: TurnMetrics.optional(),
reason: TurnStateCancelledReason,
status: core.serialization.stringLiteral("cancelled"),
});
@@ -17,6 +19,7 @@ export const TurnStateCancelled: core.serialization.ObjectSchema<
export declare namespace TurnStateCancelled {
export interface Raw {
completed_at: string;
+ metrics?: TurnMetrics.Raw | null;
reason: TurnStateCancelledReason.Raw;
status: "cancelled";
}
diff --git a/packages/sdk/src/serialization/types/TurnStateDone.ts b/packages/sdk/src/serialization/types/TurnStateDone.ts
index 1e670d715..d49861751 100644
--- a/packages/sdk/src/serialization/types/TurnStateDone.ts
+++ b/packages/sdk/src/serialization/types/TurnStateDone.ts
@@ -5,10 +5,12 @@ import * as core from "../../core/index.js";
import type * as serializers from "../index.js";
import { ActionRequiredEvent } from "./ActionRequiredEvent.js";
import { ModelMessageEvent } from "./ModelMessageEvent.js";
+import { TurnMetrics } from "./TurnMetrics.js";
export const TurnStateDone: core.serialization.ObjectSchema<serializers.TurnStateDone.Raw, TrueHarness.TurnStateDone> =
core.serialization.object({
completedAt: core.serialization.property("completed_at", core.serialization.string()),
+ metrics: TurnMetrics.optional(),
output: ModelMessageEvent.optional(),
requiredActions: core.serialization.property("required_actions", core.serialization.list(ActionRequiredEvent)),
status: core.serialization.stringLiteral("done"),
@@ -17,6 +19,7 @@ export const TurnStateDone: core.serialization.ObjectSchema<serializers.TurnStat
export declare namespace TurnStateDone {
export interface Raw {
completed_at: string;
+ metrics?: TurnMetrics.Raw | null;
output?: ModelMessageEvent.Raw | null;
required_actions: ActionRequiredEvent.Raw[];
status: "done";
diff --git a/packages/sdk/src/serialization/types/TurnStateError.ts b/packages/sdk/src/serialization/types/TurnStateError.ts
index 1adf1b7e3..21eb7d5b4 100644
--- a/packages/sdk/src/serialization/types/TurnStateError.ts
+++ b/packages/sdk/src/serialization/types/TurnStateError.ts
@@ -3,6 +3,7 @@
import type * as TrueHarness from "../../api/index.js";
import * as core from "../../core/index.js";
import type * as serializers from "../index.js";
+import { TurnMetrics } from "./TurnMetrics.js";
export const TurnStateError: core.serialization.ObjectSchema<
serializers.TurnStateError.Raw,
@@ -10,6 +11,7 @@ export const TurnStateError: core.serialization.ObjectSchema<
> = core.serialization.object({
completedAt: core.serialization.property("completed_at", core.serialization.string()),
message: core.serialization.string(),
+ metrics: TurnMetrics.optional(),
status: core.serialization.stringLiteral("error"),
});
@@ -17,6 +19,7 @@ export declare namespace TurnStateError {
export interface Raw {
completed_at: string;
message: string;
+ metrics?: TurnMetrics.Raw | null;
status: "error";
}
}
diff --git a/packages/sdk/src/serialization/types/index.ts b/packages/sdk/src/serialization/types/index.ts
index 2706372a6..5361e4cfc 100644
--- a/packages/sdk/src/serialization/types/index.ts
+++ b/packages/sdk/src/serialization/types/index.ts
@@ -100,6 +100,7 @@ export * from "./TurnCreatedEvent.js";
export * from "./TurnDoneEvent.js";
export * from "./TurnDoneEventState.js";
export * from "./TurnInputItem.js";
+export * from "./TurnMetrics.js";
export * from "./TurnState.js";
export * from "./TurnStateCancelled.js";
export * from "./TurnStateCancelledReason.js";
diff --git a/packages/server/src/db/postgres/session-store/queries/threads.ts b/packages/server/src/db/postgres/session-store/queries/threads.ts
index fac1684d2..cd8f04724 100644
--- a/packages/server/src/db/postgres/session-store/queries/threads.ts
+++ b/packages/server/src/db/postgres/session-store/queries/threads.ts
@@ -8,8 +8,8 @@ import type {
} from '@truefoundry/utils/agent-session/store/ISessionStore';
import type { JsonValue } from '@truefoundry/utils/core/capabilities/AgentCapability';
import type { AgentInfo } from '@truefoundry/utils/core/events/schema';
-import type { CompletionUsage } from '@truefoundry/utils/core/llm/LLMTypes';
import type { ContextMessage, SubAgentCompletionMarker } from '@truefoundry/utils/core/runtime/AgentThread.types';
+import type { CurrentContextUsage } from '@truefoundry/utils/core/runtime/contextUsage';
import { sql, type Kysely, type RawBuilder, type Transaction } from 'kysely';
import type { Database, TurnThreadCheckpoint } from '../../types';
import { json, jsonbSet, values } from '../sqlExpressions';
@@ -59,7 +59,7 @@ export async function addThreads(db: Kysely<Database>, input: AddThreadsInput):
thread_id: string;
checkpoint: TurnThreadCheckpoint;
agent_info: RawBuilder<AgentInfo> | null;
- current_context_usage: CompletionUsage;
+ current_context_usage: CurrentContextUsage;
}[] = [];
for (const thread of input.threads) {
@@ -188,7 +188,7 @@ function completionPatchExpr(completion: SubAgentCompletionMarker | null): RawBu
return jsonbSet<TurnThreadCheckpoint>(sql`checkpoint`, sql`'{completion}'`, json(completion));
}
-function usageSetExpr(usage: CompletionUsage | null): RawBuilder<CompletionUsage> {
+function usageSetExpr(usage: CurrentContextUsage | null): RawBuilder<CurrentContextUsage> {
if (usage === null) {
return sql`current_context_usage`;
}
@@ -203,10 +203,10 @@ async function fencedTurnThreadContextUpdate(
thread_id: string;
context: ContextMessage[];
replace_array: boolean;
- current_context_usage: CompletionUsage | null;
+ current_context_usage: CurrentContextUsage | null;
completion: SubAgentCompletionMarker | null;
/** When replace_array, usage is set unconditionally (overwrite contract). */
- usage_unconditional: CompletionUsage | null;
+ usage_unconditional: CurrentContextUsage | null;
},
): Promise<void> {
const { keys, thread_id, context, replace_array } = args;
@@ -403,7 +403,7 @@ export async function getTurnThread(
thread_id: string;
checkpoint: TurnThreadCheckpoint;
agent_info: unknown;
- current_context_usage: CompletionUsage;
+ current_context_usage: CurrentContextUsage;
context_ids: number[];
}
| undefined
diff --git a/packages/server/src/db/postgres/session-store/queries/turns.ts b/packages/server/src/db/postgres/session-store/queries/turns.ts
index 22d582a8f..4ebf6bb04 100644
--- a/packages/server/src/db/postgres/session-store/queries/turns.ts
+++ b/packages/server/src/db/postgres/session-store/queries/turns.ts
@@ -22,9 +22,9 @@ import {
} from '@truefoundry/utils/agent-session/store/SessionStoreErrors';
import type { CapabilityState, JsonValue } from '@truefoundry/utils/core/capabilities/AgentCapability';
import type { AgentInfo, AgentParent, MCPServerInitInfo } from '@truefoundry/utils/core/events/schema';
-import type { CompletionUsage } from '@truefoundry/utils/core/llm/LLMTypes';
-import { getEmptyUsage } from '@truefoundry/utils/core/llm/LLMTypes';
import type { AgentThreadSnapshot, ContextMessage } from '@truefoundry/utils/core/runtime/AgentThread.types';
+import type { CurrentContextUsage } from '@truefoundry/utils/core/runtime/contextUsage';
+import { getEmptyCurrentContextUsage } from '@truefoundry/utils/core/runtime/contextUsage';
import type { SandboxInfo } from '@truefoundry/utils/core/sandbox/Sandbox';
import { sql, type Kysely, type QueryCreator, type RawBuilder, type Transaction } from 'kysely';
import { isUniqueViolation } from '../../client';
@@ -63,7 +63,7 @@ export interface TurnKeys {
export interface NewContextAppend {
thread_id: string;
context: ContextMessage[];
- current_context_usage: CompletionUsage | null;
+ current_context_usage: CurrentContextUsage | null;
}
export interface CreateTurnTurnFields {
@@ -265,7 +265,7 @@ async function assembleTurnRecord(
{
checkpoint: TurnThreadCheckpoint;
agent_info: AgentInfo | null;
- current_context_usage: CompletionUsage;
+ current_context_usage: CurrentContextUsage;
}
>();
@@ -355,7 +355,7 @@ interface TurnThreadInsertRow {
thread_id: string;
checkpoint: TurnThreadCheckpoint;
agent_info: RawBuilder<AgentInfo> | null;
- current_context_usage: CompletionUsage;
+ current_context_usage: CurrentContextUsage;
context_ids: number[];
updated_at: Date;
}
@@ -415,7 +415,7 @@ export async function createTurn(db: Kysely<Database>, input: CreateTurnInput):
thread_id: string;
checkpoint: TurnThreadCheckpoint;
agent_info: AgentInfo | null;
- current_context_usage: CompletionUsage;
+ current_context_usage: CurrentContextUsage;
context_ids: number[];
}[] = [];
@@ -541,7 +541,7 @@ export async function createTurn(db: Kysely<Database>, input: CreateTurnInput):
}
}
- const appendUsageByThread = new Map<string, CompletionUsage>();
+ const appendUsageByThread = new Map<string, CurrentContextUsage>();
for (const append of input.new_context_appends) {
if (append.current_context_usage !== null) {
appendUsageByThread.set(append.thread_id, append.current_context_usage);
@@ -569,7 +569,7 @@ export async function createTurn(db: Kysely<Database>, input: CreateTurnInput):
for (const nt of input.new_threads) {
const newIds = newIdsByThread.get(nt.thread_id) ?? [];
- const usage = appendUsageByThread.get(nt.thread_id) ?? getEmptyUsage();
+ const usage = appendUsageByThread.get(nt.thread_id) ?? getEmptyCurrentContextUsage();
const threadCheckpoint: TurnThreadCheckpoint = {
parent: nt.parent,
completion: null,
diff --git a/packages/server/src/db/postgres/types.ts b/packages/server/src/db/postgres/types.ts
index 361634a5f..8017e35b5 100644
--- a/packages/server/src/db/postgres/types.ts
+++ b/packages/server/src/db/postgres/types.ts
@@ -12,7 +12,7 @@ import type {
SandboxInfo,
SubAgentCompletionMarker,
} from '@truefoundry/utils/core';
-import type { CompletionUsage } from '@truefoundry/utils/core/llm/LLMTypes';
+import type { CurrentContextUsage } from '@truefoundry/utils/core/runtime/contextUsage';
import type { ColumnType, Generated, JSONColumnType } from 'kysely';
/**
@@ -157,7 +157,7 @@ export interface TurnThreadTable {
* (ranges / writer-filter), move this back onto
* thread_context_log rows (previously validated design).
*/
- current_context_usage: JSONColumnType<CompletionUsage, CompletionUsage, CompletionUsage>;
+ current_context_usage: JSONColumnType<CurrentContextUsage, CurrentContextUsage, CurrentContextUsage>;
/**
* top: THE context — log append_ids in context order. Hot
* concat target (`||`), typed bigint[] operator; dominant