The open-source agent harness - the runtime layer that turns an LLM into a working agent.
fix: Omit session cost when unavailable instead of defaulting to 0. (#607)
Heer Ambavi0ec8dc6529bb
diff --git a/.changeset/optional-session-cost.md b/.changeset/optional-session-cost.mdnew file mode 100644index 000000000..6d2254a19--- /dev/null+++ b/.changeset/optional-session-cost.md@@ -0,0 +1,7 @@+---+"@truefoundry/trueforge-core": patch+"@truefoundry/trueforge": patch+"@truefoundry/trueforge-sdk": patch+---++Omit session `total_cost_in_usd` when cost is unavailable (instead of defaulting to 0), matching turn metrics.diff --git a/.github/fern/openapi/openapi.json b/.github/fern/openapi/openapi.jsonindex 6c5443586..dd5cee2fa 100644--- a/.github/fern/openapi/openapi.json+++ b/.github/fern/openapi/openapi.json@@ -3491,6 +3491,7 @@"description": "Rolled-up cost, duration, and turn counters for a session.","properties": {"total_cost_in_usd": {+ "description": "Estimated total cost in USD for this session","minimum": 0,"type": "number"},@@ -3504,7 +3505,6 @@}},"required": [- "total_cost_in_usd","total_duration_ms","total_turns"],diff --git a/docs/openapi.json b/docs/openapi.jsonindex 6c5443586..dd5cee2fa 100644--- a/docs/openapi.json+++ b/docs/openapi.json@@ -3491,6 +3491,7 @@"description": "Rolled-up cost, duration, and turn counters for a session.","properties": {"total_cost_in_usd": {+ "description": "Estimated total cost in USD for this session","minimum": 0,"type": "number"},@@ -3504,7 +3505,6 @@}},"required": [- "total_cost_in_usd","total_duration_ms","total_turns"],diff --git a/packages/trueforge-core/src/agent-session/schemas/session.ts b/packages/trueforge-core/src/agent-session/schemas/session.tsindex 215adad7d..8fea9f5fb 100644--- a/packages/trueforge-core/src/agent-session/schemas/session.ts+++ b/packages/trueforge-core/src/agent-session/schemas/session.ts@@ -35,7 +35,7 @@ export type SessionMetadata = z.infer<typeof SessionMetadataSchema>;export const SessionMetricsSchema = z.object({- total_cost_in_usd: z.number().nonnegative(),+ total_cost_in_usd: z.number().nonnegative().optional().describe('Estimated total cost in USD for this session'),total_duration_ms: z.number().int().nonnegative(),total_turns: z.number().int().nonnegative(),})diff --git a/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts b/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.tsindex cd22e8f97..05d2de4f2 100644--- a/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts+++ b/packages/trueforge-core/src/agent-session/store/InMemorySessionStore.ts@@ -197,7 +197,6 @@ export class InMemorySessionStore<updated_at: now,last_activity_timestamp_ms: Date.now(),metrics: {- total_cost_in_usd: 0,total_duration_ms: 0,total_turns: 0,},@@ -456,14 +455,17 @@ export class InMemorySessionStore<return;}- /** Cost from turn metrics; duration is completed_at − created_at, floored at 0. */+ /** Cost from turn metrics when present; duration is completed_at − created_at, floored at 0. */private addTerminalSessionMetrics(sessionId: string, created_at: Date, state: TerminalTurnState): void {const stored = this.sessions.get(sessionKey(sessionId));if (!stored) {throw new SessionNotFoundError(sessionId);}const elapsed_ms = Date.parse(state.completed_at) - created_at.getTime();- stored.record.metrics.total_cost_in_usd += state.metrics?.total_cost_in_usd ?? 0;+ const turnCost = state.metrics?.total_cost_in_usd;+ if (turnCost !== undefined) {+ stored.record.metrics.total_cost_in_usd = (stored.record.metrics.total_cost_in_usd ?? 0) + turnCost;+ }stored.record.metrics.total_duration_ms += elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;}diff --git a/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts b/packages/trueforge-core/tests/agent-session/store/storeContractSuite.tsindex cf4a1b4d7..41ef288d3 100644--- a/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts+++ b/packages/trueforge-core/tests/agent-session/store/storeContractSuite.ts@@ -181,7 +181,6 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {expect(mustGet(session).last_activity_timestamp_ms).toBeGreaterThanOrEqual(before);expect(mustGet(session).title).toBeNull();expect(mustGet(session).metrics).toEqual({- total_cost_in_usd: 0,total_duration_ms: 0,total_turns: 0,});@@ -1233,7 +1232,6 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));const afterFirst = await store.getSession({ tenant_id: tenant, session_id: sessionId });expect(mustGet(afterFirst).metrics).toEqual({- total_cost_in_usd: 0,total_duration_ms: 0,total_turns: 1,});@@ -1744,7 +1742,6 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {const afterCancel = await store.getSession({ tenant_id: tenant, session_id: sessionId });const elapsed_ms = Date.parse(record.state.completed_at) - record.created_at.getTime();expect(mustGet(afterCancel).metrics).toEqual({- total_cost_in_usd: 0,total_duration_ms: elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0,total_turns: 1,});@@ -1923,6 +1920,32 @@ export function runStoreContractSuite(createStore: () => ISessionStore) {});});+ it('leaves session.metrics.total_cost_in_usd unset when the terminal turn has no cost', async () => {+ const store = createStore();+ await seedSession(store);+ await store.createTurn(makeCreateTurnInput({ sessionId, turnId: 'turn-1' }));+ const turn = await store.getTurn({ session_id: sessionId, turn_id: 'turn-1' });+ const createdAt = mustGet(turn).created_at;+ const completedAt = new Date(createdAt.getTime() + 1500).toISOString();+ const state = {+ ...makeDoneTurnState(),+ completed_at: completedAt,+ metrics: { total_tokens: 10 },+ };+ await store.updateTurnState({+ session_id: sessionId,+ turn_id: 'turn-1',+ state,+ turn_done_event: makeTurnDoneEvent(state),+ });+ const session = await store.getSession({ tenant_id: tenant, session_id: sessionId });+ expect(mustGet(session).metrics).toEqual({+ total_duration_ms: 1500,+ total_turns: 1,+ });+ expect(mustGet(session).metrics.total_cost_in_usd).toBeUndefined();+ });+it('does not add session.metrics again on a losing terminal write', async () => {const store = createStore();await seedSession(store);diff --git a/packages/trueforge-sdk/src/api/types/SessionMetrics.ts b/packages/trueforge-sdk/src/api/types/SessionMetrics.tsindex 2c2d1c18b..907d484ec 100644--- a/packages/trueforge-sdk/src/api/types/SessionMetrics.ts+++ b/packages/trueforge-sdk/src/api/types/SessionMetrics.ts@@ -4,7 +4,8 @@* Rolled-up cost, duration, and turn counters for a session.*/export interface SessionMetrics {- totalCostInUsd: number;+ /** Estimated total cost in USD for this session */+ totalCostInUsd?: number;totalDurationMs: number;totalTurns: number;}diff --git a/packages/trueforge-sdk/src/serialization/types/SessionMetrics.ts b/packages/trueforge-sdk/src/serialization/types/SessionMetrics.tsindex 075eafff7..27bb66d4b 100644--- a/packages/trueforge-sdk/src/serialization/types/SessionMetrics.ts+++ b/packages/trueforge-sdk/src/serialization/types/SessionMetrics.ts@@ -6,14 +6,14 @@ import type * as serializers from "../index.js";export const SessionMetrics: core.serialization.ObjectSchema<serializers.SessionMetrics.Raw, TrueForge.SessionMetrics> =core.serialization.object({- totalCostInUsd: core.serialization.property("total_cost_in_usd", core.serialization.number()),+ totalCostInUsd: core.serialization.property("total_cost_in_usd", core.serialization.number().optional()),totalDurationMs: core.serialization.property("total_duration_ms", core.serialization.number()),totalTurns: core.serialization.property("total_turns", core.serialization.number()),});export declare namespace SessionMetrics {export interface Raw {- total_cost_in_usd: number;+ total_cost_in_usd?: number | null;total_duration_ms: number;total_turns: number;}diff --git a/packages/trueforge-sdk/tests/wire/sessions.test.ts b/packages/trueforge-sdk/tests/wire/sessions.test.tsindex d9344b4d4..0eb6c2dc9 100644--- a/packages/trueforge-sdk/tests/wire/sessions.test.ts+++ b/packages/trueforge-sdk/tests/wire/sessions.test.ts@@ -21,7 +21,7 @@ describe("SessionsClient", () => {},id: "id",metadata: { key: "value" },- metrics: { total_cost_in_usd: 1.1, total_duration_ms: 1, total_turns: 1 },+ metrics: { total_duration_ms: 1, total_turns: 1 },source: { id: "id", run_id: "run_id", type: "schedule" },title: "title",updated_at: "updated_at",@@ -60,7 +60,6 @@ describe("SessionsClient", () => {key: "value",},metrics: {- totalCostInUsd: 1.1,totalDurationMs: 1,totalTurns: 1,},diff --git a/packages/trueforge-ui/src/plugins/trueforge-agent-server-adapter/agentSessionsServer.ts b/packages/trueforge-ui/src/plugins/trueforge-agent-server-adapter/agentSessionsServer.tsindex f7c412324..bee2cbd31 100644--- a/packages/trueforge-ui/src/plugins/trueforge-agent-server-adapter/agentSessionsServer.ts+++ b/packages/trueforge-ui/src/plugins/trueforge-agent-server-adapter/agentSessionsServer.ts@@ -25,7 +25,7 @@ function toSessionListEntry(session: TrueForgeApi.Session): HarnessSessionListEnisMutable: session.agent.type === 'inline',metrics: {totalTurns: session.metrics.totalTurns,- totalCostInUsd: session.metrics.totalCostInUsd,+ totalCostInUsd: session.metrics.totalCostInUsd ?? 0, // TODO: fix this.totalDurationMs: session.metrics.totalDurationMs,},...(session.title === null ? {} : { title: session.title }),diff --git a/packages/trueforge/src/db/postgres/session-store/queries/sessions.ts b/packages/trueforge/src/db/postgres/session-store/queries/sessions.tsindex 6a42edb5d..ce99acddb 100644--- a/packages/trueforge/src/db/postgres/session-store/queries/sessions.ts+++ b/packages/trueforge/src/db/postgres/session-store/queries/sessions.ts@@ -116,7 +116,6 @@ export async function createSession(db: Kysely<Database>, input: CreateSessionInmetadata: json(input.metadata),external_id: input.external_id,metrics: json({- total_cost_in_usd: 0,total_duration_ms: 0,total_turns: 0,}),diff --git a/packages/trueforge/src/db/postgres/session-store/queries/turns.ts b/packages/trueforge/src/db/postgres/session-store/queries/turns.tsindex f333b8e5e..0739e3508 100644--- a/packages/trueforge/src/db/postgres/session-store/queries/turns.ts+++ b/packages/trueforge/src/db/postgres/session-store/queries/turns.ts@@ -125,21 +125,25 @@ async function addSessionCostAndDuration(input: { session_id: string; turn_created_at: Date; turn_state: TerminalTurnState },): Promise<void> {const elapsed_ms = Date.parse(input.turn_state.completed_at) - input.turn_created_at.getTime();- const total_cost_in_usd = input.turn_state.metrics?.total_cost_in_usd ?? 0;const total_duration_ms = elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;+ const turnCost = input.turn_state.metrics?.total_cost_in_usd;+ // bigint: ::int overflows at ~24.8 days of summed ms and would roll back the terminal tx.+ const withDuration = jsonbSet<SessionMetrics>(+ sql`metrics`,+ sql`'{total_duration_ms}'`,+ sql`to_jsonb((metrics->>'total_duration_ms')::bigint + ${total_duration_ms}::bigint)`,+ );await trx.updateTable('session').set({- metrics: jsonbSet<SessionMetrics>(- jsonbSet(- sql`metrics`,- sql`'{total_cost_in_usd}'`,- sql`to_jsonb((metrics->>'total_cost_in_usd')::double precision + ${total_cost_in_usd}::double precision)`,- ),- // bigint: ::int overflows at ~24.8 days of summed ms and would roll back the terminal tx.- sql`'{total_duration_ms}'`,- sql`to_jsonb((metrics->>'total_duration_ms')::bigint + ${total_duration_ms}::bigint)`,- ),+ metrics:+ turnCost === undefined+ ? withDuration+ : jsonbSet<SessionMetrics>(+ withDuration,+ sql`'{total_cost_in_usd}'`,+ sql`to_jsonb(COALESCE((metrics->>'total_cost_in_usd')::double precision, 0) + ${turnCost}::double precision)`,+ ),}).where('session_id', '=', input.session_id).execute();diff --git a/packages/trueforge/src/db/sqlite/session-store/queries/sessions.ts b/packages/trueforge/src/db/sqlite/session-store/queries/sessions.tsindex 6eca1db0b..689873ee5 100644--- a/packages/trueforge/src/db/sqlite/session-store/queries/sessions.ts+++ b/packages/trueforge/src/db/sqlite/session-store/queries/sessions.ts@@ -136,7 +136,6 @@ export async function createSession(db: Kysely<Database>, input: CreateSessionInmetadata: jsonbBind(input.metadata),external_id: input.external_id,metrics: jsonbBind({- total_cost_in_usd: 0,total_duration_ms: 0,total_turns: 0,}),diff --git a/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts b/packages/trueforge/src/db/sqlite/session-store/queries/turns.tsindex fc1fe30b4..75038bb4f 100644--- a/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts+++ b/packages/trueforge/src/db/sqlite/session-store/queries/turns.ts@@ -1,3 +1,4 @@+import type { SessionMetrics } from '@truefoundry/trueforge-core/agent-session';import type { TurnRecord, TurnSnapshot } from '@truefoundry/trueforge-core/agent-session/models/TurnRecord';import {type TerminalTurnState,@@ -115,16 +116,24 @@ async function addSessionCostAndDuration(input: { session_id: string; turn_created_at: Date; turn_state: TerminalTurnState },): Promise<void> {const elapsed_ms = Date.parse(input.turn_state.completed_at) - input.turn_created_at.getTime();- const total_cost_in_usd = input.turn_state.metrics?.total_cost_in_usd ?? 0;const total_duration_ms = elapsed_ms > 0 ? Math.trunc(elapsed_ms) : 0;+ const turnCost = input.turn_state.metrics?.total_cost_in_usd;+ const withDuration = sql<SessionMetrics>`jsonb_set(+ metrics,+ '$.total_duration_ms',+ jsonb((metrics->>'total_duration_ms') + ${total_duration_ms})+ )`;await trx.updateTable('session').set({- metrics: sql`jsonb_set(- jsonb_set(metrics, '$.total_cost_in_usd', jsonb((metrics->>'total_cost_in_usd') + ${total_cost_in_usd})),- '$.total_duration_ms',- jsonb((metrics->>'total_duration_ms') + ${total_duration_ms})- )`,+ metrics:+ turnCost === undefined+ ? withDuration+ : sql<SessionMetrics>`jsonb_set(+ ${withDuration},+ '$.total_cost_in_usd',+ jsonb(COALESCE(metrics->>'total_cost_in_usd', 0) + ${turnCost})+ )`,}).where('session_id', '=', input.session_id).execute();diff --git a/packages/trueforge/tests/unit/apis/sessionHttp.test.ts b/packages/trueforge/tests/unit/apis/sessionHttp.test.tsindex 2cb81736d..f45eed38d 100644--- a/packages/trueforge/tests/unit/apis/sessionHttp.test.ts+++ b/packages/trueforge/tests/unit/apis/sessionHttp.test.ts@@ -135,7 +135,7 @@ describe('sessions HTTP agent binding', () => {subject_type: STANDALONE_REQUEST_CONTEXT.subject.type,subject_display_name: STANDALONE_REQUEST_CONTEXT.subject.display_name,});- expect(json.data.metrics).toEqual({ total_cost_in_usd: 0, total_duration_ms: 0, total_turns: 0 });+ expect(json.data.metrics).toEqual({ total_duration_ms: 0, total_turns: 0 });});it('returns 404 when creating a session for an unknown agent name', async () => {