Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(metering): count each turn's token usage from its activity row

Petr Bauch33473297dbed
diff --git a/console/packages/agent-providers/src/host/activity-reporter.test.ts b/console/packages/agent-providers/src/host/activity-reporter.test.ts
index b16d6cb16..431038011 100644
--- a/console/packages/agent-providers/src/host/activity-reporter.test.ts
+++ b/console/packages/agent-providers/src/host/activity-reporter.test.ts
@@ -24,6 +24,7 @@ const origin: Origin = {
messageId: 'sw_asked',
};
const originOf = (turnId: string) => (turnId === 'turn' ? origin : null);
+const noUsage = () => [];
const OCCURRED = '2026-09-24T12:00:00.000Z';
function event(sequence: number, body: ServerEvent['body']): ServerEvent {
@@ -80,7 +81,7 @@ const placed = { room_id: 'room-1', thread_id: 'sw_asked', message_id: 'sw_asked
it('reports the turn itself as a row revised by event sequence, placed where it was asked', async () => {
const lines = await reporter();
- expect(rows(lines.reports(event(4, turn('running')), originOf))).toEqual([
+ expect(rows(lines.reports(event(4, turn('running')), originOf, noUsage))).toEqual([
{
turn_id: 'turn',
item_id: 'turn',
@@ -96,10 +97,33 @@ it('reports the turn itself as a row revised by event sequence, placed where it
]);
});
+it("carries an ended turn's usage on its row, and nothing on a turn still running", async () => {
+ const lines = await reporter();
+ const spent = [
+ { model: 'big', inputTokens: 10, outputTokens: 2, cacheReadTokens: 300, cacheWriteTokens: 0 },
+ ];
+ const usageOf = (turnId: string) => (turnId === 'turn' ? spent : []);
+ expect(rows(lines.reports(event(1, turn('running')), originOf, usageOf))[0]).not.toHaveProperty(
+ 'usage'
+ );
+ expect(rows(lines.reports(event(2, turn('completed')), originOf, usageOf))[0]?.usage).toEqual([
+ {
+ model: 'big',
+ input_tokens: 10,
+ output_tokens: 2,
+ cache_read_tokens: 300,
+ cache_write_tokens: 0,
+ },
+ ]);
+ expect(rows(lines.reports(event(3, turn('error')), originOf, noUsage))[0]).not.toHaveProperty(
+ 'usage'
+ );
+});
+
it('places a turn in its origin thread when it has one', async () => {
const lines = await reporter();
const [row] = rows(
- lines.reports(event(1, turn('queued')), () => ({ ...origin, threadId: 'sw_thread' }))
+ lines.reports(event(1, turn('queued')), () => ({ ...origin, threadId: 'sw_thread' }), noUsage)
);
expect(row).toMatchObject({ room_id: 'room-1', thread_id: 'sw_thread', message_id: 'sw_asked' });
});
@@ -134,7 +158,7 @@ it('reports every message and tool item, streamed parts included, as its own row
text: 'more',
})
),
- ].flatMap((e) => lines.reports(e, originOf))
+ ].flatMap((e) => lines.reports(e, originOf, noUsage))
);
expect(
reported.map((row) => [row.item_id, row.kind, row.revision, row.status, row.title, row.text])
@@ -151,7 +175,11 @@ it('reports every message and tool item, streamed parts included, as its own row
it('truncates long titles and texts', async () => {
const lines = await reporter();
const [row] = rows(
- lines.reports(event(1, item({ title: 't'.repeat(600), text: 'x'.repeat(9000) })), originOf)
+ lines.reports(
+ event(1, item({ title: 't'.repeat(600), text: 'x'.repeat(9000) })),
+ originOf,
+ noUsage
+ )
);
expect(Array.from(row!.title)).toHaveLength(500);
expect(row!.title.endsWith('…')).toBe(true);
@@ -168,7 +196,7 @@ it('reports a notice in the running turn, and in a turn that just ended', async
event(4, turn('error')),
event(5, notice('The turn failed')),
event(6, notice('Model changed')),
- ].flatMap((e) => lines.reports(e, originOf));
+ ].flatMap((e) => lines.reports(e, originOf, noUsage));
const notices = rows(reported).filter((row) => row.kind === 'notice');
expect(notices).toEqual([
{
@@ -190,7 +218,7 @@ it('reports a notice in the running turn, and in a turn that just ended', async
it('knows the running turn after a restart from the events it already reported', async () => {
const lines = await reporter();
lines.catchUp([event(1, turn('running'))]);
- expect(rows(lines.reports(event(2, notice('Still going')), originOf))).toMatchObject([
+ expect(rows(lines.reports(event(2, notice('Still going')), originOf, noUsage))).toMatchObject([
{ turn_id: 'turn', item_id: 'notice:2' },
]);
});
@@ -214,7 +242,8 @@ it('opens an approval with its options and closes it when it settles', async ()
},
},
}),
- originOf
+ originOf,
+ noUsage
);
expect(opened).toEqual([
{
@@ -243,7 +272,8 @@ it('opens an approval with its options and closes it when it settles', async ()
commandId: null,
result: null,
}),
- originOf
+ originOf,
+ noUsage
)
).toEqual([{ kind: 'approval.close', requestId: 'permission' }]);
});
@@ -278,7 +308,8 @@ it('opens questions with every question and its options', async () => {
},
},
}),
- originOf
+ originOf,
+ noUsage
);
expect(opened).toEqual([
{
diff --git a/console/packages/agent-providers/src/host/activity-reporter.ts b/console/packages/agent-providers/src/host/activity-reporter.ts
index 4dc9df0c7..84a8a693d 100644
--- a/console/packages/agent-providers/src/host/activity-reporter.ts
+++ b/console/packages/agent-providers/src/host/activity-reporter.ts
@@ -1,6 +1,7 @@
import { join } from 'node:path';
import type { Origin, ServerEvent } from '@switch-console/shared/session-v1';
import { z } from 'zod';
+import type { TokenUsage } from '../events';
import { Journal } from './journal';
/**
@@ -30,6 +31,19 @@ export type ActivityRow = {
thread_id: string | null;
message_id: string | null;
occurred_at: string;
+ /**
+ * What the turn spent, on its own row once it has ended. Left out when
+ * there is nothing to say, so a server that predates it takes the row.
+ */
+ usage?: UsageRow[];
+};
+
+export type UsageRow = {
+ model: string;
+ input_tokens: number;
+ output_tokens: number;
+ cache_read_tokens: number;
+ cache_write_tokens: number;
};
export type RequestOpening = {
@@ -59,6 +73,8 @@ export type Report =
type Placement = Pick<ActivityRow, 'room_id' | 'thread_id' | 'message_id'>;
+const ENDED = new Set(['completed', 'interrupted', 'error']);
+
const cursorSchema = z.strictObject({ through: z.number().int().nonnegative() });
export class ActivityReporter {
@@ -97,7 +113,11 @@ export class ActivityReporter {
for (const event of events) this.track(event);
}
- reports(event: ServerEvent, originOf: (turnId: string) => Origin | null): Report[] {
+ reports(
+ event: ServerEvent,
+ originOf: (turnId: string) => Origin | null,
+ usageOf: (turnId: string) => TokenUsage[]
+ ): Report[] {
const body = event.body;
const place = (turnId: string): Placement => {
const origin = originOf(turnId);
@@ -110,7 +130,8 @@ export class ActivityReporter {
const noticeTurn = body.type === 'notice' ? this.noticeTurn(event.sequence) : null;
this.track(event);
switch (body.type) {
- case 'turn.upsert':
+ case 'turn.upsert': {
+ const usage = ENDED.has(body.status) ? usageOf(body.turnId) : [];
return [
{
kind: 'activity',
@@ -125,9 +146,11 @@ export class ActivityReporter {
command_id: body.commandId,
...place(body.turnId),
occurred_at: event.occurredAt,
+ ...(usage.length > 0 ? { usage: usage.map(usageRow) } : {}),
},
},
];
+ }
case 'item.upsert': {
const { item } = body;
return [
@@ -247,6 +270,16 @@ export class ActivityReporter {
}
}
+function usageRow(usage: TokenUsage): UsageRow {
+ return {
+ model: usage.model,
+ input_tokens: usage.inputTokens,
+ output_tokens: usage.outputTokens,
+ cache_read_tokens: usage.cacheReadTokens,
+ cache_write_tokens: usage.cacheWriteTokens,
+ };
+}
+
function truncate(text: string, limit: number): string {
const characters = Array.from(text);
return characters.length <= limit ? text : `${characters.slice(0, limit - 1).join('')}…`;
diff --git a/console/packages/agent-providers/src/host/bench/adapter.ts b/console/packages/agent-providers/src/host/bench/adapter.ts
index 30df970b1..6e9f826eb 100644
--- a/console/packages/agent-providers/src/host/bench/adapter.ts
+++ b/console/packages/agent-providers/src/host/bench/adapter.ts
@@ -292,6 +292,7 @@ export function createBenchAdapter(): ProviderAdapter {
type: 'turn.completed',
turnId: input.turnId,
outcome,
+ usage: [],
...(message ? { message } : {}),
});
})();
diff --git a/console/packages/agent-providers/src/host/session-host.test.ts b/console/packages/agent-providers/src/host/session-host.test.ts
index 4eed54de8..6ec6d8e6f 100644
--- a/console/packages/agent-providers/src/host/session-host.test.ts
+++ b/console/packages/agent-providers/src/host/session-host.test.ts
@@ -326,7 +326,7 @@ it('stops active and queued turns without dispatching the queue during cleanup',
await host.command(message('queued'));
await vi.waitFor(() => expect(adapter.sendTurn).toHaveBeenCalledOnce());
vi.mocked(adapter.stopSession).mockImplementationOnce(async () => {
- emit({ type: 'turn.completed', turnId: 'active', outcome: 'interrupted' });
+ emit({ type: 'turn.completed', turnId: 'active', outcome: 'interrupted', usage: [] });
emit({ type: 'session.exited', reason: 'Stopped' });
});
expect((await host.command({ ...message('stop'), body: { type: 'session.stop' } })).status).toBe(
@@ -337,11 +337,33 @@ it('stops active and queued turns without dispatching the queue during cleanup',
expect(adapter.sendTurn).toHaveBeenCalledOnce();
});
+it('knows what a turn spent from the moment it reads as ended, and after a restart', async () => {
+ const { host, emit, root } = await start('claude');
+ const spent = [
+ { model: 'big', inputTokens: 10, outputTokens: 2, cacheReadTokens: 300, cacheWriteTokens: 0 },
+ ];
+ await host.command(message('spender'));
+ await vi.waitFor(() => expect(host.snapshot().turns[0]?.status).toBe('running'));
+ const seenWhenEnded: unknown[] = [];
+ host.onPublished((event) => {
+ if (event.body.type === 'turn.upsert' && event.body.status === 'completed')
+ seenWhenEnded.push(host.usageOf(event.body.turnId));
+ });
+ emit({ type: 'turn.completed', turnId: 'spender', outcome: 'completed', usage: spent });
+ await vi.waitFor(() => expect(seenWhenEnded).toEqual([spent]));
+ await host.shutdown();
+ const next = setup('claude');
+ const recovered = await HostedSession.start(root, next.config, next.adapter);
+ hosts.push(recovered);
+ expect(recovered.usageOf('spender')).toEqual(spent);
+ expect(recovered.usageOf('never-ran')).toEqual([]);
+});
+
it('resets into a new epoch and native conversation while retaining history', async () => {
const { host, adapter, emit, root } = await start('claude');
await host.command(message('old-turn'));
await vi.waitFor(() => expect(host.snapshot().turns[0]?.status).toBe('running'));
- emit({ type: 'turn.completed', turnId: 'old-turn', outcome: 'completed' });
+ emit({ type: 'turn.completed', turnId: 'old-turn', outcome: 'completed', usage: [] });
await vi.waitFor(() => expect(host.snapshot().turns[0]?.status).toBe('completed'));
vi.mocked(adapter.startSession).mockImplementationOnce(async () => {
emit({ type: 'session.state.changed', status: 'ready' });
diff --git a/console/packages/agent-providers/src/host/session-host.ts b/console/packages/agent-providers/src/host/session-host.ts
index f05d12fd5..43db3526c 100644
--- a/console/packages/agent-providers/src/host/session-host.ts
+++ b/console/packages/agent-providers/src/host/session-host.ts
@@ -21,7 +21,7 @@ import type {
import { z } from 'zod';
import { ProviderConversationUnavailableError } from '../adapter';
import type { ProviderAdapter, ProviderSessionStartInput, TurnAttachment } from '../adapter';
-import type { UserInputAnswers } from '../events';
+import type { TokenUsage, UserInputAnswers } from '../events';
import type { ProviderRuntimeEvent } from '../events';
import { ChatProjector } from '../session-v1/chat-projector';
import { ATTACHMENT_MIME_TYPES } from './attachments';
@@ -30,7 +30,22 @@ import { Journal } from './journal';
const recordSchema = z.discriminatedUnion('type', [
z.object({ type: z.literal('accepted'), command: commandSchema }),
z.object({ type: z.literal('dispatched'), commandId: z.string() }),
- z.object({ type: z.literal('finished'), commandId: z.string() }),
+ z.object({
+ type: z.literal('finished'),
+ commandId: z.string(),
+ // What the turn spent; absent from a host that predates it, or when the provider said nothing.
+ usage: z
+ .array(
+ z.object({
+ model: z.string(),
+ inputTokens: z.number().int().nonnegative(),
+ outputTokens: z.number().int().nonnegative(),
+ cacheReadTokens: z.number().int().nonnegative(),
+ cacheWriteTokens: z.number().int().nonnegative(),
+ })
+ )
+ .optional(),
+ }),
z.object({ type: z.literal('native'), nativeSessionId: z.string() }),
z.object({ type: z.literal('stopped') }),
z.object({ type: z.literal('resumed'), operationId: z.string() }),
@@ -70,6 +85,7 @@ export class HostedSession {
private readonly commands = new Map<string, Command>();
private readonly dispatched = new Set<string>();
private readonly finished = new Set<string>();
+ private readonly usage = new Map<string, TokenUsage[]>();
private readonly resumeOperations = new Set<string>();
private readonly queue: Command[] = [];
private readonly questions = new Map<string, PendingQuestion>();
@@ -125,7 +141,10 @@ export class HostedSession {
for (const record of inbox.records) {
if (record.type === 'accepted') this.commands.set(record.command.commandId, record.command);
if (record.type === 'dispatched') this.dispatched.add(record.commandId);
- if (record.type === 'finished') this.finished.add(record.commandId);
+ if (record.type === 'finished') {
+ this.finished.add(record.commandId);
+ if (record.usage) this.usage.set(record.commandId, record.usage);
+ }
if (record.type === 'native') this.nativeId = record.nativeSessionId;
if (record.type === 'stopped') this.stopped = true;
if (record.type === 'resumed') {
@@ -396,6 +415,11 @@ export class HostedSession {
return () => this.listeners.delete(listener);
}
+ /** What `turnId` spent, as its provider reported when it ended. */
+ usageOf(turnId: string): TokenUsage[] {
+ return this.usage.get(turnId) ?? [];
+ }
+
/** Where the command that started `turnId` came from, if a command started it. */
originOf(turnId: string): Origin | null {
return this.commands.get(turnId)?.origin ?? null;
@@ -942,10 +966,19 @@ export class HostedSession {
event.item.type === 'user_message'
)
return;
- await this.publishAll(this.projector.ingest(event, Date.now()));
+ // A turn is finished, with what it spent, before anyone can read that it
+ // ended: its report to Switch is the one chance to count the spend.
if (event.type === 'turn.completed') {
- await this.inbox.append({ type: 'finished', commandId: event.turnId });
+ await this.inbox.append({
+ type: 'finished',
+ commandId: event.turnId,
+ ...(event.usage.length > 0 ? { usage: event.usage } : {}),
+ });
this.finished.add(event.turnId);
+ if (event.usage.length > 0) this.usage.set(event.turnId, event.usage);
+ }
+ await this.publishAll(this.projector.ingest(event, Date.now()));
+ if (event.type === 'turn.completed') {
this.activeTurn = null;
void this.runNext().catch((error: unknown) => this.fail(error));
}
diff --git a/console/packages/agent-providers/src/host/shared-host.test.ts b/console/packages/agent-providers/src/host/shared-host.test.ts
index 60a0d324c..f860922c5 100644
--- a/console/packages/agent-providers/src/host/shared-host.test.ts
+++ b/console/packages/agent-providers/src/host/shared-host.test.ts
@@ -668,6 +668,58 @@ it('keeps running room messages while Switch cannot take its reports', async ()
}
});
+it('reports what an ended turn spent on its row, and the row alone to a server that predates it', async () => {
+ const host = await start({ rooms: true });
+ const spent = [
+ { model: 'big', inputTokens: 10, outputTokens: 2, cacheReadTokens: 300, cacheWriteTokens: 0 },
+ ];
+ const ended = () =>
+ host.switchCore.calls.filter(
+ (c) =>
+ c.path.endsWith('/activity') &&
+ (c.body as { kind: string; status: string }).kind === 'turn' &&
+ (c.body as { status: string }).status === 'completed'
+ );
+ try {
+ await host.parent.ask({ type: 'room', handoff: roomMessage(1, 'First') });
+ await vi.waitFor(() => expect(host.turns).toHaveLength(1), { timeout: 5000 });
+ host.emit({
+ type: 'turn.completed',
+ turnId: host.turns[0]!.turnId,
+ outcome: 'completed',
+ usage: spent,
+ });
+ await vi.waitFor(() => expect(ended()).toHaveLength(1), { timeout: 5000 });
+ expect(ended()[0]!.body).toMatchObject({
+ usage: [
+ {
+ model: 'big',
+ input_tokens: 10,
+ output_tokens: 2,
+ cache_read_tokens: 300,
+ cache_write_tokens: 0,
+ },
+ ],
+ });
+
+ host.switchCore.state.refusesUsage = true;
+ await host.parent.ask({ type: 'room', handoff: roomMessage(2, 'Second') });
+ await vi.waitFor(() => expect(host.turns).toHaveLength(2), { timeout: 5000 });
+ host.emit({
+ type: 'turn.completed',
+ turnId: host.turns[1]!.turnId,
+ outcome: 'completed',
+ usage: spent,
+ });
+ await vi.waitFor(() => expect(ended()).toHaveLength(3), { timeout: 5000 });
+ expect(ended()[1]!.body).toHaveProperty('usage');
+ expect(ended()[2]!.body).not.toHaveProperty('usage');
+ expect(ended()[2]!.body).toMatchObject({ turn_id: host.turns[1]!.turnId });
+ } finally {
+ await host.stop();
+ }
+}, 20000);
+
it('tells the session to rejoin its room once a reset asked for there has applied', async () => {
const host = await start({ resettable: true });
try {
diff --git a/console/packages/agent-providers/src/host/shared-host.ts b/console/packages/agent-providers/src/host/shared-host.ts
index 7bd860543..bfd229a84 100644
--- a/console/packages/agent-providers/src/host/shared-host.ts
+++ b/console/packages/agent-providers/src/host/shared-host.ts
@@ -314,17 +314,34 @@ export async function runSharedHost(
null
);
};
+ // A server that predates usage refuses a row that carries it. The row is
+ // what platforms draw, so it goes again without, and the spend is not counted.
+ const sentWithoutUsage = async (item: Report, error: RequestError): Promise<boolean> => {
+ if (item.kind !== 'activity' || !item.row.usage || error.status !== 422) return false;
+ const { usage: _usage, ...row } = item.row;
+ console.warn(
+ `Switch refused the usage of turn ${row.turn_id}, so it is not counted against the workspace: ${error.message}`
+ );
+ await send({ kind: 'activity', row });
+ return true;
+ };
const report = async () => {
if (reportingUnsupported) return;
const { events } = host!.replay(reporter.cursor);
if (!events.length) return;
for (const event of events) {
- for (const item of reporter.reports(event, (turnId) => host!.originOf(turnId))) {
+ const reports = reporter.reports(
+ event,
+ (turnId) => host!.originOf(turnId),
+ (turnId) => host!.usageOf(turnId)
+ );
+ for (const item of reports) {
try {
await send(item);
} catch (error) {
if (!(error instanceof RequestError)) throw error;
if (unsupported(error)) return;
+ if (await sentWithoutUsage(item, error)) continue;
// A request Switch refused, or one opened before this session
// first reported here, has nothing to close.
if (item.kind === 'approval.close' && error.code === 'NOT_FOUND') continue;
diff --git a/console/packages/agent-providers/src/testing/agent-sessions-server.ts b/console/packages/agent-providers/src/testing/agent-sessions-server.ts
index 85497904a..19bd8c590 100644
--- a/console/packages/agent-providers/src/testing/agent-sessions-server.ts
+++ b/console/packages/agent-providers/src/testing/agent-sessions-server.ts
@@ -11,11 +11,12 @@ export type AgentSessionsCall = { method: string; path: string; body: unknown };
* test) is what `GET /agent-sessions/approvals/outcomes` returns. `server`
* never sees those calls, so a test counting its own requests is unaffected.
* Setting `state.unavailable` answers every such call with 503, as an
- * unreachable Switch would.
+ * unreachable Switch would, and `state.refusesUsage` refuses an activity row
+ * carrying usage with 422, as a server that predates it does.
*/
export function stubSwitchFetch(server: Fetch, outcomes: unknown[] = []) {
const calls: AgentSessionsCall[] = [];
- const state = { outcomes, unavailable: false };
+ const state = { outcomes, unavailable: false, refusesUsage: false };
vi.stubGlobal('fetch', async (url: string, options: RequestInit = {}) => {
const path = new URL(url).pathname;
if (!path.includes('/agent-sessions/')) return server(url, options);
@@ -30,7 +31,12 @@ export function stubSwitchFetch(server: Fetch, outcomes: unknown[] = []) {
const listed = state.outcomes;
return Response.json(listed);
}
- if (path.endsWith('/activity')) return Response.json({ recorded: true });
+ if (path.endsWith('/activity')) {
+ const body = calls.at(-1)!.body as Record<string, unknown>;
+ if (state.refusesUsage && 'usage' in body)
+ return Response.json({ detail: [{ type: 'extra_forbidden' }] }, { status: 422 });
+ return Response.json({ recorded: true });
+ }
return Response.json({});
});
return { calls, state };
diff --git a/core/switch_core/bridges/agent/api/activity_routes.py b/core/switch_core/bridges/agent/api/activity_routes.py
index eac8b1d3d..0be7a3567 100644
--- a/core/switch_core/bridges/agent/api/activity_routes.py
+++ b/core/switch_core/bridges/agent/api/activity_routes.py
@@ -18,16 +18,20 @@ from switch_core.bridges.agent.dependencies import get_session_factory
from switch_core.db.models import Agent, ApprovalRequest
from switch_core.session_activity.service import (
MAX_DETAIL_CHARS,
+ MAX_MODEL_CHARS,
+ MAX_MODELS_PER_TURN,
MAX_OPTIONS,
MAX_QUESTION_OPTIONS,
MAX_QUESTIONS,
MAX_TEXT_CHARS,
MAX_TITLE_CHARS,
+ MAX_TOKENS,
ApprovalOption,
Decision,
Question,
QuestionOption,
SessionActivityService,
+ TokenSpend,
)
router = APIRouter(prefix="/agent-sessions")
@@ -37,6 +41,15 @@ AuthenticatedAgent = Annotated[Agent, Depends(get_agent_from_scope)]
_Id = Annotated[str, Field(min_length=1, max_length=200)]
+class TokenUsageIn(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+ model: str = Field(max_length=MAX_MODEL_CHARS)
+ input_tokens: int = Field(ge=0, le=MAX_TOKENS)
+ output_tokens: int = Field(ge=0, le=MAX_TOKENS)
+ cache_read_tokens: int = Field(ge=0, le=MAX_TOKENS)
+ cache_write_tokens: int = Field(ge=0, le=MAX_TOKENS)
+
+
class ActivityReport(BaseModel):
model_config = ConfigDict(extra="forbid")
turn_id: _Id
@@ -53,6 +66,10 @@ class ActivityReport(BaseModel):
thread_id: _Id | None
message_id: _Id | None
occurred_at: datetime
+ # Hosts that predate usage reporting do not send it.
+ usage: list[TokenUsageIn] = Field(
+ default_factory=list, max_length=MAX_MODELS_PER_TURN
+ )
class OptionIn(BaseModel):
@@ -152,6 +169,16 @@ async def report_activity(
thread_id=body.thread_id,
message_id=body.message_id,
occurred_at=body.occurred_at,
+ usage=[
+ TokenSpend(
+ model=u.model,
+ input_tokens=u.input_tokens,
+ output_tokens=u.output_tokens,
+ cache_read_tokens=u.cache_read_tokens,
+ cache_write_tokens=u.cache_write_tokens,
+ )
+ for u in body.usage
+ ],
)
return ActivityReceipt(recorded=recorded)
diff --git a/core/switch_core/session_activity/service.py b/core/switch_core/session_activity/service.py
index d52e4f49f..6d5362891 100644
--- a/core/switch_core/session_activity/service.py
+++ b/core/switch_core/session_activity/service.py
@@ -43,6 +43,7 @@ from switch_core.db.stores.session_activity_store import (
from switch_core.db.stores.usage_store import UsageStore
from switch_core.delivery.addressing import AddressingResolver
from switch_core.sessions.contract import (
+ TURN_ENDED,
ApprovalResult,
QuestionsResult,
RequestResult,
@@ -56,6 +57,10 @@ MAX_OPTIONS = 10
MAX_QUESTIONS = 50
MAX_QUESTION_OPTIONS = 50
MAX_CUSTOM_ANSWER_CHARS = 4000
+MAX_MODEL_CHARS = 200
+MAX_MODELS_PER_TURN = 50
+# A host counts in JavaScript numbers, exact only up to 2**53 - 1.
+MAX_TOKENS = 2**53 - 1
# What the host sends inside a question is not truncated by the host, so it is
# cut here to what any platform can show rather than refused.
_QUESTION_TITLE_CHARS = 500
@@ -80,6 +85,18 @@ DECISIONS: tuple[Decision, ...] = ("accept", "acceptForSession", "decline", "can
RequestKind = Literal["approval", "questions"]
+@dataclass(frozen=True)
+class TokenSpend:
+ """Tokens one model spent in a turn. `model` is empty when the provider
+ ran its default without naming it."""
+
+ model: str
+ input_tokens: int
+ output_tokens: int
+ cache_read_tokens: int
+ cache_write_tokens: int
+
+
@dataclass(frozen=True)
class ApprovalOption:
id: str
@@ -168,8 +185,13 @@ class SessionActivityService:
thread_id: str | None,
message_id: str | None,
occurred_at: datetime,
+ usage: list[TokenSpend],
) -> bool:
- """Record one step of a turn. False when a revision at least as new is stored."""
+ """Record one step of a turn. False when a revision at least as new is stored.
+
+ `usage` is what the turn spent, and only a turn's own row, reported as
+ ended, carries it.
+ """
if kind not in SESSION_ACTIVITY_KINDS:
raise SessionError("INVALID_EVENT", f"Unknown activity kind: {kind}")
if status not in ITEM_STATUSES[kind]:
@@ -191,6 +213,10 @@ class SessionActivityService:
)
if revision < 0:
raise SessionError("INVALID_EVENT", "A revision is never negative.")
+ if usage and not (kind == "turn" and status in TURN_ENDED):
+ raise SessionError(
+ "INVALID_EVENT", "Only an ended turn's own row carries its usage."
+ )
if len(title) > MAX_TITLE_CHARS or len(text) > MAX_TEXT_CHARS:
raise SessionError(
"INVALID_EVENT",
@@ -201,11 +227,12 @@ class SessionActivityService:
async with tenant_session(self._sessions, tenant_id) as db, db.begin():
if room_id is not None:
await self._require_member(db, agent_id, room_id)
- first_report = kind == "turn" and (
+ prior_status = (
await self._activity.status_for_update(
db, agent_id, session_id, turn_id, item_id
)
- is None
+ if kind == "turn"
+ else None
)
moved = await self._activity.upsert(
db,
@@ -227,8 +254,11 @@ class SessionActivityService:
occurred_at=occurred_at,
),
)
- if moved and first_report:
- await self._meter_turn(db, tenant_id, agent_id)
+ if moved and kind == "turn":
+ if prior_status is None:
+ await self._meter_turn(db, tenant_id, agent_id)
+ if status in TURN_ENDED and prior_status not in TURN_ENDED:
+ await self._meter_tokens(db, tenant_id, agent_id, usage)
return moved
async def _meter_turn(
@@ -239,25 +269,64 @@ class SessionActivityService:
A turn has been paid for once it exists: one that errors or is
interrupted still spent the model's time, so counting only the ones
that complete would under-report exactly the runaway loops a budget is
- for. Charged to the agent's client, the identity every other metric is
- counted against.
+ for.
"""
- client_id = await db.scalar(
- select(Agent.client_id).where(
- Agent.tenant_id == tenant_id, Agent.id == agent_id
- )
- )
- if client_id is None:
- raise SessionError("NOT_FOUND", f"Agent {agent_id} does not exist.")
await self._usage.record(
db,
tenant_id=tenant_id,
metric=UsageMetric.TURNS,
- client_id=client_id,
+ client_id=await self._agent_client_id(db, tenant_id, agent_id),
model="",
amount=1,
)
+ async def _meter_tokens(
+ self,
+ db: AsyncSession,
+ tenant_id: str,
+ agent_id: str,
+ usage: list[TokenSpend],
+ ) -> None:
+ """Count what a turn spent, once: when its row first reports it ended.
+
+ The counts are what the host says: a customer's own agent reports on
+ itself, which is enough for a budget the customer sets for itself.
+ """
+ if not usage:
+ return
+ client_id = await self._agent_client_id(db, tenant_id, agent_id)
+ # One agent's turns can end concurrently; upserting the counters in the
+ # same order in each keeps two of them from deadlocking.
+ for spend in sorted(usage, key=lambda s: s.model):
+ for metric, amount in (
+ (UsageMetric.INPUT_TOKENS, spend.input_tokens),
+ (UsageMetric.OUTPUT_TOKENS, spend.output_tokens),
+ (UsageMetric.CACHE_READ_TOKENS, spend.cache_read_tokens),
+ (UsageMetric.CACHE_WRITE_TOKENS, spend.cache_write_tokens),
+ ):
+ if amount > 0:
+ await self._usage.record(
+ db,
+ tenant_id=tenant_id,
+ metric=metric,
+ client_id=client_id,
+ model=spend.model,
+ amount=amount,
+ )
+
+ @staticmethod
+ async def _agent_client_id(db: AsyncSession, tenant_id: str, agent_id: str) -> str:
+ """The client an agent's usage is charged to, the identity every other
+ metric is counted against."""
+ client_id = await db.scalar(
+ select(Agent.client_id).where(
+ Agent.tenant_id == tenant_id, Agent.id == agent_id
+ )
+ )
+ if client_id is None:
+ raise SessionError("NOT_FOUND", f"Agent {agent_id} does not exist.")
+ return client_id
+
async def turn_items(
self, agent_id: str, session_id: str, turn_id: str
) -> list[SessionActivityItem]:
diff --git a/core/tests/switch_core/session_activity/test_bridge_publisher.py b/core/tests/switch_core/session_activity/test_bridge_publisher.py
index edef651f9..c0bf3f5fb 100644
--- a/core/tests/switch_core/session_activity/test_bridge_publisher.py
+++ b/core/tests/switch_core/session_activity/test_bridge_publisher.py
@@ -177,6 +177,7 @@ async def _step(
thread_id=ASKED,
message_id=ASKED,
occurred_at=datetime.now(UTC),
+ usage=[],
)
diff --git a/core/tests/switch_core/session_activity/test_isolation.py b/core/tests/switch_core/session_activity/test_isolation.py
index c1b8e08ed..71c486d13 100644
--- a/core/tests/switch_core/session_activity/test_isolation.py
+++ b/core/tests/switch_core/session_activity/test_isolation.py
@@ -83,6 +83,7 @@ async def test_each_tenant_sees_only_its_own_activity_and_requests(rls_harness):
thread_id=None,
message_id=None,
occurred_at=datetime.now(UTC),
+ usage=[],
)
await service.open_approval(
tenant,
diff --git a/core/tests/switch_core/session_activity/test_maintenance.py b/core/tests/switch_core/session_activity/test_maintenance.py
index ba67175fb..40c5c09a9 100644
--- a/core/tests/switch_core/session_activity/test_maintenance.py
+++ b/core/tests/switch_core/session_activity/test_maintenance.py
@@ -50,6 +50,7 @@ async def test_a_pass_expires_overdue_requests_and_prunes_only_when_asked(
thread_id=None,
message_id=None,
occurred_at=datetime.now(UTC),
+ usage=[],
)
past = datetime.now(UTC) - timedelta(days=30)
await _backdate(session_factory, ApprovalRequest, expires_at=past)
diff --git a/core/tests/switch_core/session_activity/test_routes.py b/core/tests/switch_core/session_activity/test_routes.py
index ad33f217e..e07e0bf35 100644
--- a/core/tests/switch_core/session_activity/test_routes.py
+++ b/core/tests/switch_core/session_activity/test_routes.py
@@ -10,7 +10,7 @@ from sqlalchemy import select
from switch_core.bridges.agent.api.activity_routes import router
from switch_core.bridges.agent.auth import get_agent_from_scope
from switch_core.bridges.agent.dependencies import get_session_factory
-from switch_core.db.models import Agent, ApprovalRequest
+from switch_core.db.models import Agent, ApprovalRequest, TenantUsage
from switch_core.session_activity.service import ApprovalOption, PlatformPerson
from switch_core.sessions.contract import Answer, QuestionsResult
from switch_core.sessions.errors import SessionError
@@ -110,6 +110,58 @@ async def test_malformed_activity_is_a_422(client, overrides):
assert response.status_code == 422
+_USAGE = {
+ "model": "big",
+ "input_tokens": 10,
+ "output_tokens": 2,
+ "cache_read_tokens": 300,
+ "cache_write_tokens": 0,
+}
+
+
+def _ended_turn(**overrides):
+ return _activity(
+ **{
+ "item_id": "turn",
+ "kind": "turn",
+ "status": "completed",
+ "title": "",
+ "text": "",
+ **overrides,
+ }
+ )
+
+
+async def test_an_ended_turn_row_carries_what_the_turn_spent(client, session_factory):
+ response = await client.post(
+ f"/agent-sessions/{SESSION}/activity", json=_ended_turn(usage=[_USAGE])
+ )
+ assert response.json() == {"recorded": True}
+ async with session_factory() as db:
+ rows = (await db.execute(select(TenantUsage.metric, TenantUsage.amount))).all()
+ assert sorted(rows) == [
+ ("cache_read_tokens", 300),
+ ("input_tokens", 10),
+ ("output_tokens", 2),
+ ("turns", 1),
+ ]
+
+
+ "body",
+ [
+ _activity(usage=[_USAGE]),
+ _ended_turn(status="running", usage=[_USAGE]),
+ _ended_turn(usage=[{**_USAGE, "input_tokens": -1}]),
+ _ended_turn(usage=[{**_USAGE, "input_tokens": 2**53}]),
+ _ended_turn(usage=[{**_USAGE, "reasoning_tokens": 1}]),
+ ],
+)
+async def test_usage_anywhere_but_an_ended_turn_or_malformed_is_a_422(client, body):
+ response = await client.post(f"/agent-sessions/{SESSION}/activity", json=body)
+ assert response.status_code == 422
+
+
async def test_a_questions_request_over_http(client, service, people):
opened = await client.post(
f"/agent-sessions/{SESSION}/approvals",
diff --git a/core/tests/switch_core/session_activity/test_service.py b/core/tests/switch_core/session_activity/test_service.py
index 763703e09..7085888b7 100644
--- a/core/tests/switch_core/session_activity/test_service.py
+++ b/core/tests/switch_core/session_activity/test_service.py
@@ -13,6 +13,7 @@ from switch_core.session_activity.service import (
Question,
QuestionOption,
SwitchUser,
+ TokenSpend,
)
from switch_core.sessions.contract import Answer, QuestionsResult
from switch_core.sessions.errors import SessionError
@@ -64,6 +65,7 @@ def _step(**overrides):
thread_id=None,
message_id=None,
occurred_at=datetime(2026, 9, 23, 12, 0, tzinfo=UTC),
+ usage=[],
)
values.update(overrides)
return values
@@ -118,7 +120,7 @@ async def test_a_step_is_upserted_by_revision_and_announced_by_turn(service, cha
assert all(change.row is None for change in announced)
-def _turn(turn_id: str, revision: int, status: str):
+def _turn(turn_id: str, revision: int, status: str, usage=()):
return _step(
turn_id=turn_id,
item_id="turn",
@@ -127,6 +129,7 @@ def _turn(turn_id: str, revision: int, status: str):
status=status,
title="",
text="",
+ usage=list(usage),
)
@@ -151,6 +154,52 @@ async def test_a_turn_is_counted_once_when_first_reported(service, session_facto
assert await _turns_counted(session_factory) == 2
+def _spend(model: str, input_tokens: int) -> TokenSpend:
+ return TokenSpend(
+ model=model,
+ input_tokens=input_tokens,
+ output_tokens=1,
+ cache_read_tokens=0,
+ cache_write_tokens=0,
+ )
+
+
+async def _tokens_counted(session_factory) -> list[tuple[str, str, int]]:
+ async with session_factory() as db:
+ rows = await db.execute(
+ select(TenantUsage.metric, TenantUsage.model, TenantUsage.amount).where(
+ TenantUsage.metric != "turns"
+ )
+ )
+ return sorted(tuple(row) for row in rows)
+
+
+async def test_a_turns_tokens_are_counted_once_when_it_first_ends(
+ service, session_factory
+):
+ spent = [_spend("small", 7), _spend("big", 10)]
+ await service.report_item(AGENT, SESSION, **_turn("turn-1", 1, "running"))
+ await service.report_item(AGENT, SESSION, **_turn("turn-1", 2, "completed", spent))
+ # A retried report, and a later revision of an ended turn, spend nothing new.
+ await service.report_item(AGENT, SESSION, **_turn("turn-1", 2, "completed", spent))
+ await service.report_item(
+ AGENT, SESSION, **_turn("turn-1", 3, "interrupted", spent)
+ )
+ assert await _tokens_counted(session_factory) == [
+ ("input_tokens", "big", 10),
+ ("input_tokens", "small", 7),
+ ("output_tokens", "big", 1),
+ ("output_tokens", "small", 1),
+ ]
+
+
+async def test_usage_on_a_turn_that_has_not_ended_is_refused(service):
+ with pytest.raises(SessionError, match="ended turn"):
+ await service.report_item(
+ AGENT, SESSION, **_turn("turn-1", 1, "running", [_spend("big", 1)])
+ )
+
+
async def test_a_turns_steps_are_read_in_the_order_first_reported(service):
for item_id, kind, status in [
("turn", "turn", "running"),