Multiplayer agent harness for work.
import { canonicalPerson } from "../../../directory/person.ts";
import { parseScopeId, personalScope, type Cron, type ScopeId } from "../../../types.ts";
import type { SessionOrigin, SessionStore, SpendRow } from "../../../sessions/session-store.ts";
import { constantTimeEqual } from "../../../util/crypto.ts";
import { contentDispositionAttachment, contentTypeWithUtf8Charset, sendJson } from "../../http.ts";
import { audit, authorizeAdmin, orgScope } from "../shared.ts";
import { discoverScopes } from "./common.ts";
import { type ApiCtx, type BaseCtx } from "../route.ts";
export function cacheHitRatio(s: { cacheRead?: number; cacheWrite?: number; uncachedInput?: number }): number | null {
const read = s.cacheRead;
const write = s.cacheWrite;
const uncached = s.uncachedInput;
if (read === undefined && write === undefined && uncached === undefined) return null;
const r = read ?? 0;
const denom = r + (write ?? 0) + (uncached ?? 0);
if (denom <= 0) return null;
return r / denom;
}
const DAY_MS = 86_400_000;
const DEFAULT_WINDOW_DAYS = 30;
const EXPORT_MAX_DAYS = 93;
const CRON_LOOKUP_CONCURRENCY = 8;
const SPEND_EXPORT_PRINCIPAL = "spend-export";
const DATE_ONLY = /^\d{4}-\d{2}-\d{2}$/;
const EPOCH_MS = /^\d+$/;
const CSV_FORMULA_LEAD = /^[=+\-@\t\r]/;
const CSV_NEEDS_QUOTES = /["\r\n,]/;
const BACKGROUND_SOURCES = ["cron", "loop", "swarm", "webhook", "monitor"] as const satisfies readonly Exclude<
SessionOrigin,
"conversation"
>[];
type BackgroundSource = (typeof BACKGROUND_SOURCES)[number];
type SpendSource = "live" | BackgroundSource;
const SPEND_SOURCES: readonly SpendSource[] = ["live", ...BACKGROUND_SOURCES];
const CSV_TOTALS_HEADER = [
"live_usd",
"background_usd",
...BACKGROUND_SOURCES.map((source) => `${source}_usd`),
"total_usd",
"calls",
"input_tokens",
"output_tokens",
"cache_read_tokens",
"cache_write_tokens",
"cache_hit_ratio",
];
const SPEND_BREAKDOWNS = ["entity", "model", "cron"] as const;
type SpendBreakdown = (typeof SPEND_BREAKDOWNS)[number];
type SpendBucket = "day" | "week";
interface SpendBucketTotals {
costUsd: number;
calls: number;
tokens: number;
}
interface SpendTotals {
costUsd: number;
calls: number;
tokens: number;
input: number;
output: number;
cacheRead: number;
cacheWrite: number;
cacheHitRatio: number | null;
live: SpendBucketTotals;
background: SpendBucketTotals & { sources: Record<BackgroundSource, SpendBucketTotals> };
}
interface SpendEntity extends SpendTotals {
principalId: string | null;
scopeId: ScopeId;
kind: string;
displayName: string;
}
type SpendExportCron = Pick<Cron, "id" | "title" | "owner" | "ownerScopeId" | "schedule" | "enabled" | "archived">;
type SpendCronInfo = SpendExportCron & Pick<Cron, "action" | "message">;
interface SpendCron extends SpendTotals {
cronId: string;
scopeId: ScopeId;
scopeName: string;
info: SpendCronInfo | null;
}
interface SpendSeriesPoint {
day: string;
costUsd: number;
calls: number;
models: { model: string | null; costUsd: number }[];
people: { principalId: string | null; costUsd: number }[];
live: { costUsd: number };
background: { costUsd: number; sources: Record<BackgroundSource, { costUsd: number }> };
}
export interface SpendReport {
window: { from: string; to: string; bucket: SpendBucket };
org: SpendTotals;
series: SpendSeriesPoint[];
people: SpendEntity[];
scopes: SpendEntity[];
crons: SpendCron[];
models: ({ model: string | null } & SpendTotals)[];
}
export interface SpendExportRow extends Tally {
day: string;
scopeId: ScopeId;
kind: string;
principalId: string | null;
displayName: string;
source: SpendSource;
cronId: string | null;
model: string | null;
}
export interface SpendSummaryOptions {
from: number;
to: number;
bucket: SpendBucket;
label: (scopeId: ScopeId) => string;
cron: (cronId: string) => SpendCronInfo | null;
}
interface Tally {
costUsd: number;
calls: number;
input: number;
output: number;
cacheRead: number;
cacheWrite: number;
}
interface EntityAccumulator {
principalId: string | null;
scopeId: ScopeId;
kind: string;
sources: ScopeId[];
bySource: Record<SpendSource, Tally>;
}
function emptyTally(): Tally {
return { costUsd: 0, calls: 0, input: 0, output: 0, cacheRead: 0, cacheWrite: 0 };
}
function emptySources(): Record<SpendSource, Tally> {
return Object.fromEntries(SPEND_SOURCES.map((source) => [source, emptyTally()])) as Record<SpendSource, Tally>;
}
function backgroundSources<T>(value: (source: BackgroundSource) => T): Record<BackgroundSource, T> {
return Object.fromEntries(BACKGROUND_SOURCES.map((source) => [source, value(source)])) as Record<BackgroundSource, T>;
}
function backgroundTally(bySource: Record<SpendSource, Tally>): Tally {
const total = emptyTally();
for (const source of BACKGROUND_SOURCES) addTally(total, bySource[source]);
return total;
}
function addRow(t: Tally, r: SpendRow): void {
t.costUsd += r.costUsd;
t.calls += r.calls;
t.input += r.input;
t.output += r.output;
t.cacheRead += r.cacheRead;
t.cacheWrite += r.cacheWrite;
}
function addTally(t: Tally, other: Tally): void {
t.costUsd += other.costUsd;
t.calls += other.calls;
t.input += other.input;
t.output += other.output;
t.cacheRead += other.cacheRead;
t.cacheWrite += other.cacheWrite;
}
function tokensOf(t: Tally): number {
return t.input + t.output + t.cacheRead + t.cacheWrite;
}
function bucketTotals(t: Tally): SpendBucketTotals {
return { costUsd: t.costUsd, calls: t.calls, tokens: tokensOf(t) };
}
function spendSource(origin: SessionOrigin): SpendSource {
return origin === "conversation" ? "live" : origin;
}
function isoDay(day: number): string {
return new Date(day * DAY_MS).toISOString().slice(0, 10);
}
function weekStart(day: number): number {
return day - ((day + 3) % 7);
}
function totalsOf(bySource: Record<SpendSource, Tally>): SpendTotals {
const background = backgroundTally(bySource);
const total = emptyTally();
addTally(total, bySource.live);
addTally(total, background);
return {
costUsd: total.costUsd,
calls: total.calls,
tokens: tokensOf(total),
input: total.input,
output: total.output,
cacheRead: total.cacheRead,
cacheWrite: total.cacheWrite,
cacheHitRatio: cacheHitRatio({
cacheRead: total.cacheRead,
cacheWrite: total.cacheWrite,
uncachedInput: total.input,
}),
live: bucketTotals(bySource.live),
background: {
...bucketTotals(background),
sources: backgroundSources((source) => bucketTotals(bySource[source])),
},
};
}
function resolveScope(scopeId: ScopeId): { key: ScopeId; person: string | null; kind: string } {
const parsed = parseScopeId(scopeId);
const person = parsed.kind === "personal" && parsed.ref ? canonicalPerson(parsed.ref) : null;
return {
key: person === null ? scopeId : personalScope(person),
person,
kind: person === null ? (parsed.kind ?? "unknown") : "person",
};
}
function displayNameOf(sources: readonly ScopeId[], label: (scopeId: ScopeId) => string): string {
return sources.map(label).find((name) => name) ?? "";
}
function entityOf(acc: EntityAccumulator, label: (scopeId: ScopeId) => string): SpendEntity {
return {
principalId: acc.principalId,
scopeId: acc.scopeId,
kind: acc.kind,
displayName: displayNameOf(acc.sources, label),
...totalsOf(acc.bySource),
};
}
function byCodePoint(a: string | null, b: string | null): number {
if (a === b) return 0;
if (a === null) return 1;
if (b === null) return -1;
return a < b ? -1 : 1;
}
function byCostThen<T extends { costUsd: number }>(id: (row: T) => string | null): (a: T, b: T) => number {
return (a, b) => (a.costUsd !== b.costUsd ? b.costUsd - a.costUsd : byCodePoint(id(a), id(b)));
}
const byCostThenId = byCostThen<SpendEntity>((entity) => entity.scopeId);
const byCostThenModel = byCostThen<{ model: string | null; costUsd: number }>((row) => row.model);
const byCostThenCron = byCostThen<SpendCron>((row) => row.cronId);
export function summarizeSpend(rows: readonly SpendRow[], opts: SpendSummaryOptions): SpendReport {
const accumulators = new Map<string, EntityAccumulator>();
const buckets = new Map<
number,
{ bySource: Record<SpendSource, Tally>; models: Map<string | null, number>; people: Map<string | null, number> }
>();
const modelBuckets = new Map<string | null, Record<SpendSource, Tally>>();
const cronBuckets = new Map<string, { scopeId: ScopeId; bySource: Record<SpendSource, Tally> }>();
const orgSources = emptySources();
for (const row of rows) {
const { key, person, kind } = resolveScope(row.scopeId);
let acc = accumulators.get(key);
if (!acc) {
acc = { principalId: person, scopeId: key, kind, sources: [], bySource: emptySources() };
accumulators.set(key, acc);
}
if (!acc.sources.includes(row.scopeId)) acc.sources.push(row.scopeId);
const source = spendSource(row.origin);
addRow(acc.bySource[source], row);
addRow(orgSources[source], row);
if (row.cronId !== null) {
let cron = cronBuckets.get(row.cronId);
if (!cron) {
cron = { scopeId: row.scopeId, bySource: emptySources() };
cronBuckets.set(row.cronId, cron);
}
addRow(cron.bySource[source], row);
}
let model = modelBuckets.get(row.model);
if (!model) {
model = emptySources();
modelBuckets.set(row.model, model);
}
addRow(model[source], row);
const bucketKey = opts.bucket === "week" ? weekStart(row.day) : row.day;
let bucket = buckets.get(bucketKey);
if (!bucket) {
bucket = { bySource: emptySources(), models: new Map(), people: new Map() };
buckets.set(bucketKey, bucket);
}
addRow(bucket.bySource[source], row);
bucket.models.set(row.model, (bucket.models.get(row.model) ?? 0) + row.costUsd);
bucket.people.set(person, (bucket.people.get(person) ?? 0) + row.costUsd);
}
const entities = [...accumulators.values()].map((acc) => entityOf(acc, opts.label));
const people = entities.filter((e) => e.principalId !== null).sort(byCostThenId);
const scopes = entities.filter((e) => e.principalId === null).sort(byCostThenId);
const crons = [...cronBuckets.entries()]
.map(([cronId, { scopeId, bySource }]) => ({
cronId,
scopeId,
scopeName: opts.label(scopeId),
info: opts.cron(cronId),
...totalsOf(bySource),
}))
.sort(byCostThenCron);
const models = [...modelBuckets.entries()]
.map(([model, bySource]) => ({ model, ...totalsOf(bySource) }))
.sort(byCostThenModel);
const series = [...buckets.entries()]
.sort(([a], [b]) => a - b)
.map(([day, { bySource, models, people }]) => {
const background = backgroundTally(bySource);
const total = emptyTally();
addTally(total, bySource.live);
addTally(total, background);
return {
day: isoDay(day),
costUsd: total.costUsd,
calls: total.calls,
models: [...models].map(([model, costUsd]) => ({ model, costUsd })).sort(byCostThenModel),
people: [...people]
.sort(([a], [b]) => byCodePoint(a, b))
.map(([principalId, costUsd]) => ({ principalId, costUsd })),
live: { costUsd: bySource.live.costUsd },
background: {
costUsd: background.costUsd,
sources: backgroundSources((source) => ({ costUsd: bySource[source].costUsd })),
},
};
});
return {
window: {
from: isoDay(Math.floor(opts.from / DAY_MS)),
to: isoDay(Math.floor(opts.to / DAY_MS)),
bucket: opts.bucket,
},
org: totalsOf(orgSources),
series,
people,
scopes,
crons,
models,
};
}
export function spendExportRows(rows: readonly SpendRow[], label: (scopeId: ScopeId) => string): SpendExportRow[] {
const sources = new Map<ScopeId, ScopeId[]>();
const merged = new Map<string, SpendExportRow>();
for (const row of rows) {
const { key: scopeId, person, kind } = resolveScope(row.scopeId);
const seen = sources.get(scopeId);
if (!seen) sources.set(scopeId, [row.scopeId]);
else if (!seen.includes(row.scopeId)) seen.push(row.scopeId);
const source = spendSource(row.origin);
const key = JSON.stringify([row.day, scopeId, source, row.cronId, row.model]);
let out = merged.get(key);
if (!out) {
out = {
day: isoDay(row.day),
scopeId,
kind,
principalId: person,
displayName: "",
source,
cronId: row.cronId,
model: row.model,
...emptyTally(),
};
merged.set(key, out);
}
addRow(out, row);
}
return [...merged.values()]
.map((out) => ({ ...out, displayName: displayNameOf(sources.get(out.scopeId) ?? [], label) }))
.sort(
(a, b) =>
byCodePoint(a.day, b.day) ||
byCodePoint(a.scopeId, b.scopeId) ||
SPEND_SOURCES.indexOf(a.source) - SPEND_SOURCES.indexOf(b.source) ||
byCodePoint(a.cronId ?? "", b.cronId ?? "") ||
byCodePoint(a.model ?? "", b.model ?? ""),
);
}
function csvCell(value: string | number | null): string {
if (value === null) return "";
if (typeof value === "number") return String(value);
const defanged = CSV_FORMULA_LEAD.test(value) ? `'${value}` : value;
return CSV_NEEDS_QUOTES.test(defanged) ? `"${defanged.replaceAll('"', '""')}"` : defanged;
}
function csvBreakdown(
report: SpendReport,
breakdown: SpendBreakdown,
): { header: string[]; rows: { labels: (string | null)[]; totals: SpendTotals }[] } {
if (breakdown === "model")
return {
header: ["model"],
rows: report.models.map((row) => ({ labels: [row.model ?? "Unknown model"], totals: row })),
};
if (breakdown === "cron")
return {
header: ["cron_id", "title", "owner", "scope_id"],
rows: report.crons.map((row) => ({
labels: [row.cronId, row.info?.title ?? null, row.info?.owner ?? null, row.scopeId],
totals: row,
})),
};
return {
header: ["principal_id", "scope_id", "kind", "display_name"],
rows: [...report.people, ...report.scopes].map((row) => ({
labels: [row.principalId, row.scopeId, row.kind, row.displayName],
totals: row,
})),
};
}
export function spendCsv(report: SpendReport, breakdown: SpendBreakdown = "entity"): string {
const { header, rows } = csvBreakdown(report, breakdown);
const lines = [[...header, ...CSV_TOTALS_HEADER].join(",")];
for (const { labels, totals: row } of rows) {
lines.push(
[
...labels,
row.live.costUsd,
row.background.costUsd,
...BACKGROUND_SOURCES.map((source) => row.background.sources[source].costUsd),
row.costUsd,
row.calls,
row.input,
row.output,
row.cacheRead,
row.cacheWrite,
row.cacheHitRatio,
]
.map(csvCell)
.join(","),
);
}
return lines.join("\r\n") + "\r\n";
}
function parseDay(raw: string): number | null {
const value = raw.trim();
if (!DATE_ONLY.test(value)) return null;
const parsed = Date.parse(`${value}T00:00:00.000Z`);
return Number.isFinite(parsed) && new Date(parsed).toISOString().slice(0, 10) === value ? parsed : null;
}
function parseBound(raw: string): number | null {
const day = parseDay(raw);
if (day !== null || !EPOCH_MS.test(raw.trim())) return day;
const parsed = Number(raw.trim());
return Number.isSafeInteger(parsed) && Number.isFinite(new Date(parsed).getTime()) ? parsed : null;
}
async function loadSpendRows(
sessions: SessionStore,
range: { from: number; to: number },
): Promise<{ rows: SpendRow[]; asOf?: number }> {
return sessions.spendReport ? sessions.spendReport(range) : { rows: await sessions.spendRollup(range) };
}
async function cronInfoLookup(
app: BaseCtx["app"],
rows: readonly { cronId: string | null }[],
): Promise<Map<string, SpendCronInfo>> {
const ids = [...new Set(rows.flatMap((row) => (row.cronId === null ? [] : [row.cronId])))];
const crons = new Map<string, SpendCronInfo>();
for (let start = 0; start < ids.length; start += CRON_LOOKUP_CONCURRENCY) {
const batch = await Promise.all(ids.slice(start, start + CRON_LOOKUP_CONCURRENCY).map((id) => app.getCron(id)));
for (const cron of batch) {
if (!cron) continue;
const { id, title, message, action, owner, ownerScopeId, schedule, enabled, archived } = cron;
crons.set(id, { id, title, message, action, owner, ownerScopeId, schedule, enabled, archived });
}
}
return crons;
}
export async function spendExport(ctx: BaseCtx): Promise<void> {
const { req, res, app, deps, url } = ctx;
const token = deps.spendExportToken;
if (!token) return sendJson(res, 404, { error: "not_found" });
const authorization = req.headers.authorization;
if (typeof authorization !== "string" || !constantTimeEqual(authorization, `Bearer ${token}`))
return sendJson(res, 401, { error: "unauthorized" });
if (!deps.sessions) return sendJson(res, 503, { error: "spend_unavailable" });
const from = parseDay(url.searchParams.get("from") ?? "");
const to = parseDay(url.searchParams.get("to") ?? "");
if (from === null || to === null)
return sendJson(res, 400, { error: "bad_request", message: "from and to must be YYYY-MM-DD dates (UTC)" });
if (to <= from || to - from > EXPORT_MAX_DAYS * DAY_MS)
return sendJson(res, 400, {
error: "bad_request",
message: `to must be later than from and at most ${EXPORT_MAX_DAYS} days after it`,
});
const { rows, asOf } = await loadSpendRows(deps.sessions, { from, to });
const labels = await discoverScopes(
app,
deps,
rows.map((r) => r.scopeId),
);
const scope = orgScope(deps);
const window = { from: isoDay(from / DAY_MS), to: isoDay(to / DAY_MS) };
const exported = spendExportRows(rows, (id) => labels.get(id) ?? "");
const crons: SpendExportCron[] = [...(await cronInfoLookup(app, exported)).values()]
.sort((a, b) => byCodePoint(a.id, b.id))
.map(({ id, title, owner, ownerScopeId, schedule, enabled, archived }) => ({
id,
title,
owner,
ownerScopeId,
schedule,
enabled,
archived,
}));
audit(deps, {
principalId: SPEND_EXPORT_PRINCIPAL,
action: "spend.export",
resource: "spend",
scopeLabel: scope,
detail: `${window.from}..${window.to} rows=${exported.length}`,
});
sendJson(res, 200, { scopeId: scope, window, ...(asOf === undefined ? {} : { asOf }), rows: exported, crons });
}
export async function spend(ctx: ApiCtx): Promise<void> {
const { res, app, deps, url } = ctx;
const scope = orgScope(deps);
const actor = await authorizeAdmin(ctx, scope);
if (!actor) return;
audit(deps, { principalId: actor.id, action: "spend.read", resource: "spend", scopeLabel: scope });
const bad = (message: string) => sendJson(res, 400, { error: "bad_request", message });
const bucket = url.searchParams.get("bucket") ?? "day";
if (bucket !== "day" && bucket !== "week") return bad('bucket must be "day" or "week"');
const format = url.searchParams.get("format") ?? "json";
if (format !== "json" && format !== "csv") return bad('format must be "json" or "csv"');
const breakdown = SPEND_BREAKDOWNS.find((value) => value === (url.searchParams.get("breakdown") ?? "entity"));
if (!breakdown) return bad('breakdown must be "entity", "model" or "cron"');
const toRaw = url.searchParams.get("to");
const to = toRaw === null ? Math.floor(Date.now() / DAY_MS) * DAY_MS + DAY_MS : parseBound(toRaw);
if (to === null) return bad("to must be a YYYY-MM-DD date (UTC) or epoch milliseconds");
const fromRaw = url.searchParams.get("from");
const from = fromRaw === null ? to - DEFAULT_WINDOW_DAYS * DAY_MS : parseBound(fromRaw);
if (from === null) return bad("from must be a YYYY-MM-DD date (UTC) or epoch milliseconds");
if (to <= from) return bad("to must be later than from");
const { rows, asOf } = deps.sessions
? await loadSpendRows(deps.sessions, { from, to })
: { rows: [], asOf: undefined };
const labels = await discoverScopes(
app,
deps,
rows.map((r) => r.scopeId),
);
const crons =
format === "csv" && breakdown !== "cron" ? new Map<string, SpendCronInfo>() : await cronInfoLookup(app, rows);
const report = summarizeSpend(rows, {
from,
to,
bucket,
label: (id) => labels.get(id) ?? "",
cron: (cronId) => crons.get(cronId) ?? null,
});
if (format === "json")
return sendJson(res, 200, { scopeId: scope, ...report, ...(asOf === undefined ? {} : { asOf }) });
const body = spendCsv(report, breakdown);
res.writeHead(200, {
"content-type": contentTypeWithUtf8Charset("text/csv"),
"content-length": String(Buffer.byteLength(body)),
"content-disposition": contentDispositionAttachment(
`qm-spend-${breakdown === "entity" ? "" : `${breakdown}s-`}${report.window.from}-${report.window.to}.csv`,
),
"x-content-type-options": "nosniff",
});
res.end(body);
return;
}