Skip to content

One live console to monitor Claude Code and Codex agents across machines, track context and spend, and unblock sessions that need you.

19.8 KB
Raw Blame
// A report becomes rows: the session, its agents and their texts, new events, blowouts, this second's sample and what it spent,
// all in one transaction. So does a batch of Claude Code's API requests, Claude's only source of usage.
import type { ActivityEvent, AgentReport, ContextFill, ModelUsage, Plan, PlanStepView, PlanView, Report, SessionReport } from './protocol.js';
import { sessionKey, type AgentRow, type EventRow, type FleetDb, type ReportedSession, type SessionRow } from './db.js';
import { costParts, fromJson, spendShares, tokenSum, unpricedRises, usageTotals } from './derive.js';
import { localDay } from './days.js';
import type { Pricing } from './pricing.js';
import type { ApiRequest } from './telemetry.js';

/** What storing a report did, for the fleet's log lines, live set and broadcasts. */
export interface IngestResult {
  /** The session's id on the server (sessionKey), not the reporter's. */
  sessionId: string;
  /** The session had no row before this report. */
  created: boolean;
  /** A session that had ended or been dropped reports as live again. */
  revived: boolean;
  /** The stored row already said done. */
  wasDone: boolean;
  /** This report says done. */
  ended: boolean;
  /** The events stored now; an id stored before is skipped. */
  newEvents: EventRow[];
  /** The report's sentAt less the server's time: for debugging, and to place activity entries against a lease. */
  clockSkewMs: number;
}

/** How far a session's first reported start may be from the moment the server first hears of it and still be believed. */
export const START_TOLERANCE_MS = 120_000;

/**
 * The provider whose usage comes from its harness's own telemetry, one record per API request (`ingestRequests`), and never from its
 * reports: Claude Code's `api_request` events count every request, its side requests too, which the plugin never saw.
 */
export const TELEMETRY_PROVIDER = 'claude';

/** A JSON column: null for null or undefined. */
const json = (v: unknown): string | null => (v === null || v === undefined ? null : JSON.stringify(v));
/** A boolean column: 1, 0, or null for unknown. */
const bit = (v: boolean | null | undefined): number | null => (v === null || v === undefined ? null : Number(v));
/** A reported number rounded, or null when it is not a finite number. */
const whole = (v: unknown): number | null => (typeof v === 'number' && Number.isFinite(v) ? Math.round(v) : null);

/** A loop's context fill, in the columns the session and agent tables share. */
const contextColumns = (c: ContextFill) => ({ context_tokens: c.tokens, context_window: c.window, context_percent: c.percent });

/**
 * When the session started. A Hermes session's reported start is always believed: its reporter reads it from Hermes's own store.
 * Any other is fixed once stored, and on its first report the reporter's time is believed when it is near the moment the server
 * first heard of the session; otherwise, since Codex's daemon can give every session under it one host-wide value, the session
 * started when the server first heard of it.
 */
function startedAt(r: Report, prev: SessionRow | undefined, serverTime: number): number {
  const reported = r.session.startedAt;
  if (r.provider === 'hermes') return reported;
  if (prev) return prev.started_at;
  return Math.abs(reported - serverTime) <= START_TOLERANCE_MS ? reported : serverTime;
}

/** When the session ended: the first report saying done sets it, and a repeat of that report keeps it. */
function endedAt(s: SessionReport, prev: SessionRow | undefined, serverTime: number): number | null {
  if (s.status !== 'done') return null;
  return prev?.status === 'done' && prev.ended_at !== null ? prev.ended_at : serverTime;
}

/** Since when nobody has been attached: set by the first report with nobody attached, and kept while that stays true. */
function detachedAt(s: SessionReport, prev: SessionRow | undefined, serverTime: number): number | null {
  if (s.attached !== false) return null;
  return prev?.attached === 0 && prev.detached_at !== null ? prev.detached_at : serverTime;
}

/** Whether a stored plan and a reported one are the same plan: same source and title, and shared step ids whose text agrees. */
function samePlan(stored: PlanView, reported: Plan): boolean {
  if (stored.source !== reported.source || stored.title !== reported.title) return false;
  const storedText = new Map(stored.steps.map((step) => [step.id, step.text]));
  const shared = reported.steps.filter((step) => storedText.has(step.id));
  return shared.length > 0 && shared.every((step) => storedText.get(step.id) === step.text);
}

/**
 * Numbers each done step (`doneSeq`) in the order the server saw the steps finish, so the plan can list them in that order. A
 * step keeps its number while it stays done and loses it when reopened. Steps already done in the first plan the server sees
 * get none, and a different plan from the one stored (`prevJson`) starts again.
 */
function withDoneOrder(plan: Plan | null, prevJson: string | null): PlanView | null {
  if (!plan?.steps.some((step) => step.state === 'done')) return plan;
  const stored = fromJson<PlanView>(prevJson);
  if (!stored || !samePlan(stored, plan)) return plan;
  const was = new Map(stored.steps.map((step) => [step.id, step]));
  let next = 1 + Math.max(0, ...stored.steps.map((step) => step.doneSeq ?? 0));
  const steps = plan.steps.map((step): PlanStepView => {
    if (step.state !== 'done') return step;
    const before = was.get(step.id);
    if (before?.state !== 'done') return { ...step, doneSeq: next++ };
    return before.doneSeq === undefined ? step : { ...step, doneSeq: before.doneSeq };
  });
  return { ...plan, steps };
}

/**
 * The main loop's model API failure that stands: the newest of its new `fail` events that carries a `category`,
 * until the session is seen running again with activity after it.
 */
function standingApiError(prev: SessionRow | undefined, newEvents: readonly EventRow[], s: SessionReport): { at: number | null; category: string | null } {
  let failure = { at: prev?.api_error_at ?? null, category: prev?.api_error ?? null };
  for (const e of newEvents) {
    if (e.kind !== 'fail' || e.agent_id !== null || !e.data_json) continue;
    const category = (JSON.parse(e.data_json) as { category?: unknown }).category;
    if (typeof category === 'string' && (failure.at === null || e.t >= failure.at)) failure = { at: e.t, category };
  }
  const ranSince = failure.at !== null && s.status === 'running' && s.lastActivityAt !== null && s.lastActivityAt > failure.at;
  return ranSince ? { at: null, category: null } : failure;
}

/** The session's row; `usage` is its usage as the session's source gives it, undefined when unknown, which keeps the stored usage. */
function sessionRow(r: Report, prev: SessionRow | undefined, serverTime: number, owner: string | null, newEvents: readonly EventRow[], usage: ModelUsage[] | undefined): ReportedSession {
  const s = r.session;
  const failure = standingApiError(prev, newEvents, s);
  // The plan stored last, whether the last report carried it or it was kept through a gap.
  const lastPlan = prev?.plan_json ?? prev?.plan_kept_json ?? null;
  return {
    // Whose session, and which reporter on which machine sent it.
    id: sessionKey(s.id, r.machine.id),
    native_id: s.id,
    provider: r.provider,
    owner: owner ?? prev?.owner ?? null,
    machine_id: r.machine.id,
    machine_json: JSON.stringify(r.machine),
    reporter_version: r.reporter.version,
    reporter_build: r.reporter.build ?? null,
    engine_version: r.reporter.engineVersion,
    host_surface: r.reporter.hostSurface,
    // What it is and where it runs.
    name: s.name,
    cwd: s.cwd,
    project_root: s.projectRoot,
    repo_name: s.repoName,
    git_json: json(s.git),
    // Where it stands. last_seen_at is the server's clock, never the reporter's.
    status: s.status,
    wait_reason: s.waitReason,
    wait_note: s.waitNote,
    end_reason: s.endReason,
    resume_id: s.resumeId,
    started_at: startedAt(r, prev, serverTime),
    last_seen_at: serverTime,
    ended_at: endedAt(s, prev, serverTime),
    dropped_at: null,
    attached: bit(s.attached),
    detached_at: detachedAt(s, prev, serverTime),
    // Set once; a later report without it keeps the link. A /clear continues a session on the same machine.
    cleared_from: s.clearedFrom ? sessionKey(s.clearedFrom, r.machine.id) : (prev?.cleared_from ?? null),
    // Model, usage and spend.
    model: s.model,
    // A reporter learns a loop's effort from its next model request, so one that has just started on a session already
    // running (a resume, a reloaded plugin) reports none until then: the effort stored for the same model stays.
    effort: s.effort ?? (prev && prev.model === s.model ? prev.effort : null),
    ...contextColumns(s.context),
    // Usage not known now leaves the stored usage as it is.
    usage_json: usage === undefined ? (prev?.usage_json ?? null) : json(usage),
    rate_limits_json: json(s.rateLimits),
    blowouts: s.blowouts,
    // Liveness and trouble.
    tool_failures: s.toolFailures,
    hooks_active: bit(s.hooksActive),
    last_activity_at: s.lastActivityAt,
    tool_open: s.toolOpen ? 1 : 0,
    api_error_at: failure.at,
    api_error: failure.category,
    // Its work. While a report carries no plan, the last one is kept so the finish order survives the gap.
    plan_json: json(withDoneOrder(s.plan, lastPlan)),
    plan_kept_json: s.plan ? null : lastPlan,
    loop_json: json(s.loop),
    files_json: json(s.files),
    background_json: json(s.background),
  };
}

/**
 * An agent's row, in the groups agentView reads it back in; its full prompt and answer go to agent_texts (storeAgents). `usage` is
 * as for sessionRow: undefined keeps the stored usage (upsertAgent).
 */
function agentRow(sessionId: string, a: AgentReport, usage: ModelUsage[] | undefined): AgentRow {
  return {
    // Who it is and what started it.
    session_id: sessionId,
    id: a.id,
    parent_id: a.parentId,
    kind: a.kind,
    type: a.type,
    name: a.name,
    description: a.description,
    // Where it stands.
    status: a.status,
    started_at: a.startedAt,
    ended_at: a.endedAt,
    last_activity_at: a.lastActivityAt,
    tool_open: a.toolOpen ? 1 : 0,
    failure_json: json(a.failure),
    // Its model, and what it used.
    model: a.model,
    effort: a.effort,
    effort_source: a.effortSource,
    ...contextColumns(a.context),
    usage_json: json(usage),
    blowouts: a.blowouts,
    // What it did.
    skills_json: JSON.stringify(a.skills),
    last_tool: a.lastTool,
    tool_calls: a.toolCalls,
    tool_counts_json: json(a.toolCounts),
    tool_failures: a.toolFailures,
    steps_done: a.steps?.done ?? null,
    steps_total: a.steps?.total ?? null,
    // Excerpts of its prompt and answer.
    prompt_excerpt: a.promptExcerpt,
    answer_excerpt: a.answerExcerpt,
  };
}

/** Stores the report's events, skipping ids stored before, and returns the ones stored now. */
function storeEvents(f: FleetDb, sessionId: string, events: readonly ActivityEvent[]): EventRow[] {
  const rows = events.map((e): EventRow => ({ id: e.id, session_id: sessionId, agent_id: e.agentId, t: e.t, kind: e.kind, text: e.text, data_json: json(e.data) }));
  return rows.filter((row) => f.insertEvent(row));
}

/**
 * Upserts each agent the report lists, with its usage as `usageOf` gives it and any full text it sent; one it leaves out stays, as a
 * reload can lose an agent. A report that ends the session (`endedNow`, the server's time then) aborts every agent still running
 * or queued.
 */
function storeAgents(f: FleetDb, sessionId: string, agents: readonly AgentReport[], usageOf: (a: AgentReport) => ModelUsage[] | undefined, endedNow: number | null): void {
  for (const a of agents) {
    f.upsertAgent(agentRow(sessionId, a, usageOf(a)));
    if (a.promptText || a.answerText) f.upsertAgentTexts(sessionId, a.id, { prompt: a.promptText || undefined, answer: a.answerText || undefined });
  }
  if (endedNow !== null) f.abortUnfinishedAgents(sessionId, endedNow);
}

/** A blowouts row for each new `blow` event whose data names its kind. */
function storeBlowouts(f: FleetDb, sessionId: string, newEvents: readonly EventRow[]): void {
  for (const e of newEvents) {
    const data = e.kind === 'blow' && e.data_json ? (JSON.parse(e.data_json) as Record<string, unknown>) : null;
    if (typeof data?.kind !== 'string' || data.kind === '') continue;
    f.addBlowout({ session_id: sessionId, agent_id: e.agent_id, t: e.t, kind: data.kind.slice(0, 40), tokens_before: whole(data.tokensBefore), tokens_after: whole(data.tokensAfter) });
  }
}

/**
 * This second's sample, from a report that carries usage: every token of the session, its agents' included, and the rise since the
 * sample before; the same two for its output tokens alone (what the token rates count); the fullest context among the main loop and
 * its running agents; how many agents run (none once the session has ended); and `spent`, what this report booked to the spend
 * record.
 */
function storeSample(f: FleetDb, row: ReportedSession, serverTime: number, ended: boolean, spent: number): void {
  const t = Math.floor(serverTime / 1000);
  const running = f.getAgents(row.id).filter((a) => a.status === 'running');
  const used = usageTotals(row)!;
  const before = f.previousSample(row.id, t);
  const peakPercent = Math.max(row.context_percent ?? 0, ...running.map((a) => a.context_percent ?? 0));
  const tokensRise = before ? tokenSum(used) - before.tokens_total : 0;
  const outputRise = before && before.output_total !== null ? used.output - before.output_total : 0;
  f.upsertSample({
    session_id: row.id,
    t,
    tokens_total: tokenSum(used),
    tokens_delta: Math.max(0, tokensRise),
    output_total: used.output,
    output_delta: Math.max(0, outputRise),
    peak_context: Math.max(0, Math.min(1, peakPercent / 100)),
    running_agents: ended ? 0 : running.length,
    spend_usd: spent,
  });
}

/**
 * Books what this report spent: the session's cost, as its card shows it, less the highest figure already counted, on the
 * server's day, in the transaction that stores the session, so a restart or a kill can neither lose it nor count it twice. A
 * session's first figure only sets that mark: a resumed session's first figure is its whole life, not today's spend. A
 * figure at or below the mark books nothing. Only a report that carries usage, from a provider other than TELEMETRY_PROVIDER,
 * comes here. Returns the USD booked.
 */
function recordSpend(f: FleetDb, pricing: Pricing | undefined, row: ReportedSession, prev: SessionRow | undefined, machineId: string, serverTime: number): number {
  const { usd: cost, partial } = costParts(pricing, row);
  if (!prev) {
    f.setSpendCounted(row.id, cost);
    return 0;
  }
  const where = { day: localDay(serverTime), machine_id: machineId, provider: row.provider };
  // Unknown stays unknown, never $0, whatever the cost does below: a session can mix priced and unpriced models.
  if (cost === null || partial) for (const [model] of unpricedRises(pricing, prev, row)) f.addSpend({ ...where, model, usd: null });
  const counted = prev.spend_counted;
  if (cost === null || (counted !== null && cost <= counted)) return 0;
  f.setSpendCounted(row.id, cost);
  if (counted === null) return 0;
  for (const { model, usd } of spendShares(pricing, prev.usage_json, row, cost - counted)) f.addSpend({ ...where, model, usd });
  return cost - counted;
}

/**
 * Stores one validated report under its session's id on the server (sessionKey). `serverTime`, never the reporter's clock, sets
 * last_seen_at. `owner` is `Admin` when the report carried the admin secret; null (no secret configured) leaves a stored owner as
 * it was. `pricing` prices the spend it records.
 */
export function ingestReport(f: FleetDb, report: Report, serverTime: number, owner: string | null = null, pricing?: Pricing): IngestResult {
  return f.tx(() => {
    const sid = sessionKey(report.session.id, report.machine.id);
    const prev = f.getSession(sid);
    // Events first, so the failure the row carries reads only the events this report added.
    const newEvents = storeEvents(f, sid, report.events);
    // A Claude session's usage, and each of its agents', is its requests' sums (ingestRequests), never its report's. Until it has
    // requests, what is stored stays: unknown, or its figure from before Claude usage came from telemetry.
    const fromTelemetry = report.provider === TELEMETRY_PROVIDER;
    const sums = (agentId?: string): ModelUsage[] | undefined => f.requestUsage(sid, agentId) ?? undefined;
    const row = sessionRow(report, prev, serverTime, owner, newEvents, fromTelemetry ? sums() : report.session.usageByModel);
    const ended = row.status === 'done';
    f.upsertSession(row);
    storeAgents(f, sid, report.agents, (a) => (fromTelemetry ? sums(a.id) : a.usageByModel), ended ? serverTime : null);
    storeBlowouts(f, sid, newEvents);
    // This second's sample, while the session's usage is known. A Claude report books nothing, its requests book their own spend;
    // any other report that carries usage books its rise. A report without usage leaves the figure unknown in it: it books nothing,
    // moves no mark and takes no sample; the sweep carries the last sample on (Fleet.carrySample).
    if (fromTelemetry) {
      if (row.usage_json !== null) storeSample(f, row, serverTime, ended, 0);
    } else if (report.session.usageByModel !== undefined) storeSample(f, row, serverTime, ended, recordSpend(f, pricing, row, prev, report.machine.id, serverTime));
    return {
      sessionId: sid,
      created: prev === undefined,
      revived: prev !== undefined && !ended && (prev.ended_at !== null || prev.dropped_at !== null),
      wasDone: prev?.status === 'done',
      ended,
      newEvents,
      clockSkewMs: report.sentAt - serverTime,
    };
  });
}

/**
 * Stores a batch of Claude Code's API requests from one machine, in one transaction. A request already counted (its id
 * noted) is skipped, so an exporter's resend counts nothing. Each new one adds its tokens to its session's sums, and books its price,
 * at the fast tier when it was made in fast mode, else at the long tier when its own prompt is long, to the spend record on its own day: every request is spend as it happens, with no
 * mark and no first figure. A session not reported yet keeps its sums until its first report reads them. Each stored session the
 * batch touched then takes its usage, and its agents theirs, from the sums, and this second's sample with what the batch booked for
 * it; a report in the same second writes the same sample again with the same figures (storeSample), so nothing counts twice.
 * Returns the sessions touched.
 */
export function ingestRequests(f: FleetDb, machineId: string, requests: readonly ApiRequest[], serverTime: number, pricing?: Pricing): string[] {
  return f.tx(() => {
    const touched = new Map<string, { spent: number; agents: Set<string> }>();
    for (const r of requests) {
      if (!f.addRequest(r.requestId, serverTime)) continue;
      const sid = sessionKey(r.sessionId, machineId);
      f.addRequestUsage(sid, r.agentId ?? '', r.usage, serverTime);
      // An unpriced model's spend is unknown, never $0 (addSpend); a request that cost nothing adds no row.
      const usd = pricing ? pricing.costOf([r.usage]) : null;
      if (usd !== 0) f.addSpend({ day: localDay(r.t), machine_id: machineId, provider: TELEMETRY_PROVIDER, model: r.usage.model, usd });
      const session = touched.get(sid) ?? { spent: 0, agents: new Set<string>() };
      session.spent += usd ?? 0;
      if (r.agentId !== null) session.agents.add(r.agentId);
      touched.set(sid, session);
    }
    for (const [sid, { spent, agents }] of touched) {
      for (const agentId of agents) f.setUsage(sid, agentId, f.requestUsage(sid, agentId)!);
      f.setUsage(sid, null, f.requestUsage(sid)!);
      const row = f.getSession(sid);
      if (row) storeSample(f, row, serverTime, row.status === 'done', spent);
    }
    return [...touched.keys()];
  });
}