Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

3.7 KB
Raw Blame
/**
 * Shared tokens-per-second helpers for logged/ranked calls (#13130).
 *
 * The metric is GENERATION throughput, not end-to-end wall clock:
 * `open-sse/utils/generationThroughput.ts` documents that tok/s MUST exclude
 * TTFT. Two inputs that callers had been getting wrong converge here:
 *
 * 1. Denominator: use `durationMs - ttftMs` when TTFT is known and sane,
 *    otherwise fall back to full duration (better than hiding the number).
 * 2. Numerator: providers are expected to fold reasoning/thinking tokens into
 *    `completion_tokens` (OpenAI reasoning models, Anthropic thinking blocks,
 *    Gemini thoughts all do). A provider that instead reports reasoning only in
 *    `completion_tokens_details.reasoning_tokens` leaves `tokens_out`
 *    undercounted. `max(tokensOut, tokensReasoning)` covers both shapes
 *    without double counting: when reasoning is a subset of tokensOut the max
 *    is tokensOut; when the provider excluded it, the recorded reasoning count
 *    is the better available lower bound.
 *
 * Pure + dependency-free; safe for client components, server code, and tests.
 */

/** Non-negative finite number, else 0. */
function nonNegative(value: unknown): number {
  const n = typeof value === "number" ? value : Number(value);
  return Number.isFinite(n) && n > 0 ? n : 0;
}

/**
 * Generation-time window in ms: full duration minus TTFT when TTFT is known
 * and strictly inside the request; full duration otherwise (or null when the
 * duration itself is unusable).
 */
export function resolveGenerationMs(
  durationMs: number | null | undefined,
  ttftMs: number | null | undefined
): number | null {
  const duration = nonNegative(durationMs);
  if (duration <= 0) return null;
  const ttft = nonNegative(ttftMs);
  if (ttft > 0 && ttft < duration) return duration - ttft;
  return duration;
}

/** Best available output-token count for TPS: never double-counts reasoning (#13130). */
export function resolveTpsOutputTokens(
  tokensOut: number | null | undefined,
  tokensReasoning: number | null | undefined
): number {
  return Math.max(nonNegative(tokensOut), nonNegative(tokensReasoning));
}

/**
 * Tokens per second for one logged call. Returns 0 (not null) when the inputs
 * cannot produce a rate — matches the historical `getLogTps()` display
 * contract in RequestLoggerV2.
 */
export function computeLogTps(
  tokensOut: number | null | undefined,
  tokensReasoning: number | null | undefined,
  durationMs: number | null | undefined,
  ttftMs?: number | null | undefined
): number {
  const tokens = resolveTpsOutputTokens(tokensOut, tokensReasoning);
  const generationMs = resolveGenerationMs(durationMs, ttftMs);
  if (tokens <= 0 || generationMs === null || generationMs <= 0) return 0;
  return tokens / (generationMs / 1000);
}

/**
 * Tooltip for the TPS cell (#13130): exposes the generation-time breakdown so
 * thinking-model rows (long TTFT / prefill) explain why their TPS differs from
 * wall-clock perception. `formatDuration` is injected to keep this module
 * dependency-free.
 */
export function buildLogTpsTitle(
  log: { duration?: number | null; ttft?: number | null; tokens?: { reasoning?: number | null } },
  tps: number,
  formatDuration: (ms: number) => string
): string {
  const parts = [`${tps.toFixed(2)} tokens/sec`];
  if (typeof log?.ttft === "number" && log.ttft > 0) {
    parts.push(`TTFT ${formatDuration(log.ttft)}`);
  }
  const genMs = resolveGenerationMs(log?.duration, log?.ttft);
  if (genMs != null && genMs !== log?.duration) {
    parts.push(`generation ${(genMs / 1000).toFixed(1)}s`);
  }
  const reasoning = log?.tokens?.reasoning ?? 0;
  if (reasoning > 0) parts.push(`reasoning ${reasoning} tok`);
  return parts.join(" · ");
}