Skip to content

Never stop coding. Free MIT AI gateway: one endpoint, 375 providers (150+ free), 1,400+ models Kimi, Claude, GPT, Gemini, GLM, DeepSeek, MiniMax. Works with Claude Code, Codex, Cursor, OpenCode, Cline & Copilot. Quota-aware auto-fallback, RTK+Caveman compression saves 15-95% tokens, MCP/A2A, Desktop/PWA. Built by 900+ contributors

Fix analytics history and log token accounting (#2904)

Halil Tezcan KARABULUTefcd06200209
diff --git a/src/app/api/usage/analytics/route.ts b/src/app/api/usage/analytics/route.ts
index a924837ede..6fd5475018 100644
--- a/src/app/api/usage/analytics/route.ts
+++ b/src/app/api/usage/analytics/route.ts
@@ -2,6 +2,7 @@ import { NextResponse } from "next/server";
import { requireManagementAuth } from "@/lib/api/requireManagementAuth";
import { getApiKeys } from "@/lib/db/apiKeys";
import { getDbInstance } from "@/lib/db/core";
+import { getUserDatabaseSettings } from "@/lib/db/databaseSettings";
function getRangeStartIso(range: string): string | null {
const end = new Date();
@@ -327,6 +328,14 @@ export async function GET(request: Request) {
}
}
+ // Compute the raw-data cutoff: rows older than this may have been rolled up to
+ // daily_usage_summary and deleted from usage_history.
+ const dbSettings = getUserDatabaseSettings();
+ const rawRetentionDays = dbSettings.aggregation?.rawDataRetentionDays ?? 30;
+ const rawCutoff = new Date();
+ rawCutoff.setDate(rawCutoff.getDate() - rawRetentionDays);
+ const rawCutoffIso = rawCutoff.toISOString();
+
const conditions = [];
const params: Record<string, string> = {};
@@ -351,6 +360,93 @@ export async function GET(request: Request) {
const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : "";
+ // Build a UNION data source that merges recent raw rows with aggregated history.
+ // daily_usage_summary rows are included only when the query window extends before rawCutoffIso.
+ // The api_key filter is intentionally NOT applied to daily_usage_summary (api_key not stored there).
+ const needsAggregated = !sinceIso || sinceIso < rawCutoffIso;
+
+ const rawConditions: string[] = [];
+ if (sinceIso) rawConditions.push("timestamp >= @since");
+ if (untilIso) rawConditions.push("timestamp <= @until");
+ if (apiKeyWhere) rawConditions.push(apiKeyWhere);
+ const rawWhere = rawConditions.length > 0 ? `WHERE ${rawConditions.join(" AND ")}` : "";
+
+ // Aggregated rows only span dates within the requested window (no api_key filter).
+ const aggConditions: string[] = [];
+ if (sinceIso) {
+ // Use date comparison on the summary's date column (YYYY-MM-DD).
+ const sinceDate = sinceIso.split("T")[0];
+ aggConditions.push("date >= @sinceDate");
+ params.sinceDate = sinceDate;
+ }
+ if (untilIso) {
+ const untilDate = untilIso.split("T")[0];
+ aggConditions.push("date <= @untilDate");
+ params.untilDate = untilDate;
+ }
+ const aggWhere = aggConditions.length > 0 ? `WHERE ${aggConditions.join(" AND ")}` : "";
+
+ // Unified source CTE: columns aligned to usage_history shape needed by analytics queries.
+ // Fields not available in daily_usage_summary default to 0/NULL.
+ const unifiedSource = needsAggregated
+ ? `(
+ SELECT
+ timestamp,
+ provider,
+ model,
+ tokens_input,
+ tokens_output,
+ tokens_cache_read,
+ tokens_cache_creation,
+ tokens_reasoning,
+ service_tier,
+ success,
+ latency_ms,
+ connection_id,
+ api_key_id,
+ api_key_name,
+ combo_name,
+ requested_model
+ FROM usage_history
+ ${rawWhere}
+ UNION ALL
+ SELECT
+ date || 'T12:00:00.000Z' as timestamp,
+ provider,
+ model,
+ total_input_tokens as tokens_input,
+ total_output_tokens as tokens_output,
+ 0 as tokens_cache_read,
+ 0 as tokens_cache_creation,
+ 0 as tokens_reasoning,
+ 'standard' as service_tier,
+ 1 as success,
+ 0 as latency_ms,
+ NULL as connection_id,
+ NULL as api_key_id,
+ NULL as api_key_name,
+ NULL as combo_name,
+ NULL as requested_model
+ FROM daily_usage_summary
+ ${aggWhere}
+ )`
+ : `(SELECT
+ timestamp, provider, model,
+ tokens_input, tokens_output,
+ tokens_cache_read, tokens_cache_creation, tokens_reasoning,
+ service_tier, success, latency_ms,
+ connection_id, api_key_id, api_key_name,
+ combo_name, requested_model
+ FROM usage_history
+ ${whereClause}
+ )`;
+
+ // When using the unified source the WHERE filters are already embedded inside.
+ // For the original whereClause-based queries that still reference usage_history directly
+ // (e.g. fallbackRow, accountRows) we keep them as-is since they need joins or
+ // columns only present in usage_history.
+ const unifiedWhere = ""; // no additional WHERE needed — filters embedded in unifiedSource
+
// Fetch pricing data for cost calculation (no rows loaded)
const { getPricing } = await import("@/lib/db/settings");
const rawPricingByProvider = (await getPricing()) as PricingByProvider;
@@ -383,8 +479,8 @@ export async function GET(request: Request) {
COALESCE(AVG(latency_ms), 0) as avgLatencyMs,
COALESCE(MIN(timestamp), '') as firstRequest,
COALESCE(MAX(timestamp), '') as lastRequest
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
`
)
.get(params) as Record<string, unknown>;
@@ -398,8 +494,8 @@ export async function GET(request: Request) {
COALESCE(SUM(tokens_input), 0) as promptTokens,
COALESCE(SUM(tokens_output), 0) as completionTokens,
COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY DATE(timestamp)
ORDER BY date ASC
`
@@ -419,8 +515,8 @@ export async function GET(request: Request) {
COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens,
COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY DATE(timestamp), LOWER(provider), LOWER(model), serviceTier
ORDER BY date ASC
`
@@ -478,8 +574,8 @@ export async function GET(request: Request) {
COALESCE(AVG(latency_ms), 0) as avgLatencyMs,
COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) as successfulRequests,
COALESCE(MAX(timestamp), '') as lastUsed
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY LOWER(model), LOWER(provider), serviceTier
ORDER BY requests DESC
`
@@ -498,8 +594,8 @@ export async function GET(request: Request) {
COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens,
COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY LOWER(provider), LOWER(model), serviceTier
`
)
@@ -516,8 +612,8 @@ export async function GET(request: Request) {
COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens,
COALESCE(AVG(latency_ms), 0) as avgLatencyMs,
COALESCE(SUM(CASE WHEN success = 1 THEN 1 ELSE 0 END), 0) as successfulRequests
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY LOWER(provider)
ORDER BY requests DESC
`
@@ -608,9 +704,8 @@ export async function GET(request: Request) {
COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens,
COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens,
COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens
- FROM usage_history
-
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY serviceTier, LOWER(provider), LOWER(model)
`
)
@@ -661,8 +756,8 @@ export async function GET(request: Request) {
strftime('%w', timestamp) as dayOfWeek,
COUNT(*) as requests,
COALESCE(SUM(tokens_input + tokens_output), 0) as totalTokens
- FROM usage_history
- ${whereClause}
+ FROM ${unifiedSource} AS _u
+ ${unifiedWhere}
GROUP BY DATE(timestamp), strftime('%w', timestamp)
)
GROUP BY dayOfWeek
@@ -1116,19 +1211,58 @@ export async function GET(request: Request) {
}
const presetSinceIso = getRangeStartIso(presetRange);
- const presetConditions = [];
const presetParams: Record<string, string> = {};
+
+ // Build unified source for preset cost queries (same UNION logic as main query).
+ const presetNeedsAggregated = !presetSinceIso || presetSinceIso < rawCutoffIso;
+
+ const presetRawConds: string[] = [];
if (presetSinceIso) {
- presetConditions.push("timestamp >= @presetSince");
+ presetRawConds.push("timestamp >= @presetSince");
presetParams.presetSince = presetSinceIso;
}
if (apiKeyWhere) {
- presetConditions.push(apiKeyWhere);
+ presetRawConds.push(apiKeyWhere);
Object.assign(presetParams, params);
}
+ const presetRawWhere =
+ presetRawConds.length > 0 ? `WHERE ${presetRawConds.join(" AND ")}` : "";
- const presetWhere =
- presetConditions.length > 0 ? `WHERE ${presetConditions.join(" AND ")}` : "";
+ const presetAggConds: string[] = [];
+ if (presetSinceIso) {
+ const presetSinceDate = presetSinceIso.split("T")[0];
+ presetAggConds.push("date >= @presetSinceDate");
+ presetParams.presetSinceDate = presetSinceDate;
+ }
+ const presetAggWhere =
+ presetAggConds.length > 0 ? `WHERE ${presetAggConds.join(" AND ")}` : "";
+
+ const presetUnifiedSource = presetNeedsAggregated
+ ? `(
+ SELECT timestamp, provider, model, service_tier,
+ tokens_input, tokens_output,
+ tokens_cache_read, tokens_cache_creation, tokens_reasoning
+ FROM usage_history
+ ${presetRawWhere}
+ UNION ALL
+ SELECT
+ date || 'T12:00:00.000Z' as timestamp,
+ provider, model,
+ 'standard' as service_tier,
+ total_input_tokens as tokens_input,
+ total_output_tokens as tokens_output,
+ 0 as tokens_cache_read,
+ 0 as tokens_cache_creation,
+ 0 as tokens_reasoning
+ FROM daily_usage_summary
+ ${presetAggWhere}
+ )`
+ : `(SELECT timestamp, provider, model, service_tier,
+ tokens_input, tokens_output,
+ tokens_cache_read, tokens_cache_creation, tokens_reasoning
+ FROM usage_history
+ ${presetRawWhere}
+ )`;
const presetModelRows = db
.prepare(
@@ -1142,8 +1276,7 @@ export async function GET(request: Request) {
COALESCE(SUM(tokens_cache_read), 0) as cacheReadTokens,
COALESCE(SUM(tokens_cache_creation), 0) as cacheCreationTokens,
COALESCE(SUM(tokens_reasoning), 0) as reasoningTokens
- FROM usage_history
- ${presetWhere}
+ FROM ${presetUnifiedSource} AS _pu
GROUP BY LOWER(model), LOWER(provider), serviceTier
`
)
diff --git a/src/lib/db/cleanup.ts b/src/lib/db/cleanup.ts
index 544186e22e..93c1d1d7e5 100644
--- a/src/lib/db/cleanup.ts
+++ b/src/lib/db/cleanup.ts
@@ -6,6 +6,7 @@
import { getDbInstance } from "./core";
import { getUserDatabaseSettings } from "./databaseSettings";
+import { rollupUsageHistoryBeforeDate } from "@/lib/usage/aggregateHistory";
interface CleanupResult {
deleted: number;
@@ -85,9 +86,19 @@ export async function cleanupUsageHistory(): Promise<CleanupResult> {
const cutoffDate = new Date();
cutoffDate.setDate(cutoffDate.getDate() - retentionDays);
const cutoffISO = cutoffDate.toISOString();
+ const cutoffDateStr = cutoffISO.split("T")[0];
const result: CleanupResult = { deleted: 0, errors: 0 };
+ try {
+ // Roll up rows that are about to be deleted into daily_usage_summary so that
+ // the analytics route can still surface historical data via the UNION query.
+ await rollupUsageHistoryBeforeDate(cutoffDateStr);
+ } catch (err: unknown) {
+ // Non-fatal: log but continue with deletion so cleanup still runs.
+ console.error("[Cleanup] Error rolling up usage_history before deletion:", err);
+ }
+
try {
const stmt = db.prepare("DELETE FROM usage_history WHERE timestamp < ?");
const runResult = stmt.run(cutoffISO);
diff --git a/src/lib/usage/aggregateHistory.ts b/src/lib/usage/aggregateHistory.ts
index 486ae512a7..9841ad1db5 100644
--- a/src/lib/usage/aggregateHistory.ts
+++ b/src/lib/usage/aggregateHistory.ts
@@ -1,6 +1,6 @@
/**
* Aggregation utility functions for usage data summarization.
- * Rolls up quota_snapshots into hourly and daily summary tables.
+ * Rolls up usage_history (and quota_snapshots) into daily summary tables.
*
* @module lib/usage/aggregateHistory
*/
@@ -130,6 +130,65 @@ export async function rollupHourlyQuota(
return result;
}
+/**
+ * Roll up usage_history into daily_usage_summary before raw rows are deleted.
+ * This is the authoritative rollup — sourced from actual per-request token data,
+ * not from quota_snapshots. Should be called before cleanupUsageHistory() deletes rows.
+ *
+ * The ON CONFLICT clause uses SUM so re-running is additive-safe: if a date already
+ * has a partial rollup (e.g. from a previous partial cleanup), new rows accumulate.
+ *
+ * @param beforeDate - ISO date string (YYYY-MM-DD). Rows strictly before this date are rolled up.
+ * @returns Aggregation result with counts
+ */
+export async function rollupUsageHistoryBeforeDate(beforeDate: string): Promise<AggregationResult> {
+ const db = getDbInstance();
+
+ const result: AggregationResult = {
+ processed: 0,
+ inserted: 0,
+ errors: 0,
+ };
+
+ try {
+ const aggregateQuery = `
+ INSERT INTO daily_usage_summary (provider, model, date, total_requests, total_input_tokens, total_output_tokens, total_cost)
+ SELECT
+ LOWER(provider) as provider,
+ LOWER(model) as model,
+ DATE(timestamp) as date,
+ COUNT(*) as total_requests,
+ COALESCE(SUM(tokens_input), 0) as total_input_tokens,
+ COALESCE(SUM(tokens_output), 0) as total_output_tokens,
+ 0.0 as total_cost
+ FROM usage_history
+ WHERE DATE(timestamp) < ?
+ AND provider IS NOT NULL AND provider != ''
+ AND model IS NOT NULL AND model != ''
+ GROUP BY LOWER(provider), LOWER(model), DATE(timestamp)
+ ON CONFLICT(provider, model, date) DO UPDATE SET
+ total_requests = daily_usage_summary.total_requests + excluded.total_requests,
+ total_input_tokens = daily_usage_summary.total_input_tokens + excluded.total_input_tokens,
+ total_output_tokens = daily_usage_summary.total_output_tokens + excluded.total_output_tokens
+ `;
+
+ const stmt = db.prepare(aggregateQuery);
+ const runResult = stmt.run(beforeDate);
+
+ result.processed = runResult.changes;
+ result.inserted = runResult.changes;
+
+ console.log(
+ `[Aggregation] usage_history rollup: ${result.inserted} rows for dates before ${beforeDate}`
+ );
+ } catch (err: any) {
+ console.error("[Aggregation] usage_history rollup error:", err);
+ result.errors++;
+ }
+
+ return result;
+}
+
/**
* Get the cutoff date for raw data based on retention settings.
* Data older than this should be aggregated and cleaned up.
diff --git a/src/lib/usage/tokenAccounting.ts b/src/lib/usage/tokenAccounting.ts
index 179795a432..92dd83f21c 100644
--- a/src/lib/usage/tokenAccounting.ts
+++ b/src/lib/usage/tokenAccounting.ts
@@ -48,6 +48,13 @@ export function getLoggedInputTokens(tokens: unknown): number {
return toFiniteNumber(tokenRecord.input);
}
+ // Prefer prompt_tokens when present: translators normalize provider-specific
+ // usage into OpenAI shape there, and may keep input_tokens for compatibility.
+ // Treating that input_tokens as raw Claude usage would add cache tokens again.
+ if (tokenRecord.prompt_tokens !== undefined && tokenRecord.prompt_tokens !== null) {
+ return toFiniteNumber(tokenRecord.prompt_tokens);
+ }
+
if (tokenRecord.input_tokens !== undefined && tokenRecord.input_tokens !== null) {
// Anthropic / anthropic-compatible-cc streaming: input_tokens is only the
// non-cached portion. The cache counters sit as separate top-level fields
@@ -60,13 +67,7 @@ export function getLoggedInputTokens(tokens: unknown): number {
);
}
- // prompt_tokens from translator/extractor already includes cache tokens:
- // - OpenAI format: prompt_tokens inherently includes cached
- // - Claude non-streaming: extractUsageFromResponse sums input + cache_read + cache_creation
- // - Claude streaming: extractUsage (after fix) sums input + cache_read + cache_creation
- // Do NOT add cache fields here — would double-count.
- const promptTokens = toFiniteNumber(tokenRecord.prompt_tokens);
- return promptTokens;
+ return 0;
}
export function getLoggedOutputTokens(tokens: unknown): number {
diff --git a/src/shared/validation/settingsSchemas.ts b/src/shared/validation/settingsSchemas.ts
index e278ab8173..9cefbab6bf 100644
--- a/src/shared/validation/settingsSchemas.ts
+++ b/src/shared/validation/settingsSchemas.ts
@@ -8,7 +8,7 @@
import { z } from "zod";
import { COMBO_CONFIG_MODES } from "@/shared/constants/comboConfigMode";
import { MAX_REQUEST_BODY_LIMIT_MB, MIN_REQUEST_BODY_LIMIT_MB } from "@/shared/constants/bodySize";
-import { HIDEABLE_SIDEBAR_ITEM_IDS, SIDEBAR_SECTIONS } from "@/shared/constants/sidebarVisibility";
+import { HIDEABLE_SIDEBAR_ITEM_IDS } from "@/shared/constants/sidebarVisibility";
import { ACCOUNT_FALLBACK_STRATEGY_VALUES } from "@/shared/constants/routingStrategies";
const signatureCacheModeValues = ["enabled", "bypass", "bypass-strict"] as const;
@@ -38,11 +38,6 @@ export const updateSettingsSchema = z.object({
autoRefreshProviderQuotaInterval: z.number().int().min(10).max(3600).optional(),
debugMode: z.boolean().optional(),
hiddenSidebarItems: z.array(z.enum(HIDEABLE_SIDEBAR_ITEM_IDS)).optional(),
- sidebarSectionOrder: z
- .array(z.enum(SIDEBAR_SECTIONS.map((s) => s.id) as [string, ...string[]]))
- .optional(),
- sidebarItemOrder: z.record(z.string(), z.array(z.string().max(100))).optional(),
- sidebarActivePreset: z.enum(["all", "minimal", "developer", "admin"]).nullable().optional(),
comboConfigMode: z.enum(COMBO_CONFIG_MODES).optional(),
codexServiceTier: z
.object({
@@ -316,7 +311,7 @@ export const databaseSettingsSchema = z.object(
// Aggregation settings
aggregation: z.object({
enabled: z.boolean(),
- rawDataRetentionDays: z.number().int().min(1).max(90),
+ rawDataRetentionDays: z.number().int().min(1).max(3650),
granularity: z.literal("hourly").or(z.literal("daily")).or(z.literal("weekly")),
}),
diff --git a/tests/unit/token-accounting-input-fix.test.ts b/tests/unit/token-accounting-input-fix.test.ts
index fb154286a2..8307b525eb 100644
--- a/tests/unit/token-accounting-input-fix.test.ts
+++ b/tests/unit/token-accounting-input-fix.test.ts
@@ -1,51 +1,15 @@
/**
* Unit tests for getLoggedInputTokens fix — Anthropic / anthropic-compatible-cc
*
- * The bug: Claude streaming sets prompt_tokens = input_tokens (non-cached only).
- * Fix: extractUsage in usageTracking.ts now sums input + cache_read + cache_creation
- * into prompt_tokens, consistent with the non-streaming extractor.
- *
- * getLoggedInputTokens itself also has a safety-net: when raw `input_tokens`
- * is present (e.g. from a raw API response), it adds cache tokens too.
+ * getLoggedInputTokens has a safety-net: when raw `input_tokens` is present
+ * (e.g. from a raw API response), it adds cache tokens too. When both
+ * `prompt_tokens` and `input_tokens` are present, `prompt_tokens` wins because
+ * stream translators keep `input_tokens` as a compatibility alias.
*/
import { describe, it } from "node:test";
import assert from "node:assert/strict";
-
-// ── Inline the fixed logic from tokenAccounting.ts ──────────────────────
-
-function asRecord(value) {
- return value && typeof value === "object" && !Array.isArray(value) ? value : {};
-}
-
-function toFiniteNumber(value) {
- if (typeof value === "number" && Number.isFinite(value)) return value;
- if (typeof value === "string" && value.trim().length > 0) {
- const parsed = Number(value);
- return Number.isFinite(parsed) ? parsed : 0;
- }
- return 0;
-}
-
-function getLoggedInputTokens(tokens) {
- const tokenRecord = asRecord(tokens);
-
- if (tokenRecord.input !== undefined && tokenRecord.input !== null) {
- return toFiniteNumber(tokenRecord.input);
- }
-
- if (tokenRecord.input_tokens !== undefined && tokenRecord.input_tokens !== null) {
- return (
- toFiniteNumber(tokenRecord.input_tokens) +
- toFiniteNumber(tokenRecord.cache_read_input_tokens) +
- toFiniteNumber(tokenRecord.cache_creation_input_tokens)
- );
- }
-
- // prompt_tokens from translator/extractor already includes cache tokens
- const promptTokens = toFiniteNumber(tokenRecord.prompt_tokens);
- return promptTokens;
-}
+import { getLoggedInputTokens } from "../../src/lib/usage/tokenAccounting.ts";
// ── Tests ────────────────────────────────────────────────────────────────
@@ -95,6 +59,17 @@ describe("getLoggedInputTokens — input fix for Anthropic streaming", () => {
assert.equal(getLoggedInputTokens(tokens), 113616);
});
+ it("translated Claude stream usage: prompt_tokens wins over compatibility input_tokens", () => {
+ const tokens = {
+ prompt_tokens: 600_000,
+ input_tokens: 600_000,
+ completion_tokens: 1_000,
+ output_tokens: 1_000,
+ cache_read_input_tokens: 600_000,
+ };
+ assert.equal(getLoggedInputTokens(tokens), 600_000);
+ });
+
it("OpenAI format: prompt_tokens=1000, no cache top-level fields → 1000", () => {
const tokens = {
prompt_tokens: 1000,