Skip to content

7.4 billion tokens per month. 34 free LLM providers. 635 free model endpoints. All behind one /v1 endpoint, plus any custom OpenAI-compatible endpoint. Smart routing, automatic failover, encrypted keys. Personal experimentation only.

TypeScript
42.4 KB
Raw Blame
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
import type { Express } from 'express';
import { createApp } from '../../app.js';
import { getDb, initDb } from '../../db/index.js';
import { logRequest } from '../../lib/request-log.js';
import { mintDashboardToken, isGatedApiPath } from '../helpers/auth.js';

let dashToken = '';

async function request(app: Express, path: string) {
  const server = app.listen(0, '127.0.0.1');
  if (!server.listening) await new Promise<void>(resolve => server.once('listening', () => resolve()));
  const addr = server.address() as any;
  const url = `http://127.0.0.1:${addr.port}${path}`;

  const res = await fetch(url, {
    headers: isGatedApiPath(path) ? { Authorization: `Bearer ${dashToken}` } : {},
  });
  const data = await res.json().catch(() => null);
  server.close();

  return { status: res.status, body: data };
}

function insertRequest(createdAt: string) {
  const db = getDb();
  db.prepare(`
    INSERT INTO requests (platform, model_id, status, input_tokens, output_tokens, latency_ms, error, created_at)
    VALUES ('test', 'test-model', 'success', 1, 2, 3, NULL, ?)
  `).run(createdAt);
  upsertAggregate(db, createdAt, 'success', 1, 2);
}

function insertTokensRequest(
  platform: string,
  modelId: string,
  status: 'success' | 'error',
  inputTokens: number,
  outputTokens: number,
  createdAt: string,
) {
  const db = getDb();
  db.prepare(`
    INSERT INTO requests (platform, model_id, status, input_tokens, output_tokens, latency_ms, error, created_at)
    VALUES (?, ?, ?, ?, ?, 3, NULL, ?)
  `).run(platform, modelId, status, inputTokens, outputTokens, createdAt);
  upsertAggregate(db, createdAt, status, inputTokens, outputTokens);
}

// Mirror the production aggregates written by lib/request-log.logRequest so the
// summary endpoint (which now reads from request_hourly + settings) stays
// faithful to what real traffic produces.
function upsertAggregate(
  db: ReturnType<typeof getDb>,
  createdAt: string,
  status: 'success' | 'error',
  inputTokens: number,
  outputTokens: number,
) {
  // Mirror logRequest.hourKey() exactly: created_at truncated to the hour in
  // SQLite's canonical 'YYYY-MM-DD HH:00:00' text (space separator). Using a 'T'
  // here would diverge from production and mask a writer/reader format mismatch.
  const hour = createdAt.slice(0, 13) + ':00:00';
  const isSuccess = status === 'success' ? 1 : 0;
  const isError = status === 'error' ? 1 : 0;
  db.prepare(`
    INSERT INTO request_hourly (hour, total_requests, success_count, error_count, input_tokens, output_tokens)
    VALUES (?, 1, ?, ?, ?, ?)
    ON CONFLICT(hour) DO UPDATE SET
      total_requests = total_requests + 1,
      success_count  = success_count + ?,
      error_count    = error_count + ?,
      input_tokens   = input_tokens + ?,
      output_tokens  = output_tokens + ?
  `).run(hour, isSuccess, isError, inputTokens, outputTokens, isSuccess, isError, inputTokens, outputTokens);

  const incr = db.prepare(`
    INSERT INTO settings (key, value) VALUES (?, ?)
    ON CONFLICT(key) DO UPDATE SET value = CAST(CAST(value AS INTEGER) + ? AS TEXT)
  `);
  incr.run('total_requests', '1', 1);
  incr.run('total_input_tokens', String(inputTokens), inputTokens);
  incr.run('total_output_tokens', String(outputTokens), outputTokens);
  db.prepare(`INSERT INTO settings (key, value) VALUES ('first_request_at', ?)
    ON CONFLICT(key) DO NOTHING`).run(createdAt);
}

describe('Analytics API', () => {
  let app: Express;

  beforeAll(() => {
    process.env.ENCRYPTION_KEY = '0'.repeat(64);
    initDb(':memory:');
    app = createApp();
    dashToken = mintDashboardToken();
  });

  beforeEach(() => {
    getDb().prepare('DELETE FROM requests').run();
    getDb().prepare('DELETE FROM request_hourly').run();
    getDb().prepare(`DELETE FROM settings WHERE key IN ('total_requests','total_input_tokens','total_output_tokens','first_request_at')`).run();
    vi.useFakeTimers();
    vi.setSystemTime(new Date('2026-05-29T12:00:00.000Z'));
  });

  afterEach(() => {
    vi.useRealTimers();
  });

  it('uses a rolling 24-hour window for summary analytics', async () => {
    insertRequest('2026-05-28 11:59:59');
    insertRequest('2026-05-28 12:00:00');
    insertRequest('2026-05-29 11:59:59');

    const { status, body } = await request(app, '/api/analytics/summary?range=24h');

    expect(status).toBe(200);
    expect(body.totalRequests).toBe(2);
    expect(body.totalInputTokens).toBe(2);
    expect(body.totalOutputTokens).toBe(4);
  });

  it.each([
    ['7d', '2026-05-22 11:59:59', '2026-05-22 12:00:00'],
    ['30d', '2026-04-29 11:59:59', '2026-04-29 12:00:00'],
  ])('uses a rolling %s window for summary analytics', async (range, outside, boundary) => {
    insertRequest(outside);
    insertRequest(boundary);
    insertRequest('2026-05-29 11:59:59');

    const { status, body } = await request(app, `/api/analytics/summary?range=${range}`);

    expect(status).toBe(200);
    expect(body.totalRequests).toBe(2);
  });

  // Regression guard for the hour-key FORMAT written by the real production
  // writer (lib/request-log.logRequest). The bug this prevents: the writer
  // stores keys as SQLite's 'YYYY-MM-DD HH:00:00' (space), but the summary
  // reader compared against a '...T...' cutoff, so every bucket on the window's
  // boundary day was silently dropped. The other summary tests seed via the
  // local upsertAggregate() helper; this one pins the writer's actual output so
  // the two can't drift apart unnoticed. Real timers so SQLite's datetime('now')
  // and getSinceTimestamp() agree on "now".
  it('logRequest writes space-format hour keys and they round-trip through summary', async () => {
    vi.useRealTimers();
    logRequest('groq', 'llama-3.3-70b-versatile', 0, 'success', 100, 50, 12, null);
    logRequest('groq', 'llama-3.3-70b-versatile', 0, 'success', 200, 70, 15, null);
    logRequest('groq', 'llama-3.3-70b-versatile', 0, 'error', 30, 0, 9, 'boom');

    // Tight, clock-independent guard: the stored key must match SQLite's
    // created_at text shape (space separator), never a 'T'. A 'T' here is the
    // exact desync that made the summary undercount the boundary day.
    const hours = getDb()
      .prepare('SELECT hour FROM request_hourly')
      .all() as Array<{ hour: string }>;
    expect(hours.length).toBeGreaterThanOrEqual(1); // normally 1 bucket; >1 only if the run straddled an hour tick
    for (const { hour } of hours) {
      expect(hour).toMatch(/^\d{4}-\d{2}-\d{2} \d{2}:00:00$/);
      expect(hour).not.toContain('T');
    }

    const { status, body } = await request(app, '/api/analytics/summary?range=24h');
    expect(status).toBe(200);
    expect(body.totalRequests).toBe(3);
    expect(body.totalInputTokens).toBe(330);
    expect(body.totalOutputTokens).toBe(120);
    expect(body.successRate).toBe(66.7);
    // Lifetime counter is window-independent; sourced from settings, not buckets.
    expect(body.lifetimeTotalRequests).toBe(3);
  });

  it("counts 'canceled' rows in totals but in no success/error rate (#752)", async () => {
    vi.useRealTimers();
    logRequest('groq', 'llama-3.3-70b-versatile', 0, 'success', 100, 50, 12, null);
    logRequest('groq', 'llama-3.3-70b-versatile', 0, 'error', 30, 0, 9, 'boom');
    logRequest('groq', 'llama-3.3-70b-versatile', 0, 'canceled', 0, 0, 800,
      'client disconnected after 0.8s; upstream request canceled');

    const summary = await request(app, '/api/analytics/summary?range=24h');
    expect(summary.status).toBe(200);
    expect(summary.body.totalRequests).toBe(3); // the canceled request happened
    expect(summary.body.successRate).toBe(50);  // 1 of 2 decided — not 1 of 3

    const byModel = await request(app, '/api/analytics/by-model?range=24h');
    expect(byModel.body[0].requests).toBe(3);
    expect(byModel.body[0].successRate).toBe(50);

    const byPlatform = await request(app, '/api/analytics/by-platform?range=24h');
    expect(byPlatform.body[0].successRate).toBe(50);
    expect(byPlatform.body[0].errorCount).toBe(1); // canceled is not an error
  });

  it('prices savings at the served model paid-equivalent rate', async () => {
    // groq/llama-3.3-70b-versatile is mapped at $0.10/M in, $0.32/M out
    // (db/model-pricing.ts): 10M in + 5M out → 1.00 + 1.60 = $2.60
    insertTokensRequest('groq', 'llama-3.3-70b-versatile', 'success', 10_000_000, 5_000_000, '2026-05-29 11:00:00');

    const { status, body } = await request(app, '/api/analytics/summary?range=24h');

    expect(status).toBe(200);
    expect(body.estimatedCostSavings).toBe(2.6);
    // Drives the client's span-based 30-day projection
    expect(body.firstRequestAt).toBe('2026-05-29 11:00:00');
  });

  it('falls back to modest default pricing for unmapped models', async () => {
    // Unknown model → $0.20/M in, $0.80/M out: 10M in + 5M out → 2.00 + 4.00 = $6.00
    insertTokensRequest('custom', 'mystery-model', 'success', 10_000_000, 5_000_000, '2026-05-29 11:00:00');

    const { status, body } = await request(app, '/api/analytics/summary?range=24h');

    expect(status).toBe(200);
    expect(body.estimatedCostSavings).toBe(6);
  });

  it('excludes failed requests from savings', async () => {
    insertTokensRequest('groq', 'llama-3.3-70b-versatile', 'error', 10_000_000, 0, '2026-05-29 11:00:00');

    const { status, body } = await request(app, '/api/analytics/summary?range=24h');

    expect(status).toBe(200);
    expect(body.estimatedCostSavings).toBe(0);
  });

  it('returns per-model estimated cost in the by-model breakdown', async () => {
    insertTokensRequest('groq', 'llama-3.3-70b-versatile', 'success', 10_000_000, 5_000_000, '2026-05-29 11:00:00');

    const { status, body } = await request(app, '/api/analytics/by-model?range=24h');

    expect(status).toBe(200);
    expect(body[0].estimatedCost).toBe(2.6);
  });

  describe('pinned vs auto tracking', () => {
    function insertPinnedRequest(modelId: string, requestedModel: string | null, createdAt: string) {
      getDb().prepare(`
        INSERT INTO requests (platform, model_id, requested_model, status, input_tokens, output_tokens, latency_ms, error, created_at)
        VALUES ('test', ?, ?, 'success', 1, 2, 3, NULL, ?)
      `).run(modelId, requestedModel, createdAt);
      upsertAggregate(getDb(), createdAt, 'success', 1, 2);
    }

    it('summary splits pinned, honored, and auto requests', async () => {
      insertPinnedRequest('model-a', 'model-a', '2026-05-29 11:00:00'); // pin honored
      insertPinnedRequest('model-b', 'model-a', '2026-05-29 11:01:00'); // pin overridden by failover
      insertPinnedRequest('model-b', null, '2026-05-29 11:02:00');      // auto-routed

      const { status, body } = await request(app, '/api/analytics/summary?range=24h');

      expect(status).toBe(200);
      expect(body.totalRequests).toBe(3);
      expect(body.pinnedRequests).toBe(2);
      expect(body.pinHonoredRequests).toBe(1);
    });

    it('by-model counts only requests the model served because it was pinned', async () => {
      insertPinnedRequest('model-a', 'model-a', '2026-05-29 11:00:00'); // pinned + served
      insertPinnedRequest('model-a', null, '2026-05-29 11:01:00');      // auto, same model
      insertPinnedRequest('model-a', 'model-x', '2026-05-29 11:02:00'); // failover landed here

      const { status, body } = await request(app, '/api/analytics/by-model?range=24h');

      expect(status).toBe(200);
      const row = body.find((r: any) => r.modelId === 'model-a');
      expect(row.requests).toBe(3);
      expect(row.pinnedRequests).toBe(1);
    });
  });

  // Raw-row insert covering the newer columns (ttfb_ms, request_type, key_id,
  // per-row latency). These feed the latency-percentile, TTFT, per-type, and
  // per-key analytics that only exist on the raw table. No aggregate upsert:
  // these tests assert the raw-scoped fields, not the hourly totals.
  function insertRaw(opts: {
    platform?: string;
    modelId?: string;
    keyId?: number | null;
    status?: 'success' | 'error';
    inputTokens?: number;
    outputTokens?: number;
    latencyMs?: number;
    ttfbMs?: number | null;
    requestType?: string;
    error?: string | null;
    createdAt: string;
  }) {
    const {
      platform = 'test',
      modelId = 'test-model',
      keyId = null,
      status = 'success',
      inputTokens = 0,
      outputTokens = 0,
      latencyMs = 0,
      ttfbMs = null,
      requestType = 'chat',
      error = null,
      createdAt,
    } = opts;
    getDb().prepare(`
      INSERT INTO requests (platform, model_id, key_id, status, input_tokens, output_tokens, latency_ms, ttfb_ms, request_type, error, created_at)
      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
    `).run(platform, modelId, keyId, status, inputTokens, outputTokens, latencyMs, ttfbMs, requestType, error, createdAt);
  }

  describe('extended summary fields', () => {
    it('returns latency percentiles from the raw rows', async () => {
      // Latencies 10..100 in a 24h window → p50 = 50, p95 = 90 (nearest-rank),
      // avg = 55.
      for (let ms = 10; ms <= 100; ms += 10) {
        insertRaw({ latencyMs: ms, createdAt: '2026-05-29 11:00:00' });
      }

      const { status, body } = await request(app, '/api/analytics/summary?range=24h');

      expect(status).toBe(200);
      expect(body.p50LatencyMs).toBe(50);
      expect(body.p95LatencyMs).toBe(90);
      expect(body.avgLatencyMs).toBe(55);
    });

    it('returns null percentiles and TTFT when the raw window is empty', async () => {
      const { status, body } = await request(app, '/api/analytics/summary?range=24h');

      expect(status).toBe(200);
      expect(body.p50LatencyMs).toBeNull();
      expect(body.p95LatencyMs).toBeNull();
      expect(body.avgTtfbMs).toBeNull();
    });

    it('averages TTFT over rows that recorded it and ignores NULL ttfb', async () => {
      insertRaw({ ttfbMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ ttfbMs: 200, createdAt: '2026-05-29 11:01:00' });
      insertRaw({ ttfbMs: null, createdAt: '2026-05-29 11:02:00' });

      const { status, body } = await request(app, '/api/analytics/summary?range=24h');

      expect(status).toBe(200);
      expect(body.avgTtfbMs).toBe(150);
    });

    it('splits requests into chat and embedding counts', async () => {
      insertRaw({ requestType: 'chat', createdAt: '2026-05-29 11:00:00' });
      insertRaw({ requestType: 'chat', createdAt: '2026-05-29 11:01:00' });
      insertRaw({ requestType: 'embedding', createdAt: '2026-05-29 11:02:00' });

      const { status, body } = await request(app, '/api/analytics/summary?range=24h');

      expect(status).toBe(200);
      expect(body.requestTypeCounts).toEqual({ chat: 2, embedding: 1 });
    });
  });

  describe('extended by-platform fields', () => {
    it('adds p95 latency, avg TTFT, error count, and tokens/sec per platform', async () => {
      // groq: 1 success (100ms, ttfb 20, 1000 out tok) + 1 error (300ms, ttfb 40).
      insertRaw({ platform: 'groq', status: 'success', outputTokens: 1000, latencyMs: 100, ttfbMs: 20, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'groq', status: 'error', outputTokens: 0, latencyMs: 300, ttfbMs: 40, error: 'boom', createdAt: '2026-05-29 11:01:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');

      expect(status).toBe(200);
      const groq = body.find((r: any) => r.platform === 'groq');
      expect(groq.errorCount).toBe(1);
      expect(groq.avgTtfbMs).toBe(30);
      // Only the success row qualifies (output>0 & latency>0): 1000 / 0.1 = 10000 tok/s.
      expect(groq.avgTokensPerSecond).toBe(10000);
      expect(typeof groq.p95LatencyMs).toBe('number');
    });

    it('reports null TTFT and tokens/sec when no rows qualify', async () => {
      insertRaw({ platform: 'nokey', status: 'success', outputTokens: 0, latencyMs: 0, ttfbMs: null, createdAt: '2026-05-29 11:00:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');

      expect(status).toBe(200);
      const row = body.find((r: any) => r.platform === 'nokey');
      expect(row.avgTtfbMs).toBeNull();
      expect(row.avgTokensPerSecond).toBeNull();
    });
  });

  describe('by-key endpoint', () => {
    it('groups usage per key, joins the label, and keeps deleted keys', async () => {
      getDb().prepare('DELETE FROM api_keys').run();
      getDb().prepare(`
        INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag)
        VALUES (1, 'groq', 'Prod key', 'x', 'x', 'x')
      `).run();

      // key 1 (exists): 3 rows, 2 success + 1 error, latency 100/200/300.
      insertRaw({ keyId: 1, status: 'success', inputTokens: 10, outputTokens: 5, latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ keyId: 1, status: 'success', inputTokens: 20, outputTokens: 7, latencyMs: 200, createdAt: '2026-05-29 11:01:00' });
      insertRaw({ keyId: 1, status: 'error', inputTokens: 0, outputTokens: 0, latencyMs: 300, error: 'boom', createdAt: '2026-05-29 11:02:00' });
      // key 99 (deleted — no api_keys row): 2 rows.
      insertRaw({ keyId: 99, status: 'success', latencyMs: 50, createdAt: '2026-05-29 11:03:00' });
      insertRaw({ keyId: 99, status: 'success', latencyMs: 50, createdAt: '2026-05-29 11:04:00' });
      // key_id NULL row must be excluded entirely.
      insertRaw({ keyId: null, status: 'success', createdAt: '2026-05-29 11:05:00' });

      const { status, body } = await request(app, '/api/analytics/by-key?range=24h');

      expect(status).toBe(200);
      expect(body).toHaveLength(2);

      const k1 = body.find((r: any) => r.keyId === 1);
      expect(k1.label).toBe('Prod key');
      expect(k1.platform).toBe('groq');
      expect(k1.requests).toBe(3);
      expect(k1.successRate).toBe(66.7);
      expect(k1.avgLatencyMs).toBe(200);
      expect(k1.totalInputTokens).toBe(30);
      expect(k1.totalOutputTokens).toBe(12);

      const k99 = body.find((r: any) => r.keyId === 99);
      expect(k99.label).toBeNull();
      expect(k99.platform).toBeNull();
      expect(k99.requests).toBe(2);
    });
  });

  describe('timeline tzOffset', () => {
    it('buckets timeline days by the viewer timezone instead of UTC', async () => {
      insertRequest('2026-05-28 15:30:00'); // UTC day 05-28; UTC+8 23:30, still day 05-28
      insertRequest('2026-05-28 17:30:00'); // UTC day 05-28; UTC+8 01:30 on day 05-29

      const utc = await request(app, '/api/analytics/timeline?range=7d');
      expect(utc.status).toBe(200);
      expect(utc.body.map((b: any) => b.timestamp)).toEqual(['2026-05-28']);

      const local = await request(app, '/api/analytics/timeline?range=7d&tzOffset=480');
      expect(local.status).toBe(200);
      expect(local.body.map((b: any) => b.timestamp)).toEqual(['2026-05-28', '2026-05-29']);
      expect(local.body.map((b: any) => b.requests)).toEqual([1, 1]);
    });

    it('shifts hour labels to the viewer timezone', async () => {
      insertRequest('2026-05-28 17:30:00'); // UTC 17:00 → UTC+8 next-day 01:00

      const { status, body } = await request(app, '/api/analytics/timeline?range=24h&tzOffset=480');
      expect(status).toBe(200);
      expect(body.map((b: any) => b.timestamp)).toEqual(['2026-05-29T01:00:00']);
    });

    it('falls back to UTC bucketing for an invalid tzOffset', async () => {
      insertRequest('2026-05-28 17:30:00');

      for (const bad of ['abc', '9999', '1.5', '480; DROP TABLE request_hourly']) {
        const { status, body } = await request(app, `/api/analytics/timeline?range=7d&tzOffset=${encodeURIComponent(bad)}`);
        expect(status).toBe(200);
        expect(body.map((b: any) => b.timestamp)).toEqual(['2026-05-28']);
      }
    });
  });

  describe('90d range', () => {
    it('accepts range=90d across the analytics endpoints', async () => {
      insertRequest('2026-02-01 12:00:00'); // ~117 days ago — outside 90d
      insertRequest('2026-03-15 12:00:00'); // ~75 days ago — inside 90d
      insertRequest('2026-05-29 11:00:00'); // today — inside 90d

      const summary = await request(app, '/api/analytics/summary?range=90d');
      expect(summary.status).toBe(200);
      expect(summary.body.totalRequests).toBe(2);

      const timeline = await request(app, '/api/analytics/timeline?range=90d');
      expect(timeline.status).toBe(200);
      // Day-bucketed for 90d; the two in-window rows land on two days.
      expect(Array.isArray(timeline.body)).toBe(true);
      expect(timeline.body.every((b: any) => 'inputTokens' in b && 'outputTokens' in b)).toBe(true);

      const byPlatform = await request(app, '/api/analytics/by-platform?range=90d');
      expect(byPlatform.status).toBe(200);

      const byKey = await request(app, '/api/analytics/by-key?range=90d');
      expect(byKey.status).toBe(200);
    });
  });

  // #889 — custom endpoints all share the platform id 'custom', so the
  // provider breakdown must split them by the serving key's base_url (the
  // canonical endpoint identity) instead of collapsing every relay into one
  // "custom" row. These tests seed two distinct custom endpoints (plus a
  // pooled second key on the first) and assert the split, the per-endpoint
  // p95 scoping, and the recent-calls provider filter.
  describe('custom endpoint identifiers (#889)', () => {
    // Seed a custom endpoint key and return its id. base_url is the endpoint
    // identity; the credential itself is a placeholder (never read here).
    function insertCustomKey(id: number, baseUrl: string, label: string): void {
      getDb().prepare(`
        INSERT INTO api_keys (id, platform, label, encrypted_key, iv, auth_tag, base_url)
        VALUES (?, 'custom', ?, 'x', 'x', 'x', ?)
      `).run(id, label, baseUrl);
    }

    // A model row belonging to ONE endpoint (#651): `models` is unique on
    // (platform, model_id, endpoint_scope), so two relays can each hold their
    // own row for the same model id.
    function insertScopedModel(modelId: string, displayName: string, endpointScope: string): void {
      getDb().prepare(`
        INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, endpoint_scope)
        VALUES ('custom', ?, ?, 50, 50, ?)
      `).run(modelId, displayName, endpointScope);
    }

    beforeEach(() => {
      getDb().prepare('DELETE FROM api_keys').run();
      // Only endpoint-scoped rows; the seeded catalog ('' scope) stays put.
      getDb().prepare("DELETE FROM models WHERE endpoint_scope <> ''").run();
    });

    it('splits custom endpoints into per-endpoint rows instead of one "custom" row', async () => {
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertCustomKey(2, 'https://relay-b.example.com/v1', 'Relay B');

      // Two requests on relay A, one on relay B.
      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 120, createdAt: '2026-05-29 11:01:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'success', latencyMs: 900, createdAt: '2026-05-29 11:02:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      // Two distinct custom rows — NOT a single collapsed "custom" row.
      const customRows = body.filter((r: any) => r.platform === 'custom');
      expect(customRows).toHaveLength(2);

      const a = body.find((r: any) => r.providerId === 'custom:https://relay-a.example.com/v1');
      const b = body.find((r: any) => r.providerId === 'custom:https://relay-b.example.com/v1');
      expect(a).toBeDefined();
      expect(b).toBeDefined();
      expect(a.requests).toBe(2);
      expect(b.requests).toBe(1);
      // The operator-readable identifier is the endpoint host.
      expect(a.endpoint).toBe('relay-a.example.com');
      expect(b.endpoint).toBe('relay-b.example.com');
      // Each row keeps the raw platform id for the platform-dot coloring.
      expect(a.platform).toBe('custom');
      expect(b.platform).toBe('custom');
    });

    it('groups a pooled endpoint (several keys, one base_url) into a single row', async () => {
      // Two credentials for the SAME relay — the pooling case (#619). They
      // share a base_url, so they must read as ONE endpoint, not two rows.
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A key 1');
      insertCustomKey(2, 'https://relay-a.example.com/v1', 'Relay A key 2');

      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'success', latencyMs: 110, createdAt: '2026-05-29 11:01:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      const customRows = body.filter((r: any) => r.platform === 'custom');
      expect(customRows).toHaveLength(1);
      expect(customRows[0].providerId).toBe('custom:https://relay-a.example.com/v1');
      expect(customRows[0].requests).toBe(2);
    });

    it('scopes p95 latency per endpoint, not across all custom traffic', async () => {
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertCustomKey(2, 'https://relay-b.example.com/v1', 'Relay B');

      // Relay A: fast (100ms x10). Relay B: slow (1000ms x10). If p95 bled
      // across the whole "custom" platform, A's p95 would be ~1000, not 100.
      for (let i = 0; i < 10; i++) {
        insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 100, createdAt: `2026-05-29 10:${String(i).padStart(2, '0')}:00` });
      }
      for (let i = 0; i < 10; i++) {
        insertRaw({ platform: 'custom', keyId: 2, status: 'success', latencyMs: 1000, createdAt: `2026-05-29 10:${String(i).padStart(2, '0')}:30` });
      }

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      const a = body.find((r: any) => r.providerId === 'custom:https://relay-a.example.com/v1');
      const b = body.find((r: any) => r.providerId === 'custom:https://relay-b.example.com/v1');
      expect(a.p95LatencyMs).toBe(100);
      expect(b.p95LatencyMs).toBe(1000);
    });

    it('keeps catalog providers on their bare platform id (no custom: prefix)', async () => {
      insertRaw({ platform: 'groq', keyId: null, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      const groq = body.find((r: any) => r.platform === 'groq');
      expect(groq.providerId).toBe('groq');
      expect(groq.endpoint).toBe('groq');
    });

    it('falls back to the plain "custom" id when the key was deleted', async () => {
      // A custom request whose key no longer exists: base_url is unknown, so
      // the row keeps the pre-fix "custom" shape rather than a fake host.
      insertRaw({ platform: 'custom', keyId: 99, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      const row = body.find((r: any) => r.platform === 'custom');
      expect(row.providerId).toBe('custom');
      expect(row.endpoint).toBe('custom');
    });

    it('filters recent calls by a custom endpoint providerId', async () => {
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertCustomKey(2, 'https://relay-b.example.com/v1', 'Relay B');

      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 120, createdAt: '2026-05-29 11:01:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'success', latencyMs: 900, createdAt: '2026-05-29 11:02:00' });

      const a = await request(app, '/api/analytics/requests?range=24h&provider=' + encodeURIComponent('custom:https://relay-a.example.com/v1'));
      expect(a.status).toBe(200);
      expect(a.body.total).toBe(2);
      expect(a.body.rows.every((r: any) => r.keyLabel === 'Relay A')).toBe(true);

      const b = await request(app, '/api/analytics/requests?range=24h&provider=' + encodeURIComponent('custom:https://relay-b.example.com/v1'));
      expect(b.status).toBe(200);
      expect(b.body.total).toBe(1);
      expect(b.body.rows[0].keyLabel).toBe('Relay B');
    });

    it('tells two endpoints on the same host apart', async () => {
      // One gateway, two tenants. Host alone is the same string for both, so a
      // host-only display name would show two identical rows the operator
      // cannot match to an endpoint — the #889 collision one level down.
      insertCustomKey(1, 'https://gw.example.com/tenant-a/v1', 'Tenant A');
      insertCustomKey(2, 'https://gw.example.com/tenant-b/v1', 'Tenant B');

      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'success', latencyMs: 200, createdAt: '2026-05-29 11:01:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      const rows = body.filter((r: any) => r.platform === 'custom');
      expect(rows).toHaveLength(2);
      const names = rows.map((r: any) => r.endpoint).sort();
      expect(names).toEqual(['gw.example.com/tenant-a/v1', 'gw.example.com/tenant-b/v1']);
      // Distinct names, one per endpoint — not 'gw.example.com' twice.
      expect(new Set(names).size).toBe(2);
    });

    it('groups an endpoint stored with and without a trailing slash as one row', async () => {
      // keys.ts normalizes base_url on write, but rows written before it did
      // carry the raw string. Same endpoint, so: one row, one id.
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A key 1');
      insertCustomKey(2, 'https://relay-a.example.com/v1/', 'Relay A key 2');

      insertRaw({ platform: 'custom', keyId: 1, status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'success', latencyMs: 110, createdAt: '2026-05-29 11:01:00' });

      const { status, body } = await request(app, '/api/analytics/by-platform?range=24h');
      expect(status).toBe(200);

      const rows = body.filter((r: any) => r.platform === 'custom');
      expect(rows).toHaveLength(1);
      expect(rows[0].providerId).toBe('custom:https://relay-a.example.com/v1');
      expect(rows[0].requests).toBe(2);
    });

    it('names the endpoint behind each recent error', async () => {
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertCustomKey(2, 'https://relay-b.example.com/v1', 'Relay B');

      insertRaw({ platform: 'custom', keyId: 1, status: 'error', error: 'relay A exploded', createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'error', error: 'relay B exploded', createdAt: '2026-05-29 11:01:00' });
      insertRaw({ platform: 'groq', keyId: null, status: 'error', error: 'groq exploded', createdAt: '2026-05-29 11:02:00' });

      const { status, body } = await request(app, '/api/analytics/errors?range=24h');
      expect(status).toBe(200);

      const a = body.find((r: any) => r.error === 'relay A exploded');
      const b = body.find((r: any) => r.error === 'relay B exploded');
      const groq = body.find((r: any) => r.error === 'groq exploded');
      // Each error says WHICH relay failed instead of a bare 'custom'.
      expect(a.providerId).toBe('custom:https://relay-a.example.com/v1');
      expect(a.endpoint).toBe('relay-a.example.com');
      expect(b.providerId).toBe('custom:https://relay-b.example.com/v1');
      expect(b.endpoint).toBe('relay-b.example.com');
      // The platform slug is still there for the client's dot coloring.
      expect(a.platform).toBe('custom');
      // Catalog providers are untouched.
      expect(groq.providerId).toBe('groq');
      expect(groq.endpoint).toBe('groq');
      // An orphaned custom error keeps the pre-fix shape.
      insertRaw({ platform: 'custom', keyId: 99, status: 'error', error: 'orphan exploded', createdAt: '2026-05-29 11:03:00' });
      const again = await request(app, '/api/analytics/errors?range=24h');
      const orphan = again.body.find((r: any) => r.error === 'orphan exploded');
      expect(orphan.providerId).toBe('custom');
      expect(orphan.endpoint).toBe('custom');
    });

    it('splits one model id across the relays that served it', async () => {
      // The same model name behind two relays is two different services with
      // two different latencies; one merged 'custom' row describes neither.
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertCustomKey(2, 'https://relay-b.example.com/v1', 'Relay B');
      insertScopedModel('shared-model', 'Shared (A)', 'https://relay-a.example.com/v1');
      insertScopedModel('shared-model', 'Shared (B)', 'https://relay-b.example.com/v1');

      insertRaw({ platform: 'custom', keyId: 1, modelId: 'shared-model', status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 1, modelId: 'shared-model', status: 'success', latencyMs: 100, createdAt: '2026-05-29 11:01:00' });
      insertRaw({ platform: 'custom', keyId: 2, modelId: 'shared-model', status: 'success', latencyMs: 900, createdAt: '2026-05-29 11:02:00' });

      const { status, body } = await request(app, '/api/analytics/by-model?range=24h');
      expect(status).toBe(200);

      const rows = body.filter((r: any) => r.modelId === 'shared-model');
      expect(rows).toHaveLength(2);

      const a = rows.find((r: any) => r.providerId === 'custom:https://relay-a.example.com/v1');
      const b = rows.find((r: any) => r.providerId === 'custom:https://relay-b.example.com/v1');
      expect(a.requests).toBe(2);
      expect(b.requests).toBe(1);
      // Counts are per endpoint, not the fan-out of one endpoint's requests
      // across every models row that shares the id.
      expect(a.requests + b.requests).toBe(3);
      expect(a.endpoint).toBe('relay-a.example.com');
      expect(b.endpoint).toBe('relay-b.example.com');
      // Each row's latency is its own endpoint's, not the pooled average.
      expect(a.avgLatencyMs).toBe(100);
      expect(b.avgLatencyMs).toBe(900);
      // The display name comes from the model row of the serving endpoint.
      expect(a.displayName).toBe('Shared (A)');
      expect(b.displayName).toBe('Shared (B)');
    });

    it('keeps catalog models on one row per platform in by-model', async () => {
      insertRaw({ platform: 'groq', keyId: null, modelId: 'llama-3.1-8b-instant', status: 'success', latencyMs: 50, createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'groq', keyId: null, modelId: 'llama-3.1-8b-instant', status: 'success', latencyMs: 70, createdAt: '2026-05-29 11:01:00' });

      const { status, body } = await request(app, '/api/analytics/by-model?range=24h');
      expect(status).toBe(200);

      const rows = body.filter((r: any) => r.platform === 'groq');
      expect(rows).toHaveLength(1);
      expect(rows[0].requests).toBe(2);
      expect(rows[0].providerId).toBe('groq');
      expect(rows[0].endpoint).toBe('groq');
    });

    it('splits the error distribution per custom endpoint', async () => {
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertCustomKey(2, 'https://relay-b.example.com/v1', 'Relay B');

      insertRaw({ platform: 'custom', keyId: 1, status: 'error', error: '429 rate limit', createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 1, status: 'error', error: '429 rate limit', createdAt: '2026-05-29 11:01:00' });
      insertRaw({ platform: 'custom', keyId: 2, status: 'error', error: 'timeout', createdAt: '2026-05-29 11:02:00' });
      insertRaw({ platform: 'custom', keyId: 1, status: 'success', createdAt: '2026-05-29 11:03:00' });

      const { status, body } = await request(app, '/api/analytics/error-distribution?range=24h');
      expect(status).toBe(200);

      const custom = body.byPlatform.filter((r: any) => r.platform === 'custom');
      // Two bars, one per relay — not one 'custom' bar of 3 that names no
      // endpoint the operator could go and fix.
      expect(custom).toHaveLength(2);
      const a = custom.find((r: any) => r.providerId === 'custom:https://relay-a.example.com/v1');
      const b = custom.find((r: any) => r.providerId === 'custom:https://relay-b.example.com/v1');
      expect(a.count).toBe(2);
      expect(a.endpoint).toBe('relay-a.example.com');
      expect(b.count).toBe(1);
      expect(b.endpoint).toBe('relay-b.example.com');
      // Category totals are endpoint-agnostic and must not change shape.
      expect(body.byCategory.find((c: any) => c.category === 'Rate Limited (429)').count).toBe(2);
    });

    it('classifies capitalized upstream auth and timeout messages out of Other', async () => {
      // The messages upstream providers actually send. LIKE is case-sensitive
      // in SQLite and the old '%invalid.*key%' pattern was regex syntax inside
      // a LIKE (where '.' and '*' are literals), so "Invalid API key" and
      // "Timeout" both fell into 'Other' and the dashboard hid auth failures.
      insertRaw({ status: 'error', error: 'Invalid API key: sk-broken', createdAt: '2026-05-29 11:00:00' });
      insertRaw({ status: 'error', error: 'Unauthorized: token expired', createdAt: '2026-05-29 11:01:00' });
      insertRaw({ status: 'error', error: 'Too Many Requests', createdAt: '2026-05-29 11:02:00' });
      insertRaw({ status: 'error', error: 'Timeout awaiting response headers', createdAt: '2026-05-29 11:03:00' });
      insertRaw({ status: 'error', error: 'ETIMEDOUT after 30000ms', createdAt: '2026-05-29 11:04:00' });
      // "timed out" is how our own abort paths word it; the '%500%' rule used to
      // claim this one via the "5000ms" in the message.
      insertRaw({ status: 'error', error: 'fusion tool call timed out after 5000ms', createdAt: '2026-05-29 11:05:00' });

      const { status, body } = await request(app, '/api/analytics/error-distribution?range=24h');
      expect(status).toBe(200);

      const cat = (name: string) => body.byCategory.find((c: any) => c.category === name)?.count ?? 0;
      expect(cat('Auth Error (401)')).toBe(2);
      expect(cat('Rate Limited (429)')).toBe(1);
      expect(cat('Timeout/Connection')).toBe(3);
      expect(cat('Other')).toBe(0);
    });

    it('filters recent calls by the bare "custom" id to the orphaned rows only', async () => {
      // The bare id is what /by-platform emits for rows whose endpoint is
      // unknown. Selecting it must return exactly the rows that row counted —
      // not every relay's traffic, which would contradict the count clicked.
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');

      insertRaw({ platform: 'custom', keyId: 1, status: 'success', createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'custom', keyId: 1, status: 'success', createdAt: '2026-05-29 11:01:00' });
      // Key deleted after the fact (dangling id) and a row that never had one.
      insertRaw({ platform: 'custom', keyId: 99, status: 'success', createdAt: '2026-05-29 11:02:00' });
      insertRaw({ platform: 'custom', keyId: null, status: 'success', createdAt: '2026-05-29 11:03:00' });

      const platforms = await request(app, '/api/analytics/by-platform?range=24h');
      const orphanRow = platforms.body.find((r: any) => r.providerId === 'custom');
      expect(orphanRow.requests).toBe(2);

      const filtered = await request(app, '/api/analytics/requests?range=24h&provider=custom');
      expect(filtered.status).toBe(200);
      // The list agrees with the row: 2, not all 4 custom requests.
      expect(filtered.body.total).toBe(orphanRow.requests);
      expect(filtered.body.rows.every((r: any) => r.keyLabel === null)).toBe(true);

      // The named endpoint still filters to itself...
      const relayA = await request(app, '/api/analytics/requests?range=24h&provider=' + encodeURIComponent('custom:https://relay-a.example.com/v1'));
      expect(relayA.body.total).toBe(2);
      // ...and the legacy platform param keeps its pre-#889 meaning: all of it.
      const legacy = await request(app, '/api/analytics/requests?range=24h&platform=custom');
      expect(legacy.body.total).toBe(4);
    });

    it('still accepts the legacy platform filter and rejects malformed provider ids', async () => {
      insertCustomKey(1, 'https://relay-a.example.com/v1', 'Relay A');
      insertRaw({ platform: 'custom', keyId: 1, status: 'success', createdAt: '2026-05-29 11:00:00' });
      insertRaw({ platform: 'groq', keyId: null, status: 'success', createdAt: '2026-05-29 11:01:00' });

      // Legacy platform filter still works.
      const legacy = await request(app, '/api/analytics/requests?range=24h&platform=groq');
      expect(legacy.status).toBe(200);
      expect(legacy.body.total).toBe(1);

      // A 'custom:' id carries an arbitrary base_url, but it is applied as a
      // BOUND parameter (never interpolated), so a hostile-looking value is
      // safe — it simply matches nothing.
      const customNoMatch = await request(app, '/api/analytics/requests?range=24h&provider=' + encodeURIComponent('custom:https://relay-a.example.com/v1; DROP TABLE requests'));
      expect(customNoMatch.status).toBe(200);
      expect(customNoMatch.body.total).toBe(0);

      // Non-custom ids must be short slugs; anything else is a client bug.
      for (const bad of ['has spaces', 'a\nb']) {
        const res = await request(app, '/api/analytics/requests?range=24h&provider=' + encodeURIComponent(bad));
        expect(res.status).toBe(400);
      }
    });

    it('takes the FIRST value of a repeated provider/status param instead of crashing', async () => {
      // Express hands back an ARRAY for a repeated query param. The provider
      // shape check then dies on `provider.startsWith(...)` — arrays have no
      // such method — and the dashboard's recent-calls panel shows a 500 for
      // what is at worst a client bug. Same contract as routes/logs.ts:
      // first value wins, repeated params are not two filters.
      const dup = await request(app, '/api/analytics/requests?range=24h&provider=groq&provider=openrouter');
      expect(dup.status).toBe(200);
      const badSecond = await request(app, '/api/analytics/requests?range=24h&provider=groq&provider=' + encodeURIComponent('has spaces'));
      expect(badSecond.status).toBe(200);
      const dupStatus = await request(app, '/api/analytics/requests?range=24h&status=success&status=error');
      expect(dupStatus.status).toBe(200);
    });
  });
});