Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

21.9 KB
Raw Blame
import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';

vi.mock('./runner.js', () => ({ createRun: vi.fn(), finalizeRunRecord: vi.fn() }));
vi.mock('./localLlm.js', () => ({ ensureBackendProvider: vi.fn(() => Promise.resolve()) }));
vi.mock('./providers.js', () => ({ getProviderById: vi.fn() }));
vi.mock('./providerStatus.js', () => ({ markProviderAvailable: vi.fn(() => Promise.resolve()) }));
vi.mock('./providerExecutionReadiness.js', () => ({ ensureManagedRuntimeReady: vi.fn(() => Promise.resolve({ success: true })) }));

import { buildPrompt, summarizeTimings, runLocalLlmTest } from './localLlmPlayground.js';
import { __resetUsageSupport } from '../lib/openAiChatStream.js';
import { createRun, finalizeRunRecord } from './runner.js';
import { getProviderById } from './providers.js';

describe('buildPrompt', () => {
  it('returns the bare prompt when no system instructions', () => {
    expect(buildPrompt({ systemPrompt: '', prompt: 'hi' })).toBe('hi');
    expect(buildPrompt({ systemPrompt: '   ', prompt: 'hi' })).toBe('hi');
  });

  it('prefixes a labeled system block when present (display format, not wire)', () => {
    expect(buildPrompt({ systemPrompt: 'Be terse', prompt: 'hi' }))
      .toBe('System instructions:\nBe terse\n\nUser prompt:\nhi');
  });
});

describe('summarizeTimings', () => {
  it('computes ttft, total, and rate', () => {
    const t = summarizeTimings({ startedAt: 1000, firstChunkAt: 1200, endedAt: 3000, text: 'abcdefghij' });
    expect(t.ttftMs).toBe(200);
    expect(t.totalMs).toBe(2000);
    expect(t.chars).toBe(10);
    expect(t.charsPerSecond).toBe(5); // 10 chars / 2s
  });

  it('reports null ttft when no chunk ever arrived', () => {
    const t = summarizeTimings({ startedAt: 1000, firstChunkAt: null, endedAt: 2000, text: '' });
    expect(t.ttftMs).toBeNull();
  });

  it('reports null rate (not a char count) for a zero-duration run', () => {
    const t = summarizeTimings({ startedAt: 1000, firstChunkAt: 1000, endedAt: 1000, text: 'hello' });
    expect(t.totalMs).toBe(0);
    expect(t.charsPerSecond).toBeNull();
  });

  // tokens/s is DECODE throughput — completion tokens over the time after the
  // first token — matching what llama.cpp and Ollama report. Dividing by the
  // full wall clock would make the same model look slower purely because its
  // prompt was longer, which is what TTFT and the prefill rate are for.
  it('measures tokens/s over the decode window, not the whole wall clock', () => {
    const t = summarizeTimings({
      startedAt: 0, firstChunkAt: 1000, endedAt: 3000, text: 'abc',
      usage: { completionTokens: 100, promptTokens: 4000, estimated: false },
    });
    expect(t.completionTokens).toBe(100);
    expect(t.tokensPerSecond).toBe(50); // 100 tokens / 2s of decode, not / 3s
    expect(t.promptTokensPerSecond).toBe(4000); // 4000 prompt tokens / 1s of prefill
    expect(t.tokensEstimated).toBe(false);
  });

  it('uses wall clock for a one-chunk response when no runtime decode duration exists', () => {
    const t = summarizeTimings({
      startedAt: 0, firstChunkAt: 1000, endedAt: 3000, text: 'abc', streamChunks: 1,
      usage: { completionTokens: 100, promptTokens: 4000, estimated: false },
    });
    expect(t.tokensPerSecond).toBe(33.33);
    expect(t.timingSource).toBe('wall-clock');
  });

  it('prefers native decode and prefill durations when the runtime reports them', () => {
    const t = summarizeTimings({
      startedAt: 0, firstChunkAt: 1000, endedAt: 3000, text: 'abc',
      usage: {
        completionTokens: 100, promptTokens: 4000, completionMs: 500, promptMs: 250, estimated: false,
      },
    });
    expect(t.decodeMs).toBe(500);
    expect(t.promptMs).toBe(250);
    expect(t.tokensPerSecond).toBe(200);
    expect(t.promptTokensPerSecond).toBe(16000);
  });

  it('marks a frame-counted token figure as an estimate', () => {
    const t = summarizeTimings({
      startedAt: 0, firstChunkAt: 500, endedAt: 1500, text: 'abc',
      usage: { completionTokens: 20, promptTokens: null, estimated: true },
    });
    expect(t.tokensEstimated).toBe(true);
    expect(t.tokensPerSecond).toBe(20);
    // No prompt count reported → no prefill rate invented from the char count.
    expect(t.promptTokensPerSecond).toBeNull();
  });

  // The sentinel rule: a caller that tracks no usage records "not measured",
  // never a tokens/s figure derived from characters.
  it('reports null token fields when the daemon reported no usage', () => {
    const t = summarizeTimings({ startedAt: 0, firstChunkAt: 100, endedAt: 1100, text: 'abcdefgh' });
    expect(t.completionTokens).toBeNull();
    expect(t.promptTokens).toBeNull();
    expect(t.tokensPerSecond).toBeNull();
    expect(t.tokensEstimated).toBeNull();
    // ...while the chars/s figure it CAN measure is unaffected.
    expect(t.charsPerSecond).toBe(7.27); // 8 chars / 1.1s wall clock
  });
});

// Build a fake stream reader: yields each SSE line as a chunk, then either
// finishes cleanly (done) or throws an AbortError to simulate a timeout.
function makeReader(lines, { abort = false } = {}) {
  let i = 0;
  return {
    read: vi.fn(async () => {
      if (i < lines.length) return { done: false, value: new TextEncoder().encode(lines[i++]) };
      if (abort) {
        const err = new Error('The operation was aborted');
        err.name = 'AbortError';
        throw err;
      }
      return { done: true, value: undefined };
    }),
    cancel: vi.fn(async () => {}),
  };
}

const sse = (delta) => `data: ${JSON.stringify({ choices: [{ delta }] })}\n`;

describe('runLocalLlmTest timeout/abort contract', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    getProviderById.mockResolvedValue({ id: 'lmstudio', type: 'api', endpoint: 'http://localhost:1234/v1' });
    createRun.mockResolvedValue({ runId: 'run-1', provider: { id: 'lmstudio' } });
    finalizeRunRecord.mockResolvedValue(undefined);
  });

  afterEach(() => { delete global.fetch; });

  const stubStream = (reader) => {
    global.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, body: { getReader: () => reader } });
  };

  it.each([
    { numCtx: 16384, extraBody: {} },
    { numCtx: 4096, extraBody: { num_ctx: 16384 } },
  ])('honors the configured or caller-overridden Ollama context on normal playground runs: %j', async ({ numCtx, extraBody }) => {
    getProviderById.mockResolvedValue({ id: 'ollama', type: 'api', endpoint: 'http://localhost:11434/v1', numCtx });
    createRun.mockResolvedValue({ runId: 'run-1', provider: { id: 'ollama' } });
    stubStream(makeReader([`${JSON.stringify({ message: { content: 'Done.' }, done: true, done_reason: 'stop', prompt_eval_count: 10, eval_count: 2 })}\n`]));
    const tokens = [];
    const result = await runLocalLlmTest({ backend: 'ollama', modelId: 'm1', prompt: 'hi', maxTokens: 32, extraBody, onToken: delta => tokens.push(delta) });
    expect(result.error).toBeUndefined();
    expect(result.text).toBe('Done.');
    expect(tokens).toEqual(['Done.']);
    const [url, request] = global.fetch.mock.calls[0];
    expect(url).toBe('http://localhost:11434/api/chat');
    expect(JSON.parse(request.body)).toMatchObject({ model: 'm1', stream: true, options: { num_ctx: 16384, num_predict: 32 } });
    expect(finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({ output: 'Done.', success: true }));
  });

  it('keeps the OpenAI transport for Ollama without a context override and for other backends', async () => {
    for (const [backend, numCtx] of [['ollama', undefined], ['lmstudio', 16384]]) {
      const endpoint = backend === 'ollama' ? 'http://localhost:11434/v1' : 'http://localhost:1234/v1';
      getProviderById.mockResolvedValue({ id: backend, type: 'api', endpoint, numCtx });
      createRun.mockResolvedValue({ runId: 'run-1', provider: { id: backend } });
      stubStream(makeReader([sse({ content: 'Done.' })]));
      const result = await runLocalLlmTest({ backend, modelId: 'm1', prompt: 'hi', maxTokens: 32 });
      expect(result.error).toBeUndefined();
      const [url, request] = global.fetch.mock.calls[0];
      expect(url).toBe(`${endpoint}/chat/completions`);
      expect(JSON.parse(request.body)).toEqual({
        model: 'm1', messages: [{ role: 'user', content: 'hi' }], stream: true,
        temperature: 0.3, max_tokens: 32, stream_options: { include_usage: true },
        ...(numCtx ? { num_ctx: numCtx } : {}),
      });
    }
  });

  it('returns the content streamed before an abort as text and persists it on the failed run', async () => {
    stubStream(makeReader([sse({ content: 'Hello ' }), sse({ content: 'world' })], { abort: true }));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(result.error).toContain('Timed out');
    expect(result.text).toBe('Hello world');
    expect(finalizeRunRecord).toHaveBeenCalledWith(
      expect.objectContaining({ runId: 'run-1', output: 'Hello world', success: false, exitCode: 1 }),
    );
  });

  // A big local model that needs longer than the caller allowed is not a broken
  // provider: the partial text proves it was streaming, `timeoutMs` is the
  // caller's own knob, and the error comes straight back inline. Reporting it
  // through the host's provider-failure hook filed a CoS investigation task per
  // slow playground run.
  it('does not report a run cut off by its own deadline as a provider failure', async () => {
    stubStream(makeReader([sse({ content: 'Hello ' })], { abort: true }));

    await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({ reportFailure: false }));
  });

  it('still reports a genuine provider failure through the host hook', async () => {
    global.fetch = vi.fn().mockRejectedValue(new Error('fetch failed: ECONNREFUSED'));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(result.error).toContain('ECONNREFUSED');
    expect(finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({ reportFailure: true }));
  });

  it('surfaces a reasoning-only partial when no visible content streamed before the abort', async () => {
    stubStream(makeReader([sse({ reasoning: 'thinking hard…' })], { abort: true }));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(result.error).toContain('Timed out');
    expect(result.text).toBe('thinking hard…');
    expect(finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({ output: 'thinking hard…', success: false }));
  });

  it('returns the full text and a success record when the stream finishes normally', async () => {
    stubStream(makeReader([sse({ content: 'Done.' })], { abort: false }));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(result.error).toBeUndefined();
    expect(result.text).toBe('Done.');
    expect(finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({ output: 'Done.', success: true, exitCode: 0 }));
  });

  it('forwards each content delta to onToken as it streams (streaming route)', async () => {
    stubStream(makeReader([sse({ content: 'Hel' }), sse({ content: 'lo' })], { abort: false }));
    const tokens = [];

    const result = await runLocalLlmTest({
      backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000,
      onToken: (delta, kind) => tokens.push([delta, kind]),
    });

    expect(tokens).toEqual([['Hel', 'content'], ['lo', 'content']]);
    expect(result.text).toBe('Hello');
  });

  it('forwards reasoning deltas live on the reasoning channel, content on the content channel', async () => {
    stubStream(makeReader([
      sse({ reasoning: 'let me ' }),
      sse({ reasoning: 'think…' }),
      sse({ content: 'Answer.' }),
    ], { abort: false }));
    const tokens = [];

    const result = await runLocalLlmTest({
      backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000,
      onToken: (delta, kind) => tokens.push([delta, kind]),
    });

    expect(tokens).toEqual([
      ['let me ', 'reasoning'],
      ['think…', 'reasoning'],
      ['Answer.', 'content'],
    ]);
    // The final text is content-only — reasoning streamed live but doesn't pollute the answer.
    expect(result.text).toBe('Answer.');
  });

  it('does NOT re-emit a reasoning-only stream at the end (no double output)', async () => {
    stubStream(makeReader([sse({ reasoning: 'thinking ' }), sse({ reasoning: 'aloud' })], { abort: false }));
    const tokens = [];

    const result = await runLocalLlmTest({
      backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000,
      onToken: (delta, kind) => tokens.push([delta, kind]),
    });

    // Exactly two reasoning tokens — the old end-of-stream onChunk(resolved) re-emit
    // would have appended the whole joined reasoning a third time.
    expect(tokens).toEqual([['thinking ', 'reasoning'], ['aloud', 'reasoning']]);
    // The reasoning-only run still resolves its text from reasoning for the record.
    expect(result.text).toBe('thinking aloud');
  });

  it('records TTFT for a reasoning-only run (reasoning marks first-chunk timing)', async () => {
    stubStream(makeReader([sse({ reasoning: 'hmm' })], { abort: false }));

    const result = await runLocalLlmTest({
      backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000,
      onToken: () => {},
    });

    expect(result.timings.ttftMs).not.toBeNull();
    expect(result.timings.ttftMs).toBeGreaterThanOrEqual(0);
  });

  it('awaits onToken before reading the next upstream chunk (honours backpressure)', async () => {
    const reader = makeReader([sse({ content: 'a' }), sse({ content: 'b' })], { abort: false });
    stubStream(reader);

    // A slow consumer (e.g. the streaming route awaiting a socket `drain`): hold
    // the first token's promise open and assert the read loop has NOT pulled the
    // next chunk until we let it settle. This is what makes the route's drain-await
    // actually pause upstream reading instead of buffering unbounded.
    let releaseFirst;
    const firstHeld = new Promise((resolve) => { releaseFirst = resolve; });
    let firstSeen;
    const firstArrived = new Promise((resolve) => { firstSeen = resolve; });
    const tokens = [];
    const onToken = vi.fn((delta) => {
      tokens.push(delta);
      if (tokens.length === 1) { firstSeen(); return firstHeld; }
      return undefined;
    });

    const promise = runLocalLlmTest({
      backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000, onToken,
    });

    // Wait until the first token is being handled (it's held open below). At that
    // point only one read has resolved a line — the second chunk must NOT have been
    // requested yet, since the loop is parked awaiting our held onToken. This is what
    // makes the route's drain-await actually pause upstream reading.
    await firstArrived;
    expect(tokens).toEqual(['a']);
    expect(reader.read).toHaveBeenCalledTimes(1);

    releaseFirst();
    const result = await promise;
    expect(tokens).toEqual(['a', 'b']);
    expect(result.text).toBe('ab');
  });

  it('forwards a client cancel onto the upstream fetch so the reader tears down early', async () => {
    let capturedSignal = null;
    global.fetch = vi.fn().mockImplementation((_url, init) => {
      capturedSignal = init.signal;
      return Promise.resolve({ ok: true, status: 200, body: { getReader: () => makeReader([sse({ content: 'partial' })], { abort: true }) } });
    });

    // A client that hung up before the run even started is the worst case — the
    // upstream signal must already be aborted, not run on to the 5s timeout.
    const clientController = new AbortController();
    clientController.abort();

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000, signal: clientController.signal });

    expect(capturedSignal.aborted).toBe(true);
    // The cancel surfaces as the same partial-output-on-abort contract.
    expect(result.text).toBe('partial');
    expect(result).toMatchObject({ canceled: true, error: 'Local LLM test canceled by client' });
    expect(finalizeRunRecord).toHaveBeenCalledWith(expect.objectContaining({
      error: 'Local LLM test canceled by client',
      exitCode: null,
      extras: { canceled: true, completionReason: 'client-disconnect' },
    }));
  });

  it('aborts the upstream fetch when the client signal fires mid-run', async () => {
    let capturedSignal = null;
    global.fetch = vi.fn().mockImplementation((_url, init) => {
      capturedSignal = init.signal;
      return Promise.resolve({ ok: true, status: 200, body: { getReader: () => makeReader([sse({ content: 'partial' })], { abort: true }) } });
    });

    const clientController = new AbortController();
    const promise = runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000, signal: clientController.signal });
    clientController.abort();
    await promise;

    expect(capturedSignal.aborted).toBe(true);
  });
});

describe('token counting through the stream', () => {
  beforeEach(() => {
    vi.clearAllMocks();
    __resetUsageSupport();
    getProviderById.mockResolvedValue({ id: 'lmstudio', type: 'api', endpoint: 'http://localhost:1234/v1' });
    createRun.mockResolvedValue({ runId: 'run-1', provider: { id: 'lmstudio' } });
    finalizeRunRecord.mockResolvedValue(undefined);
  });

  afterEach(() => { delete global.fetch; __resetUsageSupport(); });

  const stubStream = (reader) => {
    global.fetch = vi.fn().mockResolvedValue({ ok: true, status: 200, body: { getReader: () => reader } });
    return global.fetch;
  };

  const usageFrame = (usage) => `data: ${JSON.stringify({ choices: [], usage })}\n`;

  it('asks for usage and records the count the daemon reported', async () => {
    const fetchMock = stubStream(makeReader([
      sse({ content: 'Hi' }),
      usageFrame({ completion_tokens: 40, prompt_tokens: 900 }),
    ]));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(JSON.parse(fetchMock.mock.calls[0][1].body).stream_options).toEqual({ include_usage: true });
    expect(result.timings.completionTokens).toBe(40);
    expect(result.timings.promptTokens).toBe(900);
    expect(result.timings.tokensEstimated).toBe(false);
  });

  // No usage block: the streamed CONTENT frames are the only token-shaped
  // evidence there is, and the figure must be labelled as the estimate it is.
  it('falls back to counting content frames, marked as an estimate', async () => {
    stubStream(makeReader([sse({ content: 'a' }), sse({ content: 'b' }), sse({ content: 'c' })]));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(result.timings.completionTokens).toBe(3);
    expect(result.timings.tokensEstimated).toBe(true);
  });

  // The text this reports chars/s from is content-only, so counting reasoning
  // frames too would make the two throughput numbers describe different outputs.
  it('does not count reasoning frames toward the estimate', async () => {
    stubStream(makeReader([
      sse({ reasoning: 'thinking' }),
      sse({ reasoning: 'harder' }),
      sse({ content: 'Answer.' }),
    ]));

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });
    expect(result.timings.completionTokens).toBe(1);
  });

  it('reports no token count at all when nothing streamed', async () => {
    stubStream(makeReader([]));
    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });
    // Never 0 — that would read as a measured standstill.
    expect(result.timings.completionTokens).toBeNull();
    expect(result.timings.tokensEstimated).toBeNull();
  });

  it('retries without stream_options when the daemon rejects it, and remembers', async () => {
    const reject = { ok: false, status: 400, text: async () => 'unknown field stream_options' };
    const stream = { ok: true, status: 200, body: { getReader: () => makeReader([sse({ content: 'Hi' })]) } };
    global.fetch = vi.fn()
      .mockResolvedValueOnce(reject)
      .mockResolvedValue(stream);

    const first = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });
    expect(first.text).toBe('Hi');
    expect(JSON.parse(global.fetch.mock.calls[1][1].body).stream_options).toBeUndefined();

    // The SECOND run must not re-discover the same incompatibility: a sweep runs
    // hundreds of samples against one daemon, and each rediscovery costs a whole
    // extra prefill.
    global.fetch.mockClear();
    global.fetch.mockResolvedValue({ ok: true, status: 200, body: { getReader: () => makeReader([sse({ content: 'Hi' })]) } });
    await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });
    expect(global.fetch).toHaveBeenCalledTimes(1);
    expect(JSON.parse(global.fetch.mock.calls[0][1].body).stream_options).toBeUndefined();
  });

  it('does not cache an unrelated 400 as stream_options incompatibility', async () => {
    global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 400, text: async () => 'invalid model name' });

    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });

    expect(result.error).toContain('invalid model name');
    expect(global.fetch).toHaveBeenCalledTimes(1);
  });

  // A 401 says nothing about the body. Retrying it would double every real
  // auth failure against a key-gated vLLM.
  it('does not retry an auth failure as though it were a body rejection', async () => {
    global.fetch = vi.fn().mockResolvedValue({ ok: false, status: 401, text: async () => 'unauthorized' });
    const result = await runLocalLlmTest({ backend: 'lmstudio', modelId: 'm1', prompt: 'hi', timeoutMs: 5000 });
    expect(result.error).toContain('401');
    expect(global.fetch).toHaveBeenCalledTimes(1);
  });
});