Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

19.9 KB
Raw Blame
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { act, render, screen, waitFor, within } from '@testing-library/react';
import userEvent from '@testing-library/user-event';
import { MemoryRouter } from 'react-router';

vi.mock('../../services/api', () => ({
  runModelCapabilityTest: vi.fn(),
  deleteModelCapabilityTest: vi.fn(),
  // The report carries summaries only; the drawer fetches the model output and
  // the agent transcript for the one pairing it opens.
  getModelCapabilityTestResult: vi.fn(async () => ({ output: '', transcript: '' })),
}));
vi.mock('../ui/Toast', () => ({ default: { success: vi.fn(), error: vi.fn(), warning: vi.fn() } }));
// The live agent transcript arrives on the shared `localLlm:progress` socket
// event; these tests replay frames through the registered handler.
vi.mock('../../services/socket', () => ({ default: { on: vi.fn(), off: vi.fn() } }));

import { runModelCapabilityTest, deleteModelCapabilityTest, getModelCapabilityTestResult } from '../../services/api';
import socket from '../../services/socket';
import ModelCapabilityTests from './ModelCapabilityTests.jsx';

const TESTS = [
  { id: 'sandbox-repair', label: 'Sandbox repair', capabilities: ['tools'], prefers: ['code'], blurb: 'Fix a broken module in a sandbox.', driver: 'tui', kind: 'agent' },
  { id: 'image-analysis', label: 'Image analysis', capabilities: ['vision'], prefers: [], blurb: 'Describe a fixture image.', driver: 'chat', kind: 'vision' },
  { id: 'story-outline', label: 'Story outline', capabilities: ['chat'], prefers: ['reasoning'], blurb: 'Outline a hero journey.', driver: 'chat', kind: 'text' },
];
const FICTION_TEST = { id: 'fiction-scene', label: 'Fiction scene', capabilities: ['chat'], prefers: [], blurb: 'Write a constrained opening scene.', driver: 'chat', kind: 'text' };

const slot = (testId, over = {}) => ({ testId, state: 'applicable', missing: [], reason: null, result: null, ...over });

const model = (over = {}) => ({
  backend: 'ollama',
  runtimeLabel: 'Ollama',
  modelId: 'example-model:7b',
  capabilities: ['chat', 'tools', 'vision'],
  tests: TESTS.map((t) => slot(t.id)),
  verdict: null,
  ...over,
});

const report = (over = {}) => ({
  tests: TESTS,
  testIds: TESTS.map((t) => t.id),
  prompts: {
    'image-analysis': 'Describe everything you can see in this image.',
    'story-outline': 'Outline this as a hero journey in twelve beats.',
    'sandbox-repair': 'Fix cart-totals.mjs so the test passes.',
  },
  models: [model()],
  runtimes: [{ id: 'ollama', label: 'Ollama', endpointRuntime: false, error: null, agentDriver: { available: true, reason: null } }],
  counts: { models: 1, applicable: 3, passed: 0, failed: 0 },
  listErrors: [],
  readError: null,
  ...over,
});

const renderPanel = (props = {}) => render(
  <MemoryRouter initialEntries={['/models/performance']}>
    <ModelCapabilityTests report={report()} loading={false} onReload={vi.fn()} {...props} />
  </MemoryRouter>,
);

const fictionReport = (result) => report({
  tests: [...TESTS, FICTION_TEST],
  testIds: [...TESTS.map((t) => t.id), FICTION_TEST.id],
  prompts: { ...report().prompts, 'fiction-scene': 'Write a constrained opening scene.' },
  models: [model({ tests: [...TESTS.map((t) => slot(t.id)), slot('fiction-scene', { result })] })],
  counts: { models: 1, applicable: 4, passed: 0, failed: 0 },
});

/**
 * Replay a server progress frame through the handler the panel registered.
 * Wrapped in `act` because a socket frame drives a state update from outside
 * React's event system — the suite treats an unwrapped one as a hard failure.
 */
const emitFrame = async (frame) => {
  const handler = socket.on.mock.calls.find(([event]) => event === 'localLlm:progress')?.[1];
  await act(async () => { handler?.(frame); });
};

beforeEach(() => vi.clearAllMocks());

describe('the matrix', () => {
  it('lists every installed model with the badges it claims', () => {
    renderPanel();
    expect(screen.getByText('example-model:7b')).toBeInTheDocument();
    expect(screen.getByLabelText('Tool use')).toBeInTheDocument();
    expect(screen.getByLabelText('Vision')).toBeInTheDocument();
  });

  it('shows an unclaimed capability as a reason, never as a failure', () => {
    renderPanel({
      report: report({
        models: [model({
          capabilities: ['chat'],
          tests: [
            slot('sandbox-repair', { state: 'not-applicable', missing: ['tools'], reason: 'no tools badge' }),
            slot('image-analysis', { state: 'not-applicable', missing: ['vision'], reason: 'no vision badge' }),
            slot('story-outline'),
          ],
        })],
      }),
    });
    expect(screen.getByText('no tools badge')).toBeInTheDocument();
    expect(screen.getByText('no vision badge')).toBeInTheDocument();
    // A skipped test is not a red verdict and offers no run.
    expect(screen.queryByText('failed')).not.toBeInTheDocument();
    expect(screen.getAllByRole('button', { name: /not run/i })).toHaveLength(1);
  });

  it('says a test cannot run here when PortOS has no driver for the runtime', () => {
    renderPanel({
      report: report({
        models: [model({ tests: [slot('sandbox-repair', { state: 'unavailable', reason: 'no OpenCode TUI provider is configured for LM Studio' }), slot('image-analysis'), slot('story-outline')] })],
      }),
    });
    expect(screen.getByText(/can.t run here/)).toBeInTheDocument();
  });

  it('renders "capabilities not reported" rather than an empty badge row', () => {
    renderPanel({ report: report({ models: [model({ capabilities: null })] }) });
    expect(screen.getByText('capabilities not reported')).toBeInTheDocument();
  });

  it('summarises a recorded result in the cell', () => {
    renderPanel({
      report: report({
        models: [model({
          tests: [
            slot('sandbox-repair', { result: { verdict: 'passed', summary: 'fixed in 3 tool calls' } }),
            slot('image-analysis', { result: { verdict: 'failed', summary: '1 of 4 required' } }),
            slot('story-outline'),
          ],
        })],
      }),
    });
    expect(screen.getByText('fixed in 3 tool calls')).toBeInTheDocument();
    expect(screen.getByText('1 of 4 required')).toBeInTheDocument();
  });

  it('says so plainly when nothing is installed', () => {
    renderPanel({ report: report({ models: [], counts: { models: 0, applicable: 0, passed: 0, failed: 0 } }) });
    expect(screen.getByText(/no local models are installed/i)).toBeInTheDocument();
  });

  it('uses an explicit task specialization to break a tied evidence score', () => {
    const passingSandbox = { verdict: 'passed', summary: 'fixed in 5 tool calls' };
    renderPanel({
      report: report({
        models: [
          model({ modelId: 'generic-model:30b', tests: [slot('sandbox-repair', { result: passingSandbox }), slot('image-analysis'), slot('story-outline')] }),
          model({ modelId: 'qwen3-coder:30b', tests: [slot('sandbox-repair', { result: passingSandbox }), slot('image-analysis'), slot('story-outline')] }),
        ],
      }),
    });

    const card = screen.getByText('Best coding evidence').parentElement;
    expect(within(card).getByText('qwen3-coder:30b')).toBeInTheDocument();
  });

  it('uses themed surfaces for task and recommendation cards', () => {
    renderPanel();

    expect(screen.getByText('Fix a broken module in a sandbox.').parentElement)
      .toHaveClass('bg-port-card');
    expect(screen.getByText('Best coding evidence').parentElement)
      .toHaveClass('bg-port-card');
  });
});

describe('the run gate', () => {
  it('names the exact prompt before anything is sent', async () => {
    const user = userEvent.setup();
    renderPanel();
    await user.click(screen.getAllByRole('button', { name: /not run/i })[1]);

    expect(await screen.findByText('Describe everything you can see in this image.')).toBeInTheDocument();
    // Nothing has been called yet — the gate is the consent step.
    expect(runModelCapabilityTest).not.toHaveBeenCalled();
  });

  it('warns that the claim is unverified on a runtime that reports no capabilities', async () => {
    const user = userEvent.setup();
    renderPanel({
      report: report({
        models: [model({ backend: 'llama', runtimeLabel: 'llama.cpp', capabilities: null, tests: TESTS.map((t) => slot(t.id, { state: 'unknown', reason: 'this runtime reports no capability list' })) })],
      }),
    });
    await user.click(screen.getAllByRole('button', { name: /not run/i })[0]);
    expect(await screen.findByRole('alert')).toHaveTextContent(/lists model ids only/i);
  });

  it('runs only the model and test the gate named', async () => {
    const user = userEvent.setup();
    const onReload = vi.fn();
    runModelCapabilityTest.mockResolvedValue({ verdict: 'passed', summary: '4 of 4 required' });
    renderPanel({ onReload });

    await user.click(screen.getAllByRole('button', { name: /not run/i })[1]);
    await user.click(await screen.findByRole('button', { name: /run image analysis/i }));

    await waitFor(() => expect(runModelCapabilityTest).toHaveBeenCalledTimes(1));
    expect(runModelCapabilityTest.mock.calls[0][0]).toEqual({
      backend: 'ollama', modelId: 'example-model:7b', testId: 'image-analysis',
    });
    await waitFor(() => expect(onReload).toHaveBeenCalled());
  });
});

describe('the live run', () => {
  it('streams agent transcript lines as they arrive', async () => {
    const user = userEvent.setup();
    // Never resolves: the run stays in flight so the live view is what renders.
    runModelCapabilityTest.mockImplementation(() => new Promise(() => {}));
    renderPanel();

    await user.click(screen.getAllByRole('button', { name: /not run/i })[0]);
    await user.click(await screen.findByRole('button', { name: /run sandbox repair/i }));

    const target = { scope: 'capability-test', backend: 'ollama', modelId: 'example-model:7b', testId: 'sandbox-repair' };
    await emitFrame({ ...target, event: 'progress', message: 'Sandbox ready — 3 files copied.' });
    await emitFrame({ ...target, event: 'output', line: '● read cart-totals.mjs' });

    expect(await screen.findByText('Sandbox ready — 3 files copied.')).toBeInTheDocument();
    expect(await screen.findByText('● read cart-totals.mjs')).toBeInTheDocument();
  });

  it('ignores frames for a pairing it is not running', async () => {
    const user = userEvent.setup();
    runModelCapabilityTest.mockImplementation(() => new Promise(() => {}));
    renderPanel();

    await user.click(screen.getAllByRole('button', { name: /not run/i })[0]);
    await user.click(await screen.findByRole('button', { name: /run sandbox repair/i }));

    await emitFrame({ scope: 'capability-test', backend: 'ollama', modelId: 'some-other-model', testId: 'sandbox-repair', event: 'output', line: 'from another run' });
    // A model pull streaming on the same socket event must not render here either.
    await emitFrame({ scope: 'model-pull', backend: 'ollama', modelId: 'example-model:7b', event: 'output', line: 'pulling a layer' });

    expect(screen.queryByText('from another run')).not.toBeInTheDocument();
    expect(screen.queryByText('pulling a layer')).not.toBeInTheDocument();
  });
});

describe('a recorded result', () => {
  const withResult = (result) => report({
    models: [model({ tests: [slot('sandbox-repair'), slot('image-analysis', { result }), slot('story-outline')] })],
  });

  const openResult = async (user) => {
    await user.click(screen.getByRole('button', { name: /2 of 4 required/i }));
  };

  const RESULT = {
    verdict: 'partial',
    summary: '2 of 4 required, 1 of 3 bonus',
    ranAt: '2026-08-22T09:00:00.000Z',
    elapsedMs: 4200,
    error: null,
    timings: null,
    detail: {
      verdict: 'partial',
      requiredHit: 2,
      requiredTotal: 4,
      bonusHit: 1,
      bonusTotal: 3,
      required: [
        { id: 'bicycle', label: 'Bicycle', any: ['bicycle', 'bike'], hit: true },
        { id: 'bench', label: 'Bench', any: ['bench'], hit: true },
        { id: 'lamp', label: 'Street lamp', any: ['lamp'], hit: false },
        { id: 'dog', label: 'Dog', any: ['dog'], hit: false },
      ],
      bonus: [
        { id: 'red-bicycle', label: 'Bicycle is red', any: ['red'], hit: true },
        { id: 'blue-bench', label: 'Bench is blue', any: ['blue'], hit: false },
        { id: 'sign-number', label: 'Reads the number 3', any: ['3'], hit: false },
      ],
    },
  };

  it('leads with what the model actually said, fetched for this pairing alone', async () => {
    const user = userEvent.setup();
    getModelCapabilityTestResult.mockResolvedValue({ output: 'A bicycle and a bench at night.', transcript: '' });
    renderPanel({ report: withResult(RESULT) });
    await openResult(user);

    expect(await screen.findByText('A bicycle and a bench at night.')).toBeInTheDocument();
    expect(getModelCapabilityTestResult).toHaveBeenCalledWith(
      'ollama', 'example-model:7b', 'image-analysis', { silent: true },
    );
  });

  it('says the model returned nothing only once the record has actually arrived', async () => {
    const user = userEvent.setup();
    getModelCapabilityTestResult.mockResolvedValue({ output: '', transcript: '' });
    renderPanel({ report: withResult(RESULT) });
    await openResult(user);
    // An empty string is a silent model; "not fetched yet" must not render as one.
    expect(await screen.findByText(/returned no text/i)).toBeInTheDocument();
  });

  it('separates required from bonus, and says bonus cannot fail a run', async () => {
    const user = userEvent.setup();
    renderPanel({ report: withResult(RESULT) });
    await openResult(user);
    await user.click(await screen.findByRole('tab', { name: /checks/i }));

    expect(screen.getByText(/2 of 4 — all needed to pass/i)).toBeInTheDocument();
    expect(screen.getByText(/1 of 3 — detail work, never fails a run/i)).toBeInTheDocument();
    // Two required hits plus one bonus hit — a bonus miss is styled as a muted
    // skip, never as the red X a required miss gets.
    expect(screen.getAllByLabelText('found')).toHaveLength(3);
    expect(screen.getAllByLabelText('not found')).toHaveLength(4);
  });

  it('discards a result and refreshes rather than leaving a stale row', async () => {
    const user = userEvent.setup();
    const onReload = vi.fn();
    deleteModelCapabilityTest.mockResolvedValue({ success: true });
    renderPanel({ report: withResult(RESULT), onReload });

    await openResult(user);
    await user.click(await screen.findByRole('button', { name: /discard this result/i }));

    await waitFor(() => expect(deleteModelCapabilityTest).toHaveBeenCalledWith(
      'ollama', 'example-model:7b', 'image-analysis', { silent: true },
    ));
    await waitFor(() => expect(onReload).toHaveBeenCalled());
  });

  it('shows a mid-run error beside the score instead of replacing it', async () => {
    const user = userEvent.setup();
    getModelCapabilityTestResult.mockResolvedValue({ output: 'A bicycle and a bench at night.', transcript: '' });
    renderPanel({ report: withResult({ ...RESULT, error: 'Timed out after 300000ms' }) });
    await openResult(user);
    expect(await screen.findByRole('alert')).toHaveTextContent('Timed out after 300000ms');
    expect(await screen.findByText('A bicycle and a bench at night.')).toBeInTheDocument();
  });

  it('shows the fiction scene structure checks and warns that they are not literary quality', async () => {
    const user = userEvent.setup();
    const result = {
      verdict: 'partial',
      summary: '4 of 5 story anchors, 2 of 3 scene signals, 3 of 3 minimum craft checks, 446 words',
      detail: {
        requiredHit: 4,
        requiredTotal: 5,
        wordCount: 446,
        paragraphCount: 8,
        hasDialogue: true,
        required: [{ id: 'marsh', label: 'Tidal-marsh setting', any: ['marsh'], hit: true }],
        bonus: [{ id: 'dialogue', label: 'Spoken line', any: ['said'], hit: true }],
        craft: { minimumWords: true, paragraphs: true, dialogue: true },
      },
    };
    renderPanel({ report: fictionReport(result) });
    await user.click(screen.getByRole('button', { name: /4 of 5 story anchors/i }));
    await user.click(await screen.findByRole('tab', { name: /checks/i }));

    expect(screen.getByText('446')).toBeInTheDocument();
    expect(screen.getByText(/not a literary-quality score/i)).toBeInTheDocument();
  });
});

describe('running everything that applies to one model', () => {
  it('offers only the runnable tests, and says how many before the click', () => {
    renderPanel({
      report: report({
        models: [model({
          tests: [
            slot('sandbox-repair', { state: 'not-applicable', reason: 'no tools badge' }),
            slot('image-analysis'),
            slot('story-outline'),
          ],
        })],
      }),
    });
    expect(screen.getByRole('button', { name: /run 2/i })).toBeInTheDocument();
  });

  it('runs them one after another rather than all at once', async () => {
    const user = userEvent.setup();
    const onReload = vi.fn();
    runModelCapabilityTest.mockImplementation(async ({ testId }) => ({ testId, verdict: 'passed', summary: 'ok' }));
    renderPanel({ onReload });

    await user.click(screen.getByRole('button', { name: /run 3/i }));

    await waitFor(() => expect(runModelCapabilityTest).toHaveBeenCalledTimes(3));
    expect(runModelCapabilityTest.mock.calls.map(([payload]) => payload.testId))
      .toEqual(['sandbox-repair', 'image-analysis', 'story-outline']);
    // One refresh at the end, not one per test.
    await waitFor(() => expect(onReload).toHaveBeenCalledTimes(1));
  });

  it('stops the whole queue when a test fails rather than pressing on', async () => {
    const user = userEvent.setup();
    runModelCapabilityTest.mockRejectedValueOnce(new Error('llama.cpp is not reachable'));
    renderPanel();

    await user.click(screen.getByRole('button', { name: /run 3/i }));

    await waitFor(() => expect(runModelCapabilityTest).toHaveBeenCalledTimes(1));
    // Minutes of queued work must not run on after the user has been told the
    // runtime is down.
    expect(runModelCapabilityTest).not.toHaveBeenCalledTimes(2);
  });

  it('hides the control for a model no test applies to', () => {
    renderPanel({
      report: report({
        models: [model({
          capabilities: [],
          tests: TESTS.map((t) => slot(t.id, { state: 'not-applicable', reason: 'no badge' })),
        })],
      }),
    });
    expect(screen.queryByRole('button', { name: /^run \d/i })).not.toBeInTheDocument();
  });
});

describe('a runtime that could not be listed', () => {
  const withListError = (over) => report({
    listErrors: [{
      id: 'llama', label: 'llama.cpp', error: 'not reachable at http://127.0.0.1:5568/v1 (ECONNREFUSED)',
      offline: true, recovered: 2, manageUrl: '/models/llms', docsUrl: null, ...over,
    }],
  });

  it('says what is wrong and links to where it is fixed', () => {
    renderPanel({ report: withListError() });
    expect(screen.getByRole('alert')).toHaveTextContent(/llama\.cpp is not running/);
    expect(screen.getByRole('link', { name: /start llama\.cpp/i })).toHaveAttribute('href', '/models/llms');
  });

  it('says the models below came off disk, so they are real but unrunnable', () => {
    renderPanel({ report: withListError() });
    expect(screen.getByRole('alert')).toHaveTextContent(/showing the 2 models PortOS has on disk/);
  });

  it('falls back to the docs for a runtime PortOS cannot start', () => {
    renderPanel({
      report: withListError({
        id: 'vllm', label: 'vLLM', recovered: 0, manageUrl: null, docsUrl: 'https://docs.vllm.ai/',
      }),
    });
    expect(screen.getByRole('alert')).toHaveTextContent(/missing from this table/);
    expect(screen.getByRole('link', { name: /how to start it/i })).toHaveAttribute('href', 'https://docs.vllm.ai/');
  });

  it('marks an offline model as installed-but-not-running', () => {
    renderPanel({
      report: report({
        models: [model({ backend: 'llama', runtimeLabel: 'llama.cpp', offline: true })],
        listErrors: [],
      }),
    });
    expect(screen.getByText('not running')).toBeInTheDocument();
  });
});