Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
import { beforeEach, expect, it, vi } from 'vitest';
import { fireEvent, render, screen, waitFor } from '@testing-library/react';
import PortosModelBenchmarkPanel from './PortosModelBenchmarkPanel';
import * as api from '../../services/apiModelPerformance';
import toast from '../ui/Toast';
vi.mock('../../services/apiModelPerformance', () => ({
getModelPerformanceBenchmarks: vi.fn(),
discoverModelPerformanceModels: vi.fn(),
runModelPerformanceBenchmark: vi.fn(),
}));
vi.mock('../ui/Toast', () => ({ default: { success: vi.fn(), warning: vi.fn() } }));
vi.mock('recharts', () => ({
ResponsiveContainer: ({ children }) => <div>{children}</div>,
ScatterChart: ({ children }) => <div>{children}</div>,
Scatter: ({ name, data }) => <div data-testid={`scatter-${name}`} data-values={JSON.stringify(data?.map(({ x, y }) => [x, y]))} />,
CartesianGrid: () => null,
Tooltip: () => null,
XAxis: ({ label }) => <div data-testid="xaxis" data-label={label?.value} />,
YAxis: ({ label }) => <div data-testid="yaxis" data-label={label?.value} />,
}));
const runSource = {
url: 'portos://model-comparison/00000000-0000-4000-8000-000000000001',
retrievedAt: '2026-09-23T00:00:00.000Z',
methodology: 'PortOS Task Bench v1',
};
const metric = value => ({ value, source: runSource });
const observation = ({ id, provider, model, score, tokens, cost = null, billing = 'subscription', tokenBasis = 'measured' }) => ({
id,
provider,
model,
effort: 'high',
configuration: 'PortOS Task Bench v1; temperature=0',
billing,
benchmark: 'PortOS Task Bench v1 (deterministic)',
quality: metric(score),
costPerTask: null,
inputPerMillion: null,
outputPerMillion: null,
reasoningPerMillion: null,
responseSeconds: metric(4),
tokensPerSecond: metric(tokens / 4),
tokensPerRun: metric(tokens),
inputTokens: metric(tokens - 20),
outputTokens: metric(20),
apiEquivalentCost: cost === null ? null : metric(cost),
tokenBasis,
completedTasks: 5,
totalTasks: 5,
quota: null,
notes: 'No prompt or model response is stored.',
});
const codexRun = observation({ id: 'portos:codex-run', provider: 'Codex', model: 'gpt-6-luna', score: 80, tokens: 120, cost: 0.0002 });
const localRun = observation({ id: 'portos:local-run', provider: 'Ollama', model: 'qwen3.6:35b', score: 60, tokens: 150, billing: 'local', tokenBasis: 'estimated' });
const inventory = [
{ id: 'codex', name: 'Codex', billing: 'subscription', canBenchmark: true, canDiscover: false, models: [{ model: 'gpt-6-luna', efforts: ['low', 'high'] }] },
{ id: 'ollama', name: 'Ollama', billing: 'local', canBenchmark: true, canDiscover: true, models: [{ model: 'qwen3.6:35b', efforts: [] }] },
];
beforeEach(() => {
vi.clearAllMocks();
api.getModelPerformanceBenchmarks.mockResolvedValue({ schemaVersion: 1, observations: [codexRun, localRun], inventory });
});
it('runs the explicitly selected provider, model and effort, then displays the saved result', async () => {
api.runModelPerformanceBenchmark.mockResolvedValue({ observation: codexRun, complete: true });
render(<PortosModelBenchmarkPanel />);
await screen.findByTestId('xaxis');
await waitFor(() => expect(screen.getByLabelText('Model')).toHaveValue('gpt-6-luna'));
await waitFor(() => expect(screen.getByLabelText('Reasoning effort')).toBeEnabled());
fireEvent.change(screen.getByLabelText('Reasoning effort'), { target: { value: 'high' } });
fireEvent.click(screen.getByRole('button', { name: 'Run five tasks' }));
await waitFor(() => expect(api.runModelPerformanceBenchmark).toHaveBeenCalledWith(
{ providerId: 'codex', model: 'gpt-6-luna', effort: 'high' },
expect.objectContaining({ silent: true, signal: expect.any(AbortSignal) }),
));
await waitFor(() => expect(toast.success).toHaveBeenCalledWith('Benchmark saved: 80% correct'));
expect(screen.getByTestId('scatter-Codex')).toHaveAttribute('data-values', '[[120,80]]');
});
it('compares all runs by tokens and limits the cost view to models with known API rates', async () => {
render(<PortosModelBenchmarkPanel />);
await screen.findByTestId('scatter-Ollama');
expect(screen.getByTestId('scatter-Codex')).toHaveAttribute('data-values', '[[120,80]]');
expect(screen.getByTestId('scatter-Ollama')).toHaveAttribute('data-values', '[[150,60]]');
fireEvent.click(screen.getByRole('button', { name: 'API equivalent' }));
await screen.findByTestId('scatter-Codex');
expect(screen.queryByTestId('scatter-Ollama')).toBeNull();
expect(screen.getByTestId('scatter-Codex')).toHaveAttribute('data-values', '[[0.2,80]]');
expect(screen.getByTestId('xaxis')).toHaveAttribute('data-label', 'Estimated API equivalent per 1,000 runs');
expect(screen.getByText('~$0.20')).toBeTruthy();
});