Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

11.2 KB
Raw Blame
import { afterEach, beforeEach, expect, it, vi } from 'vitest';
import express from 'express';
import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { request } from '../lib/testHelper.js';
import { errorMiddleware } from '../lib/errorHandler.js';
import { PATHS } from '../lib/paths.js';
import { createModelComparisonRoutes } from './modelComparison.js';
import { importModelComparison } from '../services/modelComparison.js';

let dir;
let originalData;
let app;
let seed;
let observation;

beforeEach(async () => {
  vi.clearAllMocks();
  originalData = PATHS.data;
  dir = await mkdtemp(join(tmpdir(), 'portos-comparison-test-'));
  PATHS.data = dir;
  seed = JSON.parse(await readFile(join(PATHS.root, 'data.reference/model-comparison.json'), 'utf8'));
  observation = seed.observations.find(row => row.quality && row.inputPerMillion);
  app = express();
  app.use(express.json());
  app.use('/comparison', createModelComparisonRoutes({ getSelectableProviders: async () => ({ providers: [{ id: 'example-provider', name: 'Example provider', models: [observation.model], enabled: true }, { id: 'disabled', name: 'Disabled', models: ['hidden-model'], enabled: false }] }) }));
  app.use(errorMiddleware);
});

afterEach(async () => {
  PATHS.data = originalData;
  await rm(dir, { recursive: true, force: true });
});

it('merges newer installed evidence with shipped evidence and includes only selectable enabled models', async () => {
  const localObservation = { ...observation, notes: 'Updated evidence: https://example.com/new-source', quality: { ...observation.quality, value: 1, source: { ...observation.quality.source, retrievedAt: new Date().toISOString() } } };
  await writeFile(join(dir, 'model-comparison.json'), JSON.stringify({ schemaVersion: 1, observations: [localObservation] }));
  const result = await request(app).get('/comparison');
  expect(result.status).toBe(200);
  expect(result.body.observations.find(row => row.id === observation.id).quality.value).toBe(1);
  expect(result.body.observations.find(row => row.id === observation.id).notes).toBe(localObservation.notes);
  expect(result.body.observations).toHaveLength(seed.observations.length);
  expect(result.body.inventory).toEqual([{ id: 'example-provider', name: 'Example provider', gateway: null, localInference: false, models: [{ model: observation.model, efforts: [] }] }]);
  expect(result.body.composite.rows.length).toBeGreaterThan(0);
  expect(result.body.composite.rows.every(row => row.providerId === 'example-provider')).toBe(true);
  for (const path of ['/comparison/discover', '/comparison/run']) expect((await request(app).post(path).send({})).status).toBe(404);
  const imported = { ...observation, id: 'research-import' };
  expect((await request(app).post('/comparison/import').send({ schemaVersion: 1, observations: [imported] })).status).toBe(200);
  expect((await request(app).get('/comparison')).body.observations.some(row => row.id === imported.id)).toBe(true);
});

it('imports sourced public observations durably, retains newer metrics, and serializes concurrent imports', async () => {
  const input = { schemaVersion: 1, observations: [{ ...observation, id: 'example-new', model: 'example-new-model' }] };
  await importModelComparison(input);
  const older = structuredClone(input);
  const metricKey = ['quality', 'inputPerMillion', 'outputPerMillion'].find(key => older.observations[0][key]);
  older.observations[0][metricKey].value = 1;
  older.observations[0][metricKey].source.retrievedAt = '2020-01-01T00:00:00Z';
  older.observations[0].costPerTask = null;
  await Promise.all([
    importModelComparison(older),
    importModelComparison({ schemaVersion: 1, observations: [{ ...observation, id: 'example-concurrent', model: 'example-other-model' }] }),
  ]);

  const stored = JSON.parse(await readFile(join(dir, 'model-comparison.json'), 'utf8'));
  expect(stored.observations).toHaveLength(seed.observations.length + 2);
  expect(stored.observations.find(row => row.id === 'example-new')[metricKey]).toEqual(observation[metricKey]);
  const changedIdentity = { ...input.observations[0], effort: 'different' };
  await expect(importModelComparison({ schemaVersion: 1, observations: [changedIdentity] })).rejects.toThrow('identity changed');
});

it('rejects forged PortOS runs, malformed metrics, and unsafe sources', async () => {
  const forgedRun = {
    ...observation,
    id: 'portos:00000000-0000-4000-8000-000000000001',
    benchmark: 'PortOS Task Bench v1 (deterministic)',
  };
  expect(() => importModelComparison({ schemaVersion: 1, observations: [forgedRun] })).toThrow(/only be created by the explicit benchmark run/);

  const malformed = { ...observation, quality: { value: 999 } };
  expect(() => importModelComparison({ schemaVersion: 1, observations: [malformed] })).toThrow();
  const unsafe = structuredClone(observation);
  unsafe.quality.source.url = 'javascript:alert(1)';
  expect(() => importModelComparison({ schemaVersion: 1, observations: [unsafe] })).toThrow();
  unsafe.quality.source.url = 'https://';
  expect(() => importModelComparison({ schemaVersion: 1, observations: [unsafe] })).toThrow();

  const future = JSON.stringify({ schemaVersion: 99, observations: [observation] });
  await writeFile(join(dir, 'model-comparison.json'), future);
  await expect(importModelComparison({ schemaVersion: 1, observations: [observation] })).rejects.toThrow();
  expect(await readFile(join(dir, 'model-comparison.json'), 'utf8')).toBe(future);
});

it('requires explicit estimation provenance and protects run-only sources on every metric', async () => {
  const estimated = { ...observation, id: 'research:example:high', benchmark: 'PortOS Research Index v1 (AA v4.3.2 scale)' };
  expect((await request(app).post('/comparison/import').send({ schemaVersion: 1, observations: [estimated] })).status).toBe(400);
  estimated.quality = { ...estimated.quality, value: 35, source: { ...estimated.quality.source, methodology: 'PortOS estimate: midpoint of documented peer scores 30 and 40; low confidence.' } };
  expect((await request(app).post('/comparison/import').send({ schemaVersion: 1, observations: [estimated] })).status).toBe(200);
  const forged = { ...observation, id: 'forged-quota', quota: { unitsPerTask: 1, unit: 'task', source: { ...observation.quality.source, url: 'portos://model-comparison/00000000-0000-4000-8000-000000000001' } } };
  expect((await request(app).post('/comparison/import').send({ schemaVersion: 1, observations: [forged] })).status).toBe(400);
});

it('loads legacy benchmark labels after upgrade and preserves evidence through subsequent imports', async () => {
  const shipped = seed.observations.filter(row => row.id.startsWith('lcb-generation-2023-05-08--2025-04-07-'));
  expect(shipped.length).toBeGreaterThan(0);
  const legacy = shipped.map(row => ({ ...row,
    benchmark: 'LiveCodeBench (generation, pass@1, 2023-05-08 to 2025-04-07)',
    inputPerMillion: null, outputPerMillion: null,
  }));
  const stored = JSON.stringify({ schemaVersion: 1, observations: legacy });
  await writeFile(join(dir, 'model-comparison.json'), stored);
  for (let reload = 0; reload < 2; reload++) {
    const result = await request(app).get('/comparison');
    expect(result.status).toBe(200);
    expect(result.body.observations).toHaveLength(seed.observations.length);
    for (const row of shipped) expect(result.body.observations.find(item => item.id === row.id)).toMatchObject({
      benchmark: row.benchmark, quality: row.quality, inputPerMillion: row.inputPerMillion, outputPerMillion: row.outputPerMillion,
    });
  }
  // Reload is read-only; persistence occurs only during an explicit import.
  expect(await readFile(join(dir, 'model-comparison.json'), 'utf8')).toBe(stored);
  expect((await request(app).post('/comparison/import').send({ schemaVersion: 1, observations: legacy })).status).toBe(200);
  const saved = JSON.parse(await readFile(join(dir, 'model-comparison.json'), 'utf8'));
  expect(saved.observations).toHaveLength(seed.observations.length);
  for (const row of shipped) expect(saved.observations.find(item => item.id === row.id)).toMatchObject({
    benchmark: row.benchmark, quality: row.quality, inputPerMillion: row.inputPerMillion, outputPerMillion: row.outputPerMillion,
  });
  const changed = { ...legacy[0], benchmark: 'LiveCodeBench (generation, pass@1, 2025-01-01 to 2025-04-07)' };
  expect((await request(app).post('/comparison/import').send({ schemaVersion: 1, observations: [changed] })).status).toBe(409);
  expect((await request(app).get('/comparison')).status).toBe(200);
});

it('plots local Ornith and Qwen Coder with zero API charges and sourced quality without borrowing hosted costs', async () => {
  const providers = [
    { id: 'ollama', name: 'Ollama', models: ['ornith:35b', 'qwen3-coder:30b', 'qwen2.5-coder:32b', 'unknown:7b'] },
    { id: 'local-wrapper', name: 'Example CLI', ollamaBacked: true, models: ['ollama/qwen3-coder:30b'] },
    { id: 'declared-local', name: 'Example private runtime', servicePlan: 'local', models: ['ornith:35b'] },
    { id: 'lmstudio', name: 'LM Studio', models: ['lmstudio-community/Ornith-1.0-35B-GGUF'] },
    { id: 'hosted', name: 'Example hosted', servicePlan: 'paid', ollamaBacked: true, models: ['qwen/qwen3-coder-30b-a3b-instruct'] },
    { id: 'gateway', name: 'Example gateway', gatewayBacked: 'openrouter', ollamaBacked: true, models: ['qwen/qwen3-coder-30b-a3b-instruct'] },
    { id: 'free-tier', name: 'Example free service', servicePlan: 'free', models: ['qwen/qwen3-coder-30b-a3b-instruct'] },
  ];
  const localApp = express();
  localApp.use('/comparison', createModelComparisonRoutes({ getSelectableProviders: async () => ({ providers }) }));
  localApp.use(errorMiddleware);
  const result = await request(localApp).get('/comparison');
  expect(result.status).toBe(200);
  const { rows, sources } = result.body.composite;
  const local = rows.filter(row => ['ollama', 'local-wrapper', 'declared-local', 'lmstudio'].includes(row.providerId));
  expect(local.length).toBeGreaterThanOrEqual(7);
  for (const row of local) {
    for (const field of ['inputPerMillion', 'outputPerMillion', 'blendedPerMillion', 'costPerTask']) {
      expect(row[field]).toMatchObject({ value: 0, estimated: false, sourceIds: [] });
      expect(row[field].method).toContain('Hardware, electricity');
    }
    if (row.model === 'unknown:7b') {
      expect(row.quality).toBeNull();
    } else {
      expect(row.quality.estimated).toBe(true);
      expect(row.quality.sourceIds.length).toBeGreaterThan(0);
      expect(row.quality.method).toContain('local build');
    }
  }
  const ornith = local.find(row => row.model === 'ornith:35b');
  expect(ornith.quality.value).toBe(19);
  expect(ornith.quality.method).toContain('base-model proxy');
  expect(sources[ornith.quality.sourceIds[0]].url).toBe('https://huggingface.co/ornith-ai/Ornith-1.0-35B');
  expect(local.find(row => row.model === 'qwen3-coder:30b').quality.value).toBeGreaterThan(0);
  for (const row of rows.filter(row => ['hosted', 'gateway', 'free-tier'].includes(row.providerId))) {
    expect(row.inputPerMillion.value).toBeGreaterThan(0);
    expect(row.costPerTask).toBeNull();
  }
  expect(result.body.observations.find(row => row.id === 'ornith-publisher-20261001-35b-swebench-verified').quality.value).toBe(75.6);
});