Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

27.8 KB
Raw Blame
/**
 * Capability tests for local models — the runner.
 *
 * The assessments panel next door answers "how fast is this model here". This
 * one answers the question speed cannot: **can it actually do what its badges
 * claim?** Five tests, each bound to the capability badges the install catalog
 * already shows, each keeping the model's full output:
 *
 *   - `sandbox-repair` (`tools`) — a broken module and its failing test are
 *     copied into a throwaway sandbox and the configured OpenCode task driver is
 *     pointed at it. The model has to read the files, write a fix to disk, and
 *     make the test pass. Its structured transcript streams to the page while it
 *     works; the PTY-backed TUI path is measured separately by the harness check.
 *   - `image-analysis` (`vision`) — a fixture image with known contents goes in;
 *     the description is scored against required and bonus keywords.
 *   - `story-outline` (`chat`) — one hero's-journey outline from a fixed
 *     premise, scored on beat coverage and ordering.
 *   - `fiction-scene` (`chat`) — a short opening scene from the same premise,
 *     scored on story anchors and basic scene craft before a human reads it.
 *
 * ## Rules this file exists to honour
 *
 * 1. **No cold-bootstrap LLM calls** (root AGENTS.md). `getCapabilityTestReport`
 *    touches disk (plus, for Ollama, cached loopback capability probes) and
 *    calls no provider, so it is safe from a poll or a page load.
 *    `runCapabilityTest` is the ONLY path that reaches a model, and it fires
 *    solely from a deliberate click whose gate names the runtime, the model and
 *    the prompt first. There is deliberately **no scheduler and no sweep** here:
 *    a capability run is a manual act.
 * 2. **Verdicts come from disk, never from the transcript.** A model claiming
 *    "I fixed it" scores nothing — the sandbox verdict is the fixture hashes
 *    plus the exit code of a test run PortOS performs itself, after the agent
 *    has stopped.
 * 3. **An unclaimed capability is not a failure.** Applicability is decided in
 *    `lib/modelCapabilityTests.js`; a model with no `vision` badge is skipped on
 *    the image test rather than scored zero on it.
 * 4. **One heavy local job at a time.** A run holds a runtime for minutes and
 *    loads a model into memory, so it takes the same machine-wide claim
 *    assessments and sweeps take. Without it a capability run could execute on
 *    top of an overnight sweep and corrupt its timings.
 */

import { cp, readdir, readFile, rm, stat } from 'fs/promises';
import { join } from 'path';
import { fileURLToPath } from 'url';
import sharp from 'sharp';
import { ServerError } from '../lib/errorHandler.js';
import { PATHS, ensureDir, sha256File } from '../lib/fileUtils.js';
import { bufferedSpawn } from '../lib/bufferedSpawn.js';
import { withSpawnCwdEnv } from '../lib/spawnCwd.js';
import { claimHeavyLocalJob } from '../lib/heavyJobClaim.js';
import { formatAgentEvent } from '../lib/opencodeStream.js';
import {
  ASSESSABLE_RUNTIMES, LOCAL_RUNTIMES, MANAGED_ASSESSMENT_BACKENDS,
} from '../lib/localProviderRuntime.js';
import {
  CAPABILITY_TESTS, CAPABILITY_TEST_IDS, CAPABILITY_TEST_PROMPTS, SANDBOX_TASK_PROMPT,
  VISION_FIXTURE_KEYWORDS, FICTION_SCENE_KEYWORDS, HEROS_JOURNEY_BEATS,
  applicableTests, getCapabilityTest,
  scoreKeywords, scoreFictionScene, scoreStoryBeats, scoreSandboxRepair, rollUpVerdict,
} from '../lib/modelCapabilityTests.js';
import { parseRhetoricJson, scoreRhetoricReference, RHETORIC_REFERENCE_SET } from '../lib/postRhetoric.js';
import { listRuntimeModels, runtimeEndpoint, runtimeApiKey } from './localModelAssessments.js';
import { runLocalLlmTest, runEndpointLlmTest } from './localLlmPlayground.js';
import { listProviders } from './providers.js';
import { resolveOpencodeTuiProvider, runOpencodeTask } from './opencodeTask.js';
import { ollamaBadgeCapabilities } from './localLlm.js';
import * as ollamaManager from './ollamaManager.js';
import { indexResults, saveResult, capabilityResultKey, loadResults } from './modelCapabilityTestStore.js';

const FIXTURES = fileURLToPath(new URL('../fixtures/model-tests/', import.meta.url));
const VISION_FIXTURE_SVG = join(FIXTURES, 'vision', 'bicycle-bench.svg');
const SANDBOX_FIXTURE = join(FIXTURES, 'sandbox', 'cart-totals');

/** The three files the sandbox test copies, and what each one is FOR. */
export const SANDBOX_FILES = Object.freeze({
  // The only file the model is allowed to change.
  module: 'cart-totals.mjs',
  // Both of these must come back byte-identical: editing the test into
  // submission is the one way to "pass" without fixing anything.
  test: 'cart-totals.test.mjs',
  data: 'orders.json',
});

/** How PortOS itself checks the sandbox, after the agent has stopped. */
export const SANDBOX_VERIFY_COMMAND = Object.freeze(['node', SANDBOX_FILES.test]);

const AGENT_TIMEOUT_MS = 10 * 60 * 1000;
const VERIFY_TIMEOUT_MS = 60 * 1000;
const CHAT_TIMEOUT_MS = 5 * 60 * 1000;

/**
 * Sandboxes are KEPT after a run — reading the diff is half the value of the
 * test — which means something has to bound them, or a directory the user never
 * looks at grows one copy per run forever. The newest few are what anyone
 * actually opens.
 */
const SANDBOX_KEEP = 20;

// Low but not zero: a deterministic answer makes two runs of the same model
// comparable, while 0 makes some local daemons loop on a repeated phrase.
const TEST_TEMPERATURE = 0.2;

const runtimeLabel = (id) => LOCAL_RUNTIMES[id]?.label || id;

// ---- capabilities -----------------------------------------------------------

/**
 * The badge set a model claims, or `null` when nothing authoritative said.
 *
 * The distinction is the whole point: `[]` means a runtime answered and the
 * model claims nothing (every test is `not-applicable`), while `null` means
 * nobody was asked (every test is `unknown` and stays on offer).
 *
 * Ollama's `/api/tags` carries no capability flags, so `listModels` reports
 * `null` there rather than an empty list — `/api/show` is what actually knows,
 * and it is cached per model, so asking it here stays cheap enough for a read
 * path. Every other runtime either answers in its listing (LM Studio) or is a
 * bare endpoint that reports ids and nothing else.
 */
export async function resolveModelCapabilities(runtime, model) {
  const listed = Array.isArray(model?.capabilities) ? model.capabilities : null;
  if (listed?.length) return listed;
  if (runtime === 'ollama' && model?.id) {
    const raw = await ollamaManager.getModelCapabilities(model.id).catch(() => null);
    // A failed probe is unknown, not "claims nothing".
    return ollamaBadgeCapabilities(raw);
  }
  return listed;
}

/**
 * Whether PortOS can DRIVE the agent tests on a runtime, as opposed to whether a
 * model claims the capability. Surfaced separately so the UI can say "PortOS
 * cannot run this here" rather than implying the model is at fault.
 */
async function agentDriverFor(runtime, providers) {
  const provider = await resolveOpencodeTuiProvider(runtime, providers);
  return provider
    ? { available: true, provider, reason: null }
    : { available: false, provider: null, reason: `no OpenCode TUI provider is configured for ${runtimeLabel(runtime)}` };
}

// ---- fixtures ---------------------------------------------------------------

// Rendered once per process. The fixture is a committed SVG rather than a PNG so
// it stays reviewable in a diff, and rasterizing it here means the bytes a model
// sees are produced the same way on every install.
let visionFixtureCache = null;
// The fixture files never change while the process runs, so their hashes — the
// baseline every sandbox verdict is measured against — are read once.
let fixtureHashCache = null;

export async function loadVisionFixture() {
  if (visionFixtureCache) return visionFixtureCache;
  const png = await sharp(VISION_FIXTURE_SVG).png().toBuffer();
  visionFixtureCache = {
    dataUrl: `data:image/png;base64,${png.toString('base64')}`,
    bytes: png.length,
  };
  return visionFixtureCache;
}

async function sandboxFixtureHashes() {
  if (fixtureHashCache) return fixtureHashCache;
  fixtureHashCache = Object.fromEntries(await Promise.all(
    Object.entries(SANDBOX_FILES).map(async ([role, name]) => [role, await sha256File(join(SANDBOX_FIXTURE, name))]),
  ));
  return fixtureHashCache;
}

/** Test seam: drop the cached fixtures so a suite can re-read them. */
export function __resetFixtureCaches() {
  visionFixtureCache = null;
  fixtureHashCache = null;
}

// ---- run plumbing -----------------------------------------------------------

const progressFrame = (backend, modelId, testId, frame) => ({
  scope: 'capability-test',
  backend,
  modelId,
  testId,
  ...frame,
});

/**
 * Ask the right transport for one bounded generation.
 *
 * A managed backend goes through the playground's provider path (and so lands in
 * `/runs`); a bare daemon is talked to directly, because inventing a provider
 * record for something the user started outside PortOS would put a phantom
 * provider in the run history. Same result shape either way.
 */
async function generate({ backend, modelId, prompt, images, maxTokens, signal }) {
  if (MANAGED_ASSESSMENT_BACKENDS.includes(backend)) {
    return runLocalLlmTest({
      backend, modelId, prompt, images, maxTokens, signal,
      temperature: TEST_TEMPERATURE, timeoutMs: CHAT_TIMEOUT_MS,
    });
  }
  const endpoint = await runtimeEndpoint(backend);
  if (!endpoint) {
    throw new ServerError(`No endpoint is configured for ${runtimeLabel(backend)}`, {
      status: 503, code: 'CAPABILITY_TEST_RUNTIME_UNREACHABLE',
    });
  }
  return runEndpointLlmTest({
    runtime: backend, endpoint, apiKey: await runtimeApiKey(backend),
    modelId, prompt, images, maxTokens, signal,
    temperature: TEST_TEMPERATURE, timeoutMs: CHAT_TIMEOUT_MS,
  });
}

// ---- the generative tests ---------------------------------------------------

/**
 * The generative tests, which differ only in what they send and how the
 * answer is scored. Everything else — the empty-output contract, keeping the
 * text whichever way the run ended — is identical, and was worth having in one
 * place rather than twice.
 */
const CHAT_TESTS = {
  'image-analysis': {
    maxTokens: 700,
    message: (modelId) => `Sending the fixture image to ${modelId}…`,
    images: async () => [(await loadVisionFixture()).dataUrl],
    score: (text) => scoreKeywords(text, VISION_FIXTURE_KEYWORDS),
  },
  'story-outline': {
    // A twelve-beat outline runs close to a thousand tokens; a budget that
    // clipped it would score as missing beats — a scoring bug dressed up as a
    // model failure.
    maxTokens: 2400,
    message: (modelId) => `Asking ${modelId} for a twelve-beat outline…`,
    images: async () => undefined,
    score: (text) => scoreStoryBeats(text, HEROS_JOURNEY_BEATS),
  },
  'fiction-scene': {
    maxTokens: 1400,
    message: (modelId) => `Asking ${modelId} for a short fiction scene…`,
    images: async () => undefined,
    score: (text) => scoreFictionScene(text, FICTION_SCENE_KEYWORDS),
  },
  'rhetoric-reference': {
    // Forty compact reference attempts plus one JSON score per item fit within
    // this budget while leaving room for short evidence-free score output.
    maxTokens: 5000,
    message: (modelId) => `Scoring the rhetoric reference set with ${modelId}…`,
    images: async () => undefined,
    score: (text) => scoreRhetoricReference(parseRhetoricJson(text, (value) => {
      const expected = new Set(RHETORIC_REFERENCE_SET.map(({ id }) => id));
      return value && typeof value === 'object' && !Array.isArray(value)
        && Array.isArray(value.evaluations)
        && value.evaluations.length === expected.size
        && value.evaluations.every((item) => item && expected.has(item.id)
          && typeof item.score === 'number');
    })),
  },
};

async function runChatTest({ backend, modelId, testId, signal, emit }) {
  const spec = CHAT_TESTS[testId];
  const images = await spec.images();
  emit({ event: 'progress', message: spec.message(modelId) });

  const result = await generate({
    backend, modelId, prompt: CAPABILITY_TEST_PROMPTS[testId], images, maxTokens: spec.maxTokens, signal,
  });
  const output = String(result?.text || '');

  // Nothing came back at all: there is no answer to score, so the error IS the
  // result. A run that produced text AND reported an error (a mid-stream
  // timeout) is scored on what it produced, with the error kept beside it.
  if (result?.error && !output.trim()) {
    return { verdict: 'failed', summary: result.error, output, detail: null, error: result.error, timings: result.timings || null };
  }
  const detail = spec.score(output);
  return {
    verdict: detail.verdict,
    summary: detail.summary,
    output,
    detail,
    error: result?.error || null,
    timings: result?.timings || null,
  };
}

/**
 * Delete all but the newest sandboxes. `runId` starts with a base36 timestamp of
 * fixed width, so a plain sort is chronological.
 */
async function pruneSandboxes(root, keep = SANDBOX_KEEP) {
  const entries = await readdir(root, { withFileTypes: true }).catch(() => []);
  const stale = entries.filter((e) => e.isDirectory()).map((e) => e.name).sort().slice(0, -keep);
  for (const name of stale) {
    await rm(join(root, name), { recursive: true, force: true }).catch((err) => {
      console.error(`⚠️ Capability tests: could not prune sandbox ${name} — ${err.message}`);
    });
  }
}

/**
 * Sandbox repair: the one test that watches a model work.
 *
 * The sandbox is a fresh copy of the fixture for every run, so a model that
 * mangles the module cannot affect the next run — and the fixture in the repo is
 * never the thing being edited.
 */
async function runSandboxRepair({ backend, modelId, signal, emit, runId, providers }) {
  const driver = await agentDriverFor(backend, providers);
  if (!driver.available) {
    throw new ServerError(driver.reason, { status: 503, code: 'CAPABILITY_TEST_AGENT_UNAVAILABLE' });
  }

  const sandboxRoot = join(PATHS.data, 'model-tests', 'sandboxes');
  const sandbox = join(sandboxRoot, runId);
  const run = async () => {
    await ensureDir(sandbox);
    await cp(SANDBOX_FIXTURE, sandbox, { recursive: true });
    emit({ event: 'progress', message: `Sandbox ready — ${Object.keys(SANDBOX_FILES).length} files copied.` });

    const fixtureHashes = await sandboxFixtureHashes();

    let transcript = '';
    let toolCalls = 0;
    const startedAt = Date.now();

    emit({ event: 'progress', message: `Starting the ${driver.provider.name || 'OpenCode'} agent on ${modelId}…` });
    const agent = await runOpencodeTask({
      provider: driver.provider,
      modelId,
      cwd: sandbox,
      prompt: SANDBOX_TASK_PROMPT,
      timeoutMs: AGENT_TIMEOUT_MS,
      signal,
      // One parse feeds both the stored transcript and the live view, so the two
      // can never disagree about what the agent did.
      onEvent: (event) => {
        const rendered = formatAgentEvent(event);
        if (!rendered) return;
        if (rendered.toolCall) toolCalls += 1;
        transcript += `${rendered.line}\n`;
        // Watching the structured tool loop work is the point of streaming this
        // driver rather than hiding it behind an in-process helper.
        emit({ event: 'output', line: rendered.line });
      },
    });

    // ---- verification: disk only, never the transcript ----
    emit({ event: 'progress', message: 'Agent finished — checking the sandbox on disk…' });

    const hashOf = (name) => sha256File(join(sandbox, name)).catch(() => null);
    const [moduleHash, testHash, dataHash] = await Promise.all([
      hashOf(SANDBOX_FILES.module), hashOf(SANDBOX_FILES.test), hashOf(SANDBOX_FILES.data),
    ]);

    // PortOS runs the test itself rather than believing the agent's own run: the
    // exit code of a command we issued is the only evidence that survives a model
    // reporting a success it did not achieve.
    const verify = await bufferedSpawn(SANDBOX_VERIFY_COMMAND[0], SANDBOX_VERIFY_COMMAND.slice(1), {
      cwd: sandbox,
      // PWD pinned at the call site (#3193). `node` resolves the test path from
      // cwd rather than PWD, so this is belt-and-braces today — but this is the
      // kind of command that grows a package runner later, and an inherited PWD
      // pointing at the PortOS checkout is exactly how one ends up verifying the
      // wrong directory.
      env: withSpawnCwdEnv(process.env, sandbox),
      timeoutMs: VERIFY_TIMEOUT_MS,
    });

    const detail = scoreSandboxRepair({
      // A deleted module reads as "never wrote a fix", not as a change.
      moduleChanged: Boolean(moduleHash) && moduleHash !== fixtureHashes.module,
      fixturesIntact: testHash === fixtureHashes.test && dataHash === fixtureHashes.data,
      testsPass: verify.success,
      toolCalls,
    });

    return {
      verdict: detail.verdict,
      summary: detail.summary,
      // The test's "output" is what PortOS's own verification run printed — the
      // thing that decided the verdict — with the agent's narration beside it.
      output: verify.stdout || verify.stderr || '',
      transcript,
      detail: {
        ...detail,
        driver: driver.provider.id,
        driverLabel: driver.provider.name || driver.provider.id,
        verifyCommand: SANDBOX_VERIFY_COMMAND.join(' '),
        verifyExitCode: verify.code,
        sandboxPath: sandbox,
        elapsedMs: Math.max(0, Date.now() - startedAt),
      },
      // Recorded even when the verdict already stands on its own: "never wrote a
      // fix" and "never wrote a fix because the CLI was missing" need different
      // responses from the user.
      error: agent.error,
      timings: null,
    };
  };
  try {
    return await run();
  } finally {
    await pruneSandboxes(sandboxRoot);
  }
}

const RUNNERS = {
  'image-analysis': runChatTest,
  'story-outline': runChatTest,
  'fiction-scene': runChatTest,
  'rhetoric-reference': runChatTest,
  'sandbox-repair': runSandboxRepair,
};

// ---- public run path --------------------------------------------------------

/**
 * Run ONE capability test against ONE model. The only path here that reaches a
 * provider, and only ever from an explicit user action.
 *
 * @param {object} options
 * @param {string} options.backend assessable runtime id
 * @param {string} options.modelId
 * @param {string} options.testId one of CAPABILITY_TEST_IDS
 * @param {AbortSignal} [options.signal] the client's disconnect
 * @param {(frame: object) => void} [options.onProgress]
 * @param {number} [options.claimTimeoutMs] how long to wait for the machine-wide
 *   heavy-job claim before reporting it busy
 */
export async function runCapabilityTest({ backend, modelId, testId, signal, onProgress, claimTimeoutMs = 0 } = {}) {
  const test = getCapabilityTest(testId);
  if (!test) {
    throw new ServerError(`Unknown capability test: ${testId}`, { status: 400, code: 'VALIDATION_ERROR' });
  }
  if (!ASSESSABLE_RUNTIMES.includes(backend)) {
    throw new ServerError(`Unsupported runtime: ${backend}`, { status: 400, code: 'VALIDATION_ERROR' });
  }

  const emit = (frame) => onProgress?.(progressFrame(backend, modelId, testId, frame));
  // A short, sortable id: it names the sandbox directory (and so its prune
  // order) and appears in the log line, so it has to read cleanly and be unique
  // per run.
  const runId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 7)}`;

  // A run holds the runtime for minutes and loads a model into memory, so it
  // takes the same claim an assessment or a sweep takes. Without it a capability
  // run could execute on top of an overnight sweep and corrupt its timings.
  const claim = await claimHeavyLocalJob({ kind: 'model capability test', id: `${backend}/${modelId}`, timeoutMs: claimTimeoutMs });
  if (!claim.ok) {
    throw new ServerError(claim.message, { status: 409, code: 'HEAVY_LOCAL_JOB_BUSY', context: { holder: claim.holder } });
  }

  console.log(`🧪 Capability tests: ${test.id} on ${backend}/${modelId} (run ${runId})`);

  const startedAt = Date.now();
  let outcome;
  try {
    emit({ event: 'start', message: `${test.label}: starting on ${modelId}…`, runId });
    // The claim wait is bounded but can be long, and it does not observe the
    // abort signal. A cancel that landed while we waited must not turn into a
    // run now that the machine is free.
    if (signal?.aborted) {
      emit({ event: 'complete', cancelled: true, message: `${test.label}: cancelled — nothing recorded` });
      return { backend, modelId, testId: test.id, cancelled: true };
    }
    const providers = test.driver === 'tui' ? await listProviders() : null;
    outcome = await RUNNERS[test.id]({ backend, modelId, testId: test.id, signal, emit, runId, providers });
  } catch (err) {
    // A terminal frame either way, or the page's banner sits on the last
    // progress line forever waiting for a run that already died.
    emit({ event: 'complete', message: `${test.label}: ${err.message}`, verdict: 'failed' });
    throw err;
  } finally {
    await claim.release();
  }

  // A cancelled run recorded as `failed` would be a lie about the model that
  // then needs manually deleting. Report what was gathered; store nothing.
  if (signal?.aborted) {
    console.log(`🧪 Capability tests: ${test.id} on ${backend}/${modelId} cancelled — not recorded`);
    emit({ event: 'complete', cancelled: true, message: `${test.label}: cancelled — nothing recorded` });
    return { ...outcome, backend, modelId, testId: test.id, cancelled: true };
  }

  const record = await saveResult({
    backend,
    modelId,
    testId: test.id,
    runId,
    verdict: outcome.verdict,
    summary: outcome.summary,
    output: outcome.output,
    transcript: outcome.transcript ?? '',
    detail: outcome.detail ?? null,
    error: outcome.error ?? null,
    timings: outcome.timings ?? null,
    elapsedMs: Math.max(0, Date.now() - startedAt),
    ranAt: new Date().toISOString(),
  });

  console.log(`🧪 Capability tests: ${test.id} on ${backend}/${modelId} → ${record.verdict} (${record.summary})`);
  emit({ event: 'complete', verdict: record.verdict, message: `${test.label}: ${record.verdict} — ${record.summary}` });
  return record;
}

// ---- read path --------------------------------------------------------------

/**
 * What the matrix needs about one recorded result — everything EXCEPT the model
 * output and the agent transcript.
 *
 * Those two are the bulk of a record (up to ~100 KB each) and are needed only
 * when the drawer opens one pairing. Shipping every stored transcript on every
 * page load would make the report grow with the square of how much testing the
 * user has done, to render a chip that says "passed".
 */
const summarizeResult = (r) => (r ? {
  backend: r.backend,
  modelId: r.modelId,
  testId: r.testId,
  verdict: r.verdict,
  summary: r.summary,
  ranAt: r.ranAt ?? null,
  elapsedMs: r.elapsedMs ?? null,
  detail: r.detail ?? null,
  error: r.error ?? null,
  timings: r.timings ?? null,
} : null);

/** One stored result in full, for the drawer. `null` when nothing is recorded. */
export async function getCapabilityTestResult(backend, modelId, testId) {
  const results = await loadResults();
  return results.find((r) => r?.backend === backend && r?.modelId === modelId && r?.testId === testId) || null;
}

/**
 * Everything the page needs to show what each installed model claims and what it
 * has proved. Disk plus cached loopback capability probes; zero LLM calls.
 */
export async function getCapabilityTestReport() {
  const providers = await listProviders();
  const [{ index, readError }, listed, drivers] = await Promise.all([
    indexResults(),
    Promise.all(ASSESSABLE_RUNTIMES.map(async (runtime) => [runtime, await listRuntimeModels(runtime)]))
      .then(Object.fromEntries),
    Promise.all(ASSESSABLE_RUNTIMES.map(async (runtime) => [runtime, await agentDriverFor(runtime, providers)]))
      .then(Object.fromEntries),
  ]);

  // Capability resolution costs one cached `/api/show` per un-probed Ollama
  // model. Sequentially that is one round trip per model on the page-load path;
  // they are independent, so they go together.
  const offline = Object.fromEntries(ASSESSABLE_RUNTIMES.map((id) => [id, Boolean(listed[id].offline)]));

  const models = (await Promise.all(ASSESSABLE_RUNTIMES.flatMap((runtime) =>
    (listed[runtime].models || []).filter((m) => m?.id).map(async (model) => {
      const capabilities = await resolveModelCapabilities(runtime, model);
      const tests = applicableTests(capabilities).map(({ test, state, missing, reason }) => {
        const stored = index[capabilityResultKey(runtime, model.id, test.id)] || null;
        // The agent driver is a property of the RUNTIME, not of the model, and it
        // can make an otherwise-applicable test unrunnable. Surfaced as its own
        // state so the UI explains "PortOS cannot drive this" rather than
        // implying the model is at fault.
        // Two things can make an otherwise-applicable test unrunnable, and both
        // are properties of the RUNTIME rather than of the model — so they are
        // surfaced as `unavailable` with the reason, never as a failure the
        // model earned. A stopped daemon wins the explanation: starting it is
        // the fix, and the missing agent preset may not even be missing once it
        // is up.
        const runtimeDown = offline[runtime] && state !== 'not-applicable';
        const noDriver = test.driver === 'tui' && state === 'applicable' && !drivers[runtime].available;
        return {
          testId: test.id,
          state: runtimeDown || noDriver ? 'unavailable' : state,
          missing,
          reason: runtimeDown
            ? `${runtimeLabel(runtime)} is not running`
            : (noDriver ? drivers[runtime].reason : reason),
          result: summarizeResult(stored),
        };
      });
      return {
        backend: runtime,
        runtimeLabel: runtimeLabel(runtime),
        // These came off disk rather than from the daemon: installed, but
        // nothing can run against them until the runtime is started.
        offline: Boolean(offline[runtime]),
        // `null` = nothing authoritative reported a badge set. The UI must say
        // "unverified", never render it as an empty badge row.
        capabilities,
        modelId: model.id,
        tests,
        verdict: rollUpVerdict(tests.map((t) => t.result?.verdict).filter(Boolean)),
      };
    })
  ))).filter(Boolean);

  const countTests = (predicate) => models.reduce((acc, m) => acc + m.tests.filter(predicate).length, 0);

  return {
    // Serialized rather than referenced so the client renders one catalog and
    // cannot drift from the gate the server enforces.
    tests: CAPABILITY_TESTS.map((t) => ({ ...t, capabilities: [...t.capabilities], prefers: [...t.prefers] })),
    prompts: { ...CAPABILITY_TEST_PROMPTS, 'sandbox-repair': SANDBOX_TASK_PROMPT },
    models,
    counts: {
      models: models.length,
      applicable: countTests((t) => t.state === 'applicable' || t.state === 'unknown'),
      passed: countTests((t) => t.result?.verdict === 'passed'),
      failed: countTests((t) => t.result?.verdict === 'failed'),
    },
    // Runtimes whose model list could not be read live — distinct from "listed
    // and legitimately empty". `manageUrl` is what makes this actionable rather
    // than just a complaint: for the runtimes PortOS can start, it points at the
    // page that starts them. `recovered` says PortOS listed what is on disk
    // anyway, so the user knows the models below are real and merely unreachable.
    listErrors: ASSESSABLE_RUNTIMES
      .filter((id) => listed[id].error)
      .map((id) => ({
        id,
        label: runtimeLabel(id),
        error: listed[id].error,
        offline: Boolean(listed[id].offline),
        recovered: Array.isArray(listed[id].models) ? listed[id].models.length : 0,
        manageUrl: LOCAL_RUNTIMES[id]?.manageUrl || null,
        docsUrl: LOCAL_RUNTIMES[id]?.docsUrl || null,
      })),
    readError,
  };
}

export { CAPABILITY_TEST_IDS };