Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

60.2 KB
Raw Blame
/**
 * Assessment run + store behavior, driven against a real on-disk store with
 * `PATHS.data` re-rooted at a temp dir. The provider seam (`runLocalLlmTest`)
 * is the only thing stubbed — that is the boundary where a real LLM call would
 * otherwise happen, and stubbing it is what lets these tests assert the "never
 * call a provider from a read path" contract.
 */

import { describe, it, expect, beforeEach, afterAll, vi } from 'vitest';
import { rmSync, existsSync, mkdirSync, writeFileSync, readFileSync, readdirSync } from 'fs';
import { join } from 'path';
import { createTempDataRoot, makePathsProxy } from '../lib/mockPathsDataRoot.js';

const tempRoot = createTempDataRoot('portos-local-assessment-');

vi.mock('../lib/fileUtils.js', async (importOriginal) =>
  makePathsProxy(await importOriginal(), { dataRoot: tempRoot }));

const runLocalLlmTest = vi.fn();
const runEndpointLlmTest = vi.fn();
vi.mock('./localLlmPlayground.js', () => ({
  runLocalLlmTest: (...args) => runLocalLlmTest(...args),
  runEndpointLlmTest: (...args) => runEndpointLlmTest(...args),
}));

// The endpoint runtimes (llama.cpp / MTPLX / vLLM) are reached over loopback
// HTTP. Left unmocked these tests probe whatever the DEVELOPER's machine happens
// to be serving — a live llama-server on this box put its real model id into the
// unassessed list. Default them to "up, serving nothing", which is the only
// state that keeps the managed-backend assertions about what they are about.
const probeOpenAiModels = vi.fn();
vi.mock('../lib/openAiModelsProbe.js', () => ({ probeOpenAiModels: (...args) => probeOpenAiModels(...args) }));

// `runtimeApiKey` reads the provider registry to authenticate a key-gated
// runtime (a vLLM container started behind VLLM_API_KEY).
//
// `getAllProviders` resolves the toolkit ENVELOPE (`{ activeProvider, providers }`)
// and `listProviders` is the wrapper that unwraps it — mocked in that shape on
// purpose. A bare-array mock here is what let `runtimeApiKey`'s
// `Array.isArray(providers)` guard read as satisfied in tests while never being
// true in production, so a key-gated vLLM silently got no key.
const getAllProviders = vi.fn(async () => ({ activeProvider: null, providers: [] }));
vi.mock('./providers.js', () => ({
  getAllProviders: (...args) => getAllProviders(...args),
  listProviders: async () => {
    const data = await getAllProviders().catch(() => null);
    return Array.isArray(data?.providers) ? data.providers : [];
  },
}));

const getLlamaServerEndpoint = vi.fn();
const relaunchLlamaServerWithTuning = vi.fn();
const captureLlamaServerConfig = vi.fn(async () => null);
const restoreLlamaServerConfig = vi.fn(async () => ({ restored: false, reason: 'nothing was captured' }));
vi.mock('./llamaServerManager.js', () => ({
  getLlamaServerEndpoint: (...args) => getLlamaServerEndpoint(...args),
  relaunchLlamaServerWithTuning: (...args) => relaunchLlamaServerWithTuning(...args),
  captureLlamaServerConfig: (...args) => captureLlamaServerConfig(...args),
  restoreLlamaServerConfig: (...args) => restoreLlamaServerConfig(...args),
}));

// MTPLX's knobs are `mtplx serve` flags, so applying one relaunches the PM2
// daemon. Left unmocked this suite would shell out to the developer's real PM2
// and probe their real :8000 — the same reason llama-server is mocked above.
const getMtplxServerEndpoint = vi.fn(async () => 'http://127.0.0.1:8000/v1');
const relaunchMtplxServerWithTuning = vi.fn(async () => ({ applied: true, reason: null, config: null }));
// Consulted only when the live probe fails, to list what is on disk for a
// runtime that is not running.
const getMtplxServerStatus = vi.fn(async () => ({ cachedModels: [] }));
vi.mock('./mtplxServerManager.js', () => ({
  getMtplxServerEndpoint: (...args) => getMtplxServerEndpoint(...args),
  getMtplxServerStatus: (...args) => getMtplxServerStatus(...args),
  relaunchMtplxServerWithTuning: (...args) => relaunchMtplxServerWithTuning(...args),
}));

const getSpecDecodePresetStatus = vi.fn(async () => []);
vi.mock('./specDecodeModels.js', () => ({
  getSpecDecodePresetStatus: (...args) => getSpecDecodePresetStatus(...args),
}));

// The managed-backend listing carries its own "empty vs unreadable" sentinel
// (`{ models, error }`), so these tests drive that shape directly rather than
// re-deriving it from a manager error getter. Its composition is covered in
// localLlm.test.js.
const listManagedBackendModels = vi.fn();
const managedModels = (models, error = null) => ({ models, error });
vi.mock('./localLlm.js', () => ({
  listManagedBackendModels: (...args) => listManagedBackendModels(...args),
}));

const getLoadedModels = vi.fn();
const ollamaVersion = vi.fn(async () => '0.0.0-test');
// Ollama's tuning knobs are daemon environment, so applying one restarts the
// daemon. Default to "it worked" and let a test override to assert the refusal path.
const restartOllamaWithEnv = vi.fn(async () => ({ applied: true, reason: 'restarted' }));
vi.mock('./ollamaManager.js', () => ({
  getLoadedModels: (...args) => getLoadedModels(...args),
  // Recorded with each assessment so a backend UPDATE can later be detected as
  // staleness — see localModelAssessmentStore.captureEnvironment.
  getVersion: (...args) => ollamaVersion(...args),
  restartWithEnv: (...args) => restartOllamaWithEnv(...args),
}));

// LM Studio's knobs are load-time, so applying one reloads the model via `lms load`.
const loadLmStudioModelWithArgs = vi.fn(async () => ({ success: true }));
vi.mock('./lmStudioManager.js', () => ({
  loadModelWithArgs: (...args) => loadLmStudioModelWithArgs(...args),
}));

// A fixed, generous memory budget so the memory axis is deterministic.
vi.mock('./localMemory.js', () => ({ getAvailableMemoryGb: async () => 64 }));

const svc = await import('./localModelAssessments.js');
// The durable store is a separate module (no path to a provider); the read-only
// projections the catalog badge consumes live there.
const store = await import('./localModelAssessmentStore.js');
// Dynamic, like the two above: a static import hoists ABOVE the fileUtils mock,
// and this module resolves its claim path from `PATHS.data` at load time.
const { claimHeavyLocalJob } = await import('../lib/heavyJobClaim.js');
const { TUNING_SPECS, tuningSpecsFor } = await import('../lib/localModelTuning.js');

const STORE = join(tempRoot, 'local-llm', 'assessments.json');

const okRun = (chars = 40, charsPerSecond = 120, ttftMs = 250) => ({
  text: 'The answer is 4.',
  timings: { charsPerSecond, ttftMs, totalMs: 800, chars },
});

afterAll(() => rmSync(tempRoot, { recursive: true, force: true }));

beforeEach(() => {
  rmSync(join(tempRoot, 'local-llm'), { recursive: true, force: true });
  runLocalLlmTest.mockReset();
  listManagedBackendModels.mockReset().mockResolvedValue(managedModels([{ id: 'example-model:7b', params: '7B' }]));
  getLoadedModels.mockReset().mockResolvedValue([{ id: 'example-model:7b', name: 'example-model:7b', size: 5 * 2 ** 30 }]);
  runEndpointLlmTest.mockReset();
  probeOpenAiModels.mockReset().mockResolvedValue({ reachable: true, models: [], error: null });
  getLlamaServerEndpoint.mockReset().mockResolvedValue('http://127.0.0.1:5568/v1');
  getAllProviders.mockReset().mockResolvedValue([]);
  // Mirrors each manager's real contract: an EMPTY request asks the daemon to
  // drop a tuning PortOS applied, and reports "nothing needed to change" when it
  // is already untuned. A flat `applied: true` here would let an untuned run
  // record the very claim `lib/localModelTuning.js` forbids.
  const empty = (v) => !v || (Array.isArray(v) ? v.length === 0 : Object.keys(v).length === 0);
  relaunchLlamaServerWithTuning.mockReset().mockImplementation(async (tuning) => (empty(tuning)
    ? { applied: null, reason: null, config: null }
    : { applied: true, reason: null, config: null }));
  restartOllamaWithEnv.mockReset().mockImplementation(async (env) => (empty(env)
    ? { applied: null, reason: 'already-untuned' }
    : { applied: true, reason: 'restarted' }));
  loadLmStudioModelWithArgs.mockReset().mockImplementation(async (_modelId, args) => (empty(args)
    ? { success: true, unchanged: true }
    : { success: true }));
  getMtplxServerEndpoint.mockReset().mockResolvedValue('http://127.0.0.1:8000/v1');
  relaunchMtplxServerWithTuning.mockReset().mockResolvedValue({ applied: true, reason: null, config: null });
});

describe('buildSamplePrompt', () => {
  it('scales the filler to the requested nominal context', () => {
    const small = svc.buildSamplePrompt(512);
    const large = svc.buildSamplePrompt(4096);
    expect(large.length).toBeGreaterThan(small.length * 4);
  });

  it('always ends with the question, so a model that ignores the filler still answers', () => {
    expect(svc.buildSamplePrompt(512).trimEnd()).toMatch(/what is 2 \+ 2\?/i);
  });

  it('uses distinct filler lines so a prefix cache cannot fake a cheap long prompt', () => {
    const prompt = svc.buildSamplePrompt(1024);
    expect(prompt).toContain('Reference item 1:');
    expect(prompt).toContain('Reference item 2:');
  });
});

describe('toSample', () => {
  it('records timings from a successful run', () => {
    expect(svc.toSample(512, okRun())).toMatchObject({
      contextTokens: 512, ok: true, charsPerSecond: 120, ttftMs: 250, totalMs: 800, error: null,
    });
  });

  it('treats an empty-text run as a FAILURE, not a zero-throughput success', () => {
    // runLocalLlmTest resolves rather than throwing on timeout, so an
    // error-free empty result is the shape a timed-out run actually takes.
    // Recording it as ok would feed a fabricated 0 into the speed average.
    const sample = svc.toSample(512, { text: '', timings: { totalMs: 120000 } });
    expect(sample.ok).toBe(false);
    expect(sample.charsPerSecond).toBeNull();
    expect(sample.error).toBe('model produced no output');
  });

  it('keeps the backend error verbatim', () => {
    expect(svc.toSample(512, { text: '', error: 'model requires more system memory' }).error)
      .toBe('model requires more system memory');
  });
});

describe('captureEnvironment', () => {
  it('records hardware shape without any machine identity', async () => {
    const env = await svc.captureEnvironment();
    expect(env).toHaveProperty('platform');
    expect(env).toHaveProperty('arch');
    expect(env.totalMemoryGb).toBeGreaterThan(0);
    expect(env.memoryBudgetGb).toBeGreaterThan(0);
    // Privacy: assessments end up in bug reports, so the record must never
    // carry a hostname, username, or path.
    const keys = Object.keys(env);
    expect(keys).not.toContain('hostname');
    expect(keys).not.toContain('username');
    for (const value of Object.values(env)) {
      if (typeof value === 'string') expect(value).not.toContain('/');
    }
  });
});

describe('loadAssessments / getAssessmentReport (read path)', () => {
  it('never calls a provider', async () => {
    await svc.loadAssessments();
    await svc.getAssessmentReport();
    // The AI Provider Usage Policy boundary: a read must be safe from boot,
    // from a poll, from anywhere.
    expect(runLocalLlmTest).not.toHaveBeenCalled();
  });

  it('reports an empty store as empty, with no read error', async () => {
    expect(await svc.loadAssessments()).toEqual([]);
    expect((await svc.getAssessmentReport()).readError).toBeNull();
  });

  it('lists installed-but-unmeasured models without ranking or penalizing them', async () => {
    const report = await svc.getAssessmentReport();
    expect(report.ranked).toEqual([]);
    expect(report.unassessed).toContainEqual({ backend: 'ollama', modelId: 'example-model:7b', params: '7B' });
  });

  // An embedding model has no chat/generation to measure, so it is not an
  // unanswered question — listing it here offered a Measure button that could
  // only ever spend a provider call to prove it fails.
  it('leaves embedding models out of the not-yet-measured list', async () => {
    listManagedBackendModels.mockImplementation(async (backend) => managedModels(backend === 'ollama'
      ? [{ id: 'example-model:7b', params: '7B' }, { id: 'all-minilm:latest' }, { id: 'nomic-embed-text:latest' }]
      : []));
    const report = await svc.getAssessmentReport();
    expect(report.unassessed.map((u) => u.modelId)).toEqual(['example-model:7b']);
    // ...and the consent gate's counts follow, since they derive from the same list.
    expect(report.sweepScopes.unmeasured).toBe(1);
  });

  // The backend's own capability metadata outranks the name heuristic in both
  // directions, so a chat model that merely LOOKS like an embedding one stays.
  it('keeps a model the backend reports as chat-capable', async () => {
    listManagedBackendModels.mockImplementation(async (backend) => managedModels(backend === 'ollama'
      ? [{ id: 'all-minilm:latest', capabilities: ['completion'] }]
      : []));
    const report = await svc.getAssessmentReport();
    expect(report.unassessed.map((u) => u.modelId)).toEqual(['all-minilm:latest']);
  });

  it('flags a backend whose model list could not be read, even though it returned []', async () => {
    // Both managers cache an EMPTY list on a failed read instead of throwing, so
    // the accompanying `error` is the only signal that separates "no models
    // installed" from "the list could not be read".
    listManagedBackendModels.mockImplementation(async (backend) => (backend === 'lmstudio'
      ? managedModels([], 'LM Studio is unavailable')
      : managedModels([{ id: 'example-model:7b', params: '7B' }])));
    const report = await svc.getAssessmentReport();
    expect(report.listErrors).toEqual(['lmstudio']);
  });

  it('still flags a backend when the model list throws outright', async () => {
    listManagedBackendModels.mockImplementation(async (backend) => (backend === 'ollama'
      ? managedModels(null, 'Ollama unreachable')
      : managedModels([])));
    expect((await svc.getAssessmentReport()).listErrors).toEqual(['ollama']);
  });

  it('falls back to the balanced intent for an unrecognized one', async () => {
    expect((await svc.getAssessmentReport({ intent: 'nonsense' })).intent).toBe('balanced');
  });

  it('leaves embedding-only models out of the measurable list', async () => {
    // They serve /api/embed, never /api/chat, so the benchmark sample comes back
    // `400 "<model>" does not support chat` — a failed run that also fires the
    // provider-failure hook and files a CoS investigation task. Offering a
    // Measure button for one, or sweeping it, is work that cannot succeed.
    listManagedBackendModels.mockImplementation(async (backend) => managedModels(backend === 'ollama'
      ? [{ id: 'example-model:7b', params: '7B' }, { id: 'nomic-embed-text:latest', params: '137M' }]
      : []));
    const report = await svc.getAssessmentReport();
    expect(report.unassessed.map((u) => u.modelId)).toEqual(['example-model:7b']);
    // Still counted as installed — the runtime really does serve it, and the
    // count is about the catalog, not about what a benchmark can measure.
    expect(report.runtimes.find((r) => r.id === 'ollama').modelCount).toBe(2);
  });
});

describe('runAssessment — machine-wide accelerator claim', () => {
  // A measurement is only valid if it had the machine to itself, so this is
  // enforced in the SERVICE, not by a disabled button: a second tab, ⌘K, or curl
  // has to hit the same wall. Refusing beats recording a corrupt reading.
  it('refuses when another heavy local job already holds the machine', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const held = await claimHeavyLocalJob({ kind: 'LoRA training', id: 'run-1' });
    expect(held.ok).toBe(true);
    try {
      await expect(svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] }))
        .rejects.toMatchObject({ code: 'HEAVY_LOCAL_JOB_BUSY' });
      // Nothing was measured, so nothing was recorded.
      expect(runLocalLlmTest).not.toHaveBeenCalled();
    } finally {
      await held.release();
    }
  });

  // The claim wait does not observe the abort signal, so a run cancelled WHILE
  // waiting would otherwise wake up and measure — relaunching llama-server under
  // the abandoned run's tuning and holding the machine against its replacement.
  it('does not measure a run that was cancelled while waiting for the claim', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const controller = new AbortController();
    controller.abort();

    const result = await svc.runAssessment({
      backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], signal: controller.signal,
    });

    expect(result.cancelled).toBe(true);
    expect(runLocalLlmTest).not.toHaveBeenCalled();
    expect(relaunchLlamaServerWithTuning).not.toHaveBeenCalled();
    // Nothing measured means nothing recorded.
    expect(await svc.loadAssessments()).toEqual([]);
    // ...and the claim went back, so the replacement sweep can start.
    const after = await claimHeavyLocalJob({ kind: 'probe', id: 'after-cancel' });
    expect(after.ok).toBe(true);
    await after.release();
  });

  it('releases the claim afterwards, so the next measurement can run', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    const after = await claimHeavyLocalJob({ kind: 'probe', id: 'after' });
    expect(after.ok).toBe(true);
    await after.release();
  });

  it('releases the claim even when the run throws', async () => {
    // A relaunch failure escapes before any sample runs; a leaked claim would
    // wedge every local render on the machine until the process exited.
    relaunchLlamaServerWithTuning.mockRejectedValueOnce(new Error('boom'));
    getLlamaServerEndpoint.mockResolvedValue(null);
    await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 } })
      .catch(() => null);
    const after = await claimHeavyLocalJob({ kind: 'probe', id: 'after-throw' });
    expect(after.ok).toBe(true);
    await after.release();
  });
});

describe('runAssessment — embedding models are not assessable', () => {
  // An assessment measures a model by GENERATING with it. Ollama answers every
  // sample aimed at an embedding-only model with
  // `400 "<model>" does not support chat`, so the run is doomed by the model id
  // alone — and each doomed sample raised an AI-provider investigation task.
  // `selectSweepTargets` keeps these out of a sweep; this covers the direct
  // `POST /api/local-llm/assessments/run` route, which never goes through it.
  it('refuses before claiming the machine or calling a provider', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await expect(svc.runAssessment({ backend: 'ollama', modelId: 'all-minilm:latest', contextTokens: [512] }))
      .rejects.toMatchObject({ code: 'MODEL_NOT_ASSESSABLE', status: 400 });
    expect(runLocalLlmTest).not.toHaveBeenCalled();
    // Nothing recorded — a stored `incompatible` reading would read as "this
    // machine cannot run it", which is not what happened.
    expect(await svc.loadAssessments()).toEqual([]);
    // ...and the machine claim was never taken, so the next job can start.
    const after = await claimHeavyLocalJob({ kind: 'probe', id: 'after-embed' });
    expect(after.ok).toBe(true);
    await after.release();
  });

  it('still measures a generation model whose id merely looks unusual', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    expect(result.verdict).toBe('fits');
  });
});

describe('runAssessment', () => {
  it('refuses an embedding-only model before claiming the machine', async () => {
    // Reaching here means a direct API call or a stale page — the report already
    // keeps embedding models out of `unassessed`. Every sample would come back
    // `400 "<model>" does not support chat`, which is not a verdict about this
    // machine, so fail fast with a reason instead of recording one.
    await expect(svc.runAssessment({ backend: 'ollama', modelId: 'nomic-embed-text:latest', contextTokens: [512] }))
      .rejects.toMatchObject({ code: 'MODEL_NOT_ASSESSABLE', status: 400 });
    expect(runLocalLlmTest).not.toHaveBeenCalled();
    expect(await svc.loadAssessments()).toEqual([]);
    // The claim was never taken, so the next measurement can start immediately.
    const after = await claimHeavyLocalJob({ kind: 'probe', id: 'after-embed-refusal' });
    expect(after.ok).toBe(true);
    await after.release();
  });

  it('samples every context in ascending order and persists the result', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [4096, 512] });

    expect(runLocalLlmTest.mock.calls.map((c) => c[0].prompt.length))
      .toEqual([...runLocalLlmTest.mock.calls.map((c) => c[0].prompt.length)].sort((a, b) => a - b));
    expect(result.verdict).toBe('fits');
    expect(result.samples.map((s) => s.contextTokens)).toEqual([512, 4096]);
    expect(existsSync(STORE)).toBe(true);
    expect((await svc.loadAssessments())[0].modelId).toBe('example-model:7b');
  });

  it('stops after a resource failure instead of burning minutes on larger contexts', async () => {
    runLocalLlmTest
      .mockResolvedValueOnce(okRun())
      .mockResolvedValueOnce({ text: '', error: 'model requires more system memory' })
      .mockResolvedValue(okRun());

    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512, 4096, 16384] });
    expect(runLocalLlmTest).toHaveBeenCalledTimes(2);
    // A smaller context worked, so the model DOES fit — the largest working
    // context is what carries the nuance.
    expect(result.verdict).toBe('fits');
    expect(result.performance.maxWorkingContextTokens).toBe(512);
  });

  it('records does-not-fit when every context exhausted memory', async () => {
    runLocalLlmTest.mockResolvedValue({ text: '', error: 'out of memory' });
    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512, 4096] });
    expect(result.verdict).toBe('does-not-fit');
    expect(result.verdictReason).toBe('out of memory');
    expect(result.residentGb).toBeNull();
  });

  it('turns a pre-stream throw into a recorded sample rather than losing the whole run', async () => {
    // runLocalLlmTest throws (not resolves) when the provider is unconfigured.
    // Letting that escape would abort the assessment with zero evidence.
    runLocalLlmTest.mockRejectedValue(new Error('Local provider "ollama" is not configured'));
    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    expect(result.samples).toHaveLength(1);
    expect(result.samples[0].error).toMatch(/not configured/);
    // Not attributable to hardware — must not become a permanent does-not-fit.
    expect(result.verdict).toBe('unknown');
  });

  it('records resident size for Ollama and null for LM Studio, which does not report one', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const ollama = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    expect(ollama.residentGb).toBe(5);

    const lmstudio = await svc.runAssessment({ backend: 'lmstudio', modelId: 'example-model:7b', contextTokens: [512] });
    // Copying the weight-file size here would silently re-introduce the estimate
    // this feature exists to replace.
    expect(lmstudio.residentGb).toBeNull();
  });

  it('replaces the previous measurement for the same model rather than accumulating', async () => {
    runLocalLlmTest.mockResolvedValue(okRun(40, 100));
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    runLocalLlmTest.mockResolvedValue(okRun(40, 200));
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });

    const stored = await svc.loadAssessments();
    expect(stored).toHaveLength(1);
    expect(stored[0].performance.meanCharsPerSecond).toBe(200);
  });

  it('keeps separate records per backend for the same model id', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    await svc.runAssessment({ backend: 'lmstudio', modelId: 'example-model:7b', contextTokens: [512] });
    expect(await svc.loadAssessments()).toHaveLength(2);
  });

  it('de-duplicates and sorts the requested context list', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [4096, 512, 4096] });
    expect(result.samples.map((s) => s.contextTokens)).toEqual([512, 4096]);
  });

  it('honors an already-aborted signal without calling the provider', async () => {
    const controller = new AbortController();
    controller.abort();
    const result = await svc.runAssessment({
      backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], signal: controller.signal,
    });
    expect(runLocalLlmTest).not.toHaveBeenCalled();
    expect(result.verdict).toBe('unknown');
  });

  it('feeds the ranking once evidence exists', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512, 4096] });
    const report = await svc.getAssessmentReport({ intent: 'fastest' });
    expect(report.ranked.map((r) => r.modelId)).toEqual(['example-model:7b']);
    // The stub lists the same model under both backends, so only the ollama
    // copy leaves the unassessed list — assessments are per (backend, model).
    expect(report.unassessed).toEqual([{ backend: 'lmstudio', modelId: 'example-model:7b', params: '7B' }]);
  });
});

describe('uninstalled models', () => {
  it('stops recommending a model the user has since deleted', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512, 4096] });

    listManagedBackendModels.mockResolvedValue(managedModels([]));
    const report = await svc.getAssessmentReport();
    // It can no longer run, so it must not be ranked — but the measurement stays
    // on disk so a re-install doesn't cost another run.
    expect(report.ranked).toEqual([]);
    expect(report.uninstalled).toContainEqual({ backend: 'ollama', modelId: 'example-model:7b', tuningLabel: null });
    expect(report.assessments).toHaveLength(1);
  });

  it('keeps recommending when the backend list is UNTRUSTWORTHY rather than wiping it', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512, 4096] });

    // An unreadable list returning [] must not be read as "everything was
    // uninstalled" — that is the same failed-read-as-empty mistake.
    listManagedBackendModels.mockResolvedValue(managedModels([], 'Ollama is unavailable'));
    const report = await svc.getAssessmentReport();
    expect(report.ranked.map((r) => r.modelId)).toEqual(['example-model:7b']);
    expect(report.uninstalled).toEqual([]);
  });
});

describe('cancellation', () => {
  it('does not persist a run the client aborted mid-way', async () => {
    // runLocalLlmTest turns a client disconnect into the same "Timed out" result
    // a real resource failure produces. Persisting it would record the user
    // closing the tab as `does-not-fit`.
    const controller = new AbortController();
    runLocalLlmTest.mockImplementation(async () => {
      controller.abort();
      return { text: '', error: 'Timed out after 120000ms' };
    });

    const result = await svc.runAssessment({
      backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512, 4096], signal: controller.signal,
    });
    expect(result.cancelled).toBe(true);
    expect(await svc.loadAssessments()).toEqual([]);
  });

  it('leaves an earlier measurement intact when a re-run is cancelled', async () => {
    runLocalLlmTest.mockResolvedValue(okRun(40, 150));
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });

    const controller = new AbortController();
    controller.abort();
    await svc.runAssessment({
      backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], signal: controller.signal,
    });

    const stored = await svc.loadAssessments();
    expect(stored).toHaveLength(1);
    expect(stored[0].performance.meanCharsPerSecond).toBe(150);
  });
});

describe('an unreadable store', () => {
  const corrupt = () => {
    mkdirSync(join(tempRoot, 'local-llm'), { recursive: true });
    writeFileSync(STORE, '{ this is not json');
  };

  it('reports the read error instead of claiming nothing was assessed', async () => {
    corrupt();
    expect((await svc.getAssessmentReport()).readError).toBeTruthy();
  });

  it('parks the bad file rather than overwriting it with a single new record', async () => {
    corrupt();
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });

    // Writing straight through would have replaced whatever the old file held
    // with just this record — a read failure destroying minutes of measured
    // compute. The old bytes survive alongside a fresh, working store.
    const parked = readdirSync(join(tempRoot, 'local-llm')).filter((f) => f.includes('.corrupt-'));
    expect(parked).toHaveLength(1);
    expect(await svc.loadAssessments()).toHaveLength(1);
  });

  it('refuses to rewrite the file on a delete it cannot read', async () => {
    corrupt();
    expect(await svc.deleteAssessment('ollama', 'example-model:7b')).toEqual({ deleted: false });
    expect(existsSync(STORE)).toBe(true);
  });
});

describe('deleteAssessment', () => {
  it('removes a record and reports that it did', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    expect(await svc.deleteAssessment('ollama', 'example-model:7b')).toEqual({ deleted: true });
    expect(await svc.loadAssessments()).toEqual([]);
  });

  it('reports a miss rather than a phantom success', async () => {
    expect(await svc.deleteAssessment('ollama', 'example-model:404')).toEqual({ deleted: false });
  });
});

describe('progress streaming', () => {
  it('reports each sample as it lands, then one terminal complete frame', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const frames = [];
    await svc.runAssessment({
      backend: 'ollama',
      modelId: 'example-model:7b',
      contextTokens: [512, 4096],
      onProgress: (frame) => frames.push(frame),
    });

    // Every frame carries enough to be correlated on a channel shared with
    // model pulls and migrations.
    expect(frames.every((f) => f.scope === 'assessment' && f.backend === 'ollama' && f.modelId === 'example-model:7b')).toBe(true);
    expect(frames.filter((f) => f.event === 'complete')).toHaveLength(1);
    expect(frames.at(-1)).toMatchObject({ event: 'complete', verdict: 'fits' });
    // Per-sample: one "about to run" and one "here is what it measured" each.
    const withContext = frames.filter((f) => f.contextTokens);
    expect(withContext.map((f) => f.contextTokens)).toEqual([512, 512, 4096, 4096]);
    expect(withContext.every((f) => f.sampleCount === 2)).toBe(true);
  });

  it('emits a terminal frame for a cancelled run so a listener never hangs', async () => {
    const controller = new AbortController();
    runLocalLlmTest.mockImplementation(async () => { controller.abort(); return okRun(); });
    const frames = [];
    await svc.runAssessment({
      backend: 'ollama',
      modelId: 'example-model:7b',
      contextTokens: [512],
      signal: controller.signal,
      onProgress: (frame) => frames.push(frame),
    });
    expect(frames.at(-1)).toMatchObject({ event: 'complete', cancelled: true });
  });

  it('does not let a broken progress listener abort the measurement', async () => {
    // The listener runs outside the request error path; a throw there must not
    // cost the user the minutes of compute already spent.
    runLocalLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({
      backend: 'ollama',
      modelId: 'example-model:7b',
      contextTokens: [512],
      onProgress: () => { throw new Error('listener exploded'); },
    });
    expect(result.verdict).toBe('fits');
  });
});

describe('staleness', () => {
  it('flags a stored reading taken on a different machine state', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });

    // Rewrite the recorded environment as if the box had half the RAM then.
    const store = JSON.parse(readFileSync(STORE, 'utf8'));
    store.assessments[0].environment.totalMemoryGb = 1;
    writeFileSync(STORE, JSON.stringify(store));

    const report = await svc.getAssessmentReport({});
    expect(report.assessments[0].staleness.stale).toBe(true);
    expect(report.assessments[0].staleness.description).toMatch(/installed memory/i);
    // And it travels with the ranked entry, so the panel doesn't recompute it.
    expect(report.ranked[0].staleness.stale).toBe(true);
  });

  it('does not flag a reading taken on the machine as it is now', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    const report = await svc.getAssessmentReport({});
    expect(report.assessments[0].staleness.stale).toBe(false);
    expect(report.liveEnvironments.ollama.platform).toBe(process.platform);
  });
});

describe('getMeasuredFits', () => {
  it('projects a stored assessment into the catalog fit vocabulary', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });

    const fits = await store.getMeasuredFits('ollama');
    expect(fits['example-model:7b']).toMatchObject({
      fit: 'comfortable', verdict: 'fits', stale: false, meanCharsPerSecond: 120, residentGb: 5,
    });
    // Scoped to the backend — an Ollama measurement says nothing about LM Studio.
    expect(await store.getMeasuredFits('lmstudio')).toEqual({});
  });

  it('returns an empty map when nothing has been measured', async () => {
    expect(await store.getMeasuredFits('ollama')).toEqual({});
  });

  it('marks a reading from a different machine state stale rather than hiding it', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    const raw = JSON.parse(readFileSync(STORE, 'utf8'));
    raw.assessments[0].environment.cpuCount = 1;
    writeFileSync(STORE, JSON.stringify(raw));

    const fits = await store.getMeasuredFits('ollama');
    expect(fits['example-model:7b'].stale).toBe(true);
    expect(fits['example-model:7b'].staleReason).toMatch(/CPU count/i);
  });
});

describe('backend-version staleness on the read paths', () => {
  beforeEach(() => store.__resetBackendVersionCache());

  it('marks a measurement stale after the backend is updated under it', async () => {
    ollamaVersion.mockResolvedValue('0.1.0');
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    expect((await store.getMeasuredFits('ollama'))['example-model:7b'].stale).toBe(false);

    store.__resetBackendVersionCache();
    ollamaVersion.mockResolvedValue('0.2.0');
    const fits = await store.getMeasuredFits('ollama');
    expect(fits['example-model:7b'].stale).toBe(true);
    expect(fits['example-model:7b'].staleReason).toMatch(/backend version 0\.1\.0 → 0\.2\.0/);
  });

  it('probes the version at most once per cache window, not once per keystroke', async () => {
    // The catalog path calls this on every debounced keystroke; an unconditional
    // probe there would be one loopback GET per character typed.
    ollamaVersion.mockClear();
    await store.getMeasuredFits('ollama');
    await store.getMeasuredFits('ollama');
    await store.getMeasuredFits('ollama');
    expect(ollamaVersion).toHaveBeenCalledTimes(1);
  });

  it('does not re-probe a backend that is down — null is a fetched answer', async () => {
    store.__resetBackendVersionCache();
    ollamaVersion.mockClear().mockResolvedValue(null);
    await store.getMeasuredFits('ollama');
    await store.getMeasuredFits('ollama');
    expect(ollamaVersion).toHaveBeenCalledTimes(1);
  });

  it('never probes for LM Studio, which reports no version at all', async () => {
    store.__resetBackendVersionCache();
    ollamaVersion.mockClear();
    await store.getMeasuredFits('lmstudio');
    expect(ollamaVersion).not.toHaveBeenCalled();
  });
});

// ---------------------------------------------------------------------------
// Endpoint runtimes (llama.cpp / MTPLX / vLLM) and launch tuning
// ---------------------------------------------------------------------------

describe('endpoint runtimes', () => {
  it('lists a bare daemon\'s models from GET /v1/models, with no params to guess from', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    const result = await svc.listRuntimeModels('mtplx');
    expect(result).toEqual({ models: [{ id: 'dflash', params: null, quantization: null }], error: null });
  });

  // A stopped daemon has no catalog on disk to fall back to, so "unreachable"
  // must surface as an ERROR. Reported as an empty list it would silently hide
  // every model behind a daemon the user only needs to start.
  it('reports an unreachable daemon as an error, never as an empty catalog', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: false, models: null, error: 'ECONNREFUSED' });
    const result = await svc.listRuntimeModels('vllm');
    expect(result.models).toBeNull();
    expect(result.error).toMatch(/not reachable/);
  });

  it('reads llama.cpp\'s endpoint from the running server, not from the default port', async () => {
    getLlamaServerEndpoint.mockResolvedValue('http://127.0.0.1:9999/v1');
    expect(await svc.runtimeEndpoint('llama')).toBe('http://127.0.0.1:9999/v1');
  });

  it('falls back to the canonical base URL when llama-server cannot be reached', async () => {
    getLlamaServerEndpoint.mockRejectedValue(new Error('pm2 is not installed'));
    expect(await svc.runtimeEndpoint('llama')).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/v1$/);
  });

  // The read path runs on every Performance page load. `getLlamaServerStatus`
  // costs a network probe plus an `execPm2 logs` subprocess whose output this
  // caller would throw away.
  it('resolves the endpoint without paying for a status probe', async () => {
    await svc.getAssessmentReport();
    expect(getLlamaServerEndpoint).toHaveBeenCalled();
  });

  it('measures an endpoint runtime directly instead of through the provider path', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({ backend: 'mtplx', modelId: 'dflash', contextTokens: [512] });
    expect(runLocalLlmTest).not.toHaveBeenCalled();
    expect(runEndpointLlmTest).toHaveBeenCalledWith(expect.objectContaining({ runtime: 'mtplx', modelId: 'dflash' }));
    expect(result.verdict).toBe('fits');
    expect(result.endpoint).toBeTruthy();
  });

  // Only Ollama's /api/ps reports a footprint. Copying a weight-file size in
  // would re-introduce the estimate this whole feature exists to replace.
  it('records no resident footprint for a runtime that reports none', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512] });
    expect(result.residentGb).toBeNull();
  });
});

// The refusal in `applyLaunchTuning` catches a runtime with NO applier. This
// catches the sharper case: a runtime WITH one, whose catalog declares a knob
// that applier's transport cannot carry. Nothing fails at runtime there — the
// relaunch runs, `tuningApplied: true` is recorded, and the knob is dropped on
// the way to the launch line, which is exactly the un-applied-but-claimed
// reading the catalog exists to prevent.
describe('launch transports', () => {
  it('carries every launch knob each runtime with an applier declares', () => {
    for (const [runtime, transport] of Object.entries(svc.LAUNCH_TRANSPORTS)) {
      expect(transport, `${runtime} applier declares no transport`).toBeTruthy();
      for (const spec of tuningSpecsFor(runtime)) {
        if (spec.applies !== 'launch') continue;
        expect(spec[transport], `${runtime}/${spec.id} is not a \`${transport}\` knob`).toBeTruthy();
      }
    }
  });

  // The mirror: a runtime that declares launch knobs but has no applier would
  // offer a form whose every field is refused at run time.
  it('gives every runtime that declares a launch knob an applier to apply it', () => {
    for (const [runtime, specs] of Object.entries(TUNING_SPECS)) {
      if (!specs.some((spec) => spec.applies === 'launch')) continue;
      expect(svc.LAUNCH_TRANSPORTS, `${runtime} declares launch knobs`).toHaveProperty(runtime);
    }
  });
});

describe('tuning', () => {
  it('keeps two tunings of one model as two records rather than overwriting', async () => {
    runLocalLlmTest.mockResolvedValue(okRun(40, 120));
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
    const stored = await svc.loadAssessments();
    expect(stored).toHaveLength(2);
    expect(stored.map((a) => a.tuningKey).sort()).toEqual(['', 'numCtx=8192']);
  });

  it('re-running the SAME tuning replaces that record, not the untuned one', async () => {
    runLocalLlmTest.mockResolvedValue(okRun(40, 120));
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
    expect(await svc.loadAssessments()).toHaveLength(2);
  });

  // Ollama's OpenAI-compatible endpoint drops unknown body fields, so the only
  // spelling of a context window is the daemon's environment. Sending one in the
  // body would look applied and change nothing.
  it('restarts Ollama with the tuning env instead of putting it in the request body', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({
      backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192, flashAttention: true },
    });
    expect(restartOllamaWithEnv).toHaveBeenCalledWith({ OLLAMA_CONTEXT_LENGTH: '8192', OLLAMA_FLASH_ATTENTION: '1' });
    expect(runLocalLlmTest).toHaveBeenCalledWith(expect.objectContaining({ extraBody: {} }));
    expect(result.tuningApplied).toBe(true);
  });

  it('records the reason when Ollama could not be restarted with the tuning', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    restartOllamaWithEnv.mockResolvedValueOnce({ applied: false, reason: 'stop-failed', error: 'Ollama would not stop' });
    const result = await svc.runAssessment({
      backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 },
    });
    expect(result.tuningApplied).toBe(false);
    expect(result.tuningNotApplied).toBe('Ollama would not stop');
  });

  // LM Studio picks context/offload/parallelism when the model LOADS, so a tuned
  // reading has to reload it through `lms load` first.
  it('reloads the LM Studio model with the tuning flags', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({
      backend: 'lmstudio', modelId: 'example-model:7b', contextTokens: [512], tuning: { contextLength: 8192, gpuOffload: 0.5 },
    });
    expect(loadLmStudioModelWithArgs).toHaveBeenCalledWith('example-model:7b', ['--context-length', '8192', '--gpu', '0.5']);
    expect(result.tuningApplied).toBe(true);
    expect(result.tuningLabel).toBe('Context length 8k · GPU offload 0.5');
  });

  it('records the reason when LM Studio refused the tuned load', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    loadLmStudioModelWithArgs.mockResolvedValueOnce({ success: false, error: 'Model does not fit at that context length' });
    const result = await svc.runAssessment({
      backend: 'lmstudio', modelId: 'example-model:7b', contextTokens: [512], tuning: { contextLength: 1048576 },
    });
    expect(result.tuningApplied).toBe(false);
    expect(result.tuningNotApplied).toBe('Model does not fit at that context length');
  });

  it('puts llama.cpp launch knobs on the command line before the first sample', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({
      backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 },
    });
    expect(relaunchLlamaServerWithTuning).toHaveBeenCalledWith({ ubatchSize: 512 }, { reset: false });
    expect(result.tuningApplied).toBe(true);
    expect(result.tuningLabel).toBe('Micro-batch size 512');
  });

  // A reading taken under a tuning PortOS could NOT apply describes some other
  // configuration. Recording it as evidence for the requested tuning would be
  // the same lie as a fabricated measurement.
  it('records that a launch tuning was not applied, with the reason', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    relaunchLlamaServerWithTuning.mockResolvedValue({ applied: false, reason: 'llama-server is not running', config: null });
    const result = await svc.runAssessment({
      backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 },
    });
    expect(result.tuningApplied).toBe(false);
    expect(result.tuningNotApplied).toBe('llama-server is not running');
  });

  // An untuned run applied nothing — `true` there is the exact claim
  // lib/localModelTuning.js forbids, and `false` would imply something went
  // wrong. `null` is the honest answer.
  it('records tuningApplied as null when there was nothing to apply', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    const untuned = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    expect(untuned.tuningApplied).toBeNull();
    expect(untuned.tuningNotApplied).toBeNull();
  });

  // The bug this covers: an untuned run took the no-applier branch, so a daemon
  // the PREVIOUS run tuned kept serving under that tuning while the reading was
  // stored with `tuningKey: ''` and rendered as "Backend defaults". Every applier
  // that CAN reset has to be asked to.
  it.each([
    ['ollama', () => restartOllamaWithEnv, (m) => expect(m).toHaveBeenCalledWith({})],
    ['llama', () => relaunchLlamaServerWithTuning, (m) => expect(m).toHaveBeenCalledWith({}, { reset: false })],
    ['lmstudio', () => loadLmStudioModelWithArgs, (m) => expect(m).toHaveBeenCalledWith('a-model', [])],
  ])('asks the %s applier to drop any tuning still on the daemon for an untuned run', async (backend, getMock, assertCall) => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['a-model'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    runLocalLlmTest.mockResolvedValue(okRun());

    const result = await svc.runAssessment({ backend, modelId: 'a-model', contextTokens: [512] });

    assertCall(getMock());
    expect(result.tuningKey).toBe('');
  });

  // A daemon that could NOT be put back on its defaults is measuring some other
  // configuration, and the record has to say so rather than call it a baseline.
  it('records that an untuned run could not clear the tuning still on the daemon', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    restartOllamaWithEnv.mockResolvedValue({ applied: false, reason: 'stop-failed', error: 'Ollama would not stop' });

    const result = await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });

    expect(result.tuningKey).toBe('');
    expect(result.tuningApplied).toBe(false);
    expect(result.tuningNotApplied).toBe('Ollama would not stop');
  });

  // A runtime with no launch path must not silently swallow a knob: the catalog
  // offers none for vLLM (a container from the shipped compose stack), and if
  // one is ever added without a transport the reading has to say so.
  it('refuses launch tuning for a runtime PortOS does not start', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['a-model'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({
      backend: 'vllm', modelId: 'a-model', contextTokens: [512], tuning: { maxKvSize: 8192 },
    });
    // Not in the catalog, so it normalizes away entirely — no claim made.
    expect(result.tuningApplied).toBeNull();
    expect(result.tuningKey).toBe('');
  });

  it('relaunches `mtplx serve` to apply an MTPLX tuning', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['a-model'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    const result = await svc.runAssessment({
      backend: 'mtplx', modelId: 'a-model', contextTokens: [512], tuning: { depth: 5, kvQuant: 'q8' },
    });
    expect(relaunchMtplxServerWithTuning).toHaveBeenCalledWith({ depth: 5, kvQuant: 'q8' });
    expect(result.tuningApplied).toBe(true);
    expect(result.tuningKey).toBe('depth=5,kvQuant=q8');
  });

  // A launch line MTPLX rejects must not be filed as evidence FOR that tuning —
  // the manager puts the previous configuration back and says why, and the
  // reading has to carry the refusal rather than the claim.
  it('records an MTPLX tuning the daemon refused as not applied', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['a-model'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    relaunchMtplxServerWithTuning.mockResolvedValueOnce({
      applied: false, reason: 'MTPLX rejected that tuning: unrecognized arguments', config: null,
    });
    const result = await svc.runAssessment({
      backend: 'mtplx', modelId: 'a-model', contextTokens: [512], tuning: { contextWindow: 1048576 },
    });
    expect(result.tuningApplied).toBe(false);
    expect(result.tuningNotApplied).toMatch(/unrecognized arguments/);
  });

  // The user's launch flags live only in the running llama-server process, so an
  // ordinary measurement must never touch them. A reset is something a SWEEP
  // asks for, and only a sweep, because only a sweep puts the configuration back.
  it('never RESETS llama-server for an untuned run', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512] });
    // Asked to undo what PortOS applied, never to clear the user's own flags.
    expect(relaunchLlamaServerWithTuning).toHaveBeenCalledWith({}, { reset: false });
  });

  // The baseline variant of a tuning sweep: no knobs, which only means something
  // when the caller asked for a complete tuning.
  it('applies an empty tuning as a reset when the caller asked for one', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512], resetTuning: true });
    expect(relaunchLlamaServerWithTuning).toHaveBeenCalledWith({}, { reset: true });
  });

  // Only a runtime PortOS can reset AND put back may be swept — otherwise the
  // sweep leaves its knobs set for good and every later "backend defaults"
  // reading is taken under them (#4763).
  it('reports which runtimes a tuning sweep may drive', () => {
    expect(svc.isTuningSweepable('llama')).toBe(true);
    expect(svc.isTuningSweepable('ollama')).toBe(false);
    expect(svc.isTuningSweepable('lmstudio')).toBe(false);
    expect(svc.isTuningSweepable('mtplx')).toBe(false);
  });

  it('captures and restores nothing for a runtime with no launch state', async () => {
    expect(await svc.captureLaunchState('ollama')).toBeNull();
    expect(await svc.restoreLaunchState('ollama', null)).toEqual({ restored: false, reason: 'nothing to restore' });
  });

  it('deletes one tuning of a model and leaves the others', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
    expect(await svc.deleteAssessment('ollama', 'example-model:7b', 'numCtx=8192')).toEqual({ deleted: true });
    const stored = await svc.loadAssessments();
    expect(stored.map((a) => a.tuningKey)).toEqual(['']);
  });

  it('reports which tuning won once a model has two measurements', async () => {
    runLocalLlmTest.mockResolvedValueOnce(okRun(40, 90)).mockResolvedValueOnce(okRun(40, 150));
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512] });
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
    const report = await svc.getAssessmentReport();
    expect(report.tuningComparison).toHaveLength(1);
    expect(report.tuningComparison[0].best.label).toBe('Context size 8k');
  });

  // Once ONE tuning is measured the model is no longer an unanswered question.
  // Re-listing it under every un-run tuning would make the section unbounded.
  it('drops a model from "not yet measured" as soon as any tuning is recorded', async () => {
    runLocalLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'ollama', modelId: 'example-model:7b', contextTokens: [512], tuning: { numCtx: 8192 } });
    const report = await svc.getAssessmentReport();
    expect(report.unassessed.some((u) => u.backend === 'ollama')).toBe(false);
  });
});

describe('runtime roster', () => {
  it('reports every assessable runtime with its knob catalog', async () => {
    const report = await svc.getAssessmentReport();
    expect(report.runtimes.map((r) => r.id)).toEqual(['ollama', 'lmstudio', 'llama', 'mtplx', 'vllm', 'sglang', 'slotstream']);
    expect(report.runtimes.find((r) => r.id === 'llama').tuningSpecs.some((s) => s.id === 'ubatchSize')).toBe(true);
  });

  // `null` = the listing failed so the count is UNKNOWN; `0` = read, and this
  // runtime genuinely serves nothing. A UI must be able to tell them apart.
  it('reports an unknown model count as null and an empty one as 0', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: false, models: null, error: 'ECONNREFUSED' });
    const report = await svc.getAssessmentReport();
    expect(report.runtimes.find((r) => r.id === 'mtplx').modelCount).toBeNull();
    expect(report.runtimes.find((r) => r.id === 'ollama').modelCount).toBe(1);
  });
});


// ---------------------------------------------------------------------------
// Key-gated runtimes, and evidence hygiene for a tuning that never landed
// ---------------------------------------------------------------------------

describe('key-gated runtimes', () => {
  // `vllmBacked` is the structural marker `localRuntimeKind` keys on — it
  // deliberately does NOT derive the backend from an editable name or endpoint.
  const vllmProvider = { id: 'vllm', name: 'vLLM', vllmBacked: true, endpoint: 'http://127.0.0.1:18020/v1', apiKey: 'secret-key' };

  // A vLLM container from the shipped compose stack sets VLLM_API_KEY and 401s
  // an unauthenticated request. Without the key the listing reads as
  // "unreadable" and every sample fails auth — recorded as a fit verdict.
  it('authenticates the model listing with the provider record\'s key', async () => {
    getAllProviders.mockResolvedValue({ activeProvider: null, providers: [vllmProvider] });
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['qwen'], error: null });
    await svc.listRuntimeModels('vllm');
    expect(probeOpenAiModels).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ apiKey: 'secret-key' }));
  });

  it('authenticates the measurement with the same key', async () => {
    getAllProviders.mockResolvedValue({ activeProvider: null, providers: [vllmProvider] });
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['qwen'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun());
    await svc.runAssessment({ backend: 'vllm', modelId: 'qwen', contextTokens: [512] });
    expect(runEndpointLlmTest).toHaveBeenCalledWith(expect.objectContaining({ apiKey: 'secret-key' }));
  });

  // The usual loopback daemon is unauthenticated; attaching a key from an
  // unrelated provider would be worse than sending none.
  it('sends no key when no provider for that runtime carries one', async () => {
    getAllProviders.mockResolvedValue({ activeProvider: null, providers: [{ id: 'ollama', ollamaBacked: true, endpoint: 'http://localhost:11434/v1', apiKey: 'not-mine' }] });
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: [], error: null });
    await svc.listRuntimeModels('mtplx');
    expect(probeOpenAiModels).toHaveBeenCalledWith(expect.any(String), expect.objectContaining({ apiKey: '' }));
  });
});

describe('unapplied tuning is not evidence', () => {
  const measureWithUnappliedTuning = async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun(40, 150));
    relaunchLlamaServerWithTuning.mockResolvedValue({ applied: false, reason: 'llama-server is not running', config: null });
    await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 } });
  };

  // The numbers describe the configuration that was ACTUALLY running. Ranking
  // them would recommend a tuning nobody measured.
  it('keeps the record but never ranks it', async () => {
    await measureWithUnappliedTuning();
    const report = await svc.getAssessmentReport();
    expect(report.assessments).toHaveLength(1);
    expect(report.ranked).toEqual([]);
    expect(report.excluded[0].reason).toMatch(/tuning was not applied/);
    expect(report.excluded[0].tuningKey).toBe('ubatchSize=512');
  });

  // Comparing it would credit the previous config's throughput to knobs that
  // never reached the daemon.
  it('never lets it win a tuning comparison', async () => {
    // A real, applied backend-defaults reading first — llama is an endpoint
    // runtime, so it measures through runEndpointLlmTest either way.
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun(40, 90));
    await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512] });
    // …then a faster one whose tuning never reached the daemon. Two records
    // exist, so the only thing stopping a comparison is the exclusion itself.
    await measureWithUnappliedTuning();
    expect(await svc.loadAssessments()).toHaveLength(2);
    const report = await svc.getAssessmentReport();
    expect(report.tuningComparison).toEqual([]);
  });

  it('still ranks a tuning that WAS applied', async () => {
    probeOpenAiModels.mockResolvedValue({ reachable: true, models: ['dflash'], error: null });
    runEndpointLlmTest.mockResolvedValue(okRun(40, 150));
    await svc.runAssessment({ backend: 'llama', modelId: 'dflash', contextTokens: [512], tuning: { ubatchSize: 512 } });
    const report = await svc.getAssessmentReport();
    expect(report.ranked.map((r) => r.tuningKey)).toEqual(['ubatchSize=512']);
  });
});