Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

37.2 KB
Raw Blame
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';

// The sweep's only two collaborators: the report (what to measure) and the
// single-model runner (measure it). Both are mocked so the queue's own behavior
// — order, cancellation, failure isolation — is what's under test.
// The restore stops and starts the daemon, so it takes the same machine-wide
// claim a measurement does — otherwise it can bounce llama-server underneath a
// reading somebody started the moment Stop re-enabled the button.
const claimHeavyLocalJob = vi.fn(async () => ({ ok: true, release: vi.fn(async () => {}) }));
vi.mock('../lib/heavyJobClaim.js', () => ({ claimHeavyLocalJob: (...args) => claimHeavyLocalJob(...args) }));

vi.mock('./localModelAssessments.js', () => ({
  getAssessmentReport: vi.fn(),
  runAssessment: vi.fn(),
  // A sweep may only drive a runtime it can reset AND put back. llama.cpp can;
  // the default here mirrors that so the tests exercise the real gate.
  isTuningSweepable: vi.fn((backend) => backend === 'llama'),
  captureLaunchState: vi.fn(async () => ({ model: '/models/example.gguf' })),
  restoreLaunchState: vi.fn(async () => ({ restored: true, reason: null })),
}));

import {
  captureLaunchState, getAssessmentReport, isTuningSweepable, restoreLaunchState, runAssessment,
} from './localModelAssessments.js';
import { startSweep, getSweepStatus, cancelSweep, __resetSweep } from './localModelAssessmentSweep.js';
// Not mocked: the grid is pure catalog data, and asserting the queue against the
// real one is what proves the service runs the configurations it advertises.
import { tuningGridFor } from '../lib/localModelTuning.js';

// Let the detached loop run to completion. The sweep deliberately does NOT
// return a promise for its own queue (the HTTP handler must return immediately),
// so a test has to drain the microtask queue instead.
const settle = async () => { for (let i = 0; i < 50; i += 1) await Promise.resolve(); };

const report = ({ unassessed = [], assessments = [], uninstalled = [] } = {}) => ({
  unassessed, assessments, uninstalled,
});

const measured = (verdict = 'fits', meanTokensPerSecond = 42) => ({
  verdict, performance: { meanTokensPerSecond, meanCharsPerSecond: 170 },
});

beforeEach(() => {
  vi.clearAllMocks();
  claimHeavyLocalJob.mockResolvedValue({ ok: true, release: vi.fn(async () => {}) });
  isTuningSweepable.mockImplementation((backend) => backend === 'llama');
  captureLaunchState.mockResolvedValue({ model: '/models/example.gguf' });
  restoreLaunchState.mockResolvedValue({ restored: true, reason: null });
  __resetSweep();
});
afterEach(() => __resetSweep());

describe('startSweep', () => {
  it('measures every unmeasured model in order and records what each produced', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'ollama', modelId: 'a' }, { backend: 'ollama', modelId: 'b' }],
    }));
    runAssessment.mockResolvedValue(measured());

    const changes = [];
    const started = await startSweep({ scope: 'unmeasured', onChange: (state) => changes.push(state) });
    expect(started.status).toBe('running');
    expect(started.total).toBe(2);

    await settle();
    const status = getSweepStatus();
    expect(status.status).toBe('complete');
    expect(status.results.map((r) => r.modelId)).toEqual(['a', 'b']);
    expect(status.results[0].meanTokensPerSecond).toBe(42);
    // Sequential by design — two models at once would measure the contention.
    expect(runAssessment).toHaveBeenCalledTimes(2);
    expect(changes.map((state) => [state.status, state.completed, state.current?.modelId ?? null, state.settled])).toEqual([
      ['running', 0, null, false],
      ['running', 0, 'a', false],
      ['running', 1, null, false],
      ['running', 1, 'b', false],
      ['running', 2, null, false],
      ['complete', 2, null, true],
    ]);
    expect(changes[0].results).toEqual([]);
    for (const state of changes) {
      expect(state).not.toHaveProperty('controller');
      expect(state).not.toHaveProperty('notifyChange');
      expect(state).not.toHaveProperty('launchStates');
    }
  });

  it('passes each target\'s recorded tuning through, so a re-measure reproduces it', async () => {
    getAssessmentReport.mockResolvedValue(report({
      assessments: [{
        backend: 'llama', modelId: 'tuned', tuningKey: 'ctx=8192', tuningLabel: '8k',
        tuning: { contextSize: 8192 }, staleness: { stale: true },
      }],
    }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'stale' });
    await settle();
    expect(runAssessment).toHaveBeenCalledWith(expect.objectContaining({
      backend: 'llama', modelId: 'tuned', tuning: { contextSize: 8192 },
    }));
  });

  // The point of an overnight run is that it gets through the list. One model
  // that throws must not abandon the rest of the queue.
  it('records a failed model as a result and keeps going', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'ollama', modelId: 'broken' }, { backend: 'ollama', modelId: 'fine' }],
    }));
    runAssessment
      .mockRejectedValueOnce(new Error('backend refused the model'))
      .mockResolvedValueOnce(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();
    const status = getSweepStatus();
    expect(status.status).toBe('complete');
    expect(status.results[0]).toMatchObject({ modelId: 'broken', error: 'backend refused the model', verdict: null });
    expect(status.results[1]).toMatchObject({ modelId: 'fine', verdict: 'fits' });
  });

  it('never queues a model that is no longer installed', async () => {
    getAssessmentReport.mockResolvedValue(report({
      assessments: [
        { backend: 'ollama', modelId: 'gone', tuningKey: '', tuning: {}, staleness: { stale: true } },
        { backend: 'ollama', modelId: 'here', tuningKey: '', tuning: {}, staleness: { stale: true } },
      ],
      uninstalled: [{ backend: 'ollama', modelId: 'gone' }],
    }));
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ scope: 'stale' });
    expect(started.total).toBe(1);
    await settle();
    expect(runAssessment).toHaveBeenCalledWith(expect.objectContaining({ modelId: 'here' }));
  });

  // A refused start has to SAY so — a silent no-op would leave the page waiting
  // for progress that never arrives.
  it('refuses a second sweep while one is running', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    // Never resolves: the first sweep stays in flight for the duration.
    runAssessment.mockImplementation(() => new Promise(() => {}));

    await startSweep({ scope: 'unmeasured' });
    const second = await startSweep({ scope: 'unmeasured' });
    expect(second.rejected).toBe('a sweep is already running');
  });

  // The "already running" check and the slot claim must not be split by the
  // report await: two requests landing in that window would each publish a
  // `sweep` and launch a detached loop, so two overnight queues would measure the
  // same models against each other with only the second one's status visible.
  it('refuses a second sweep that arrives while the first is still reading the report', async () => {
    let releaseReport;
    getAssessmentReport.mockImplementationOnce(() => new Promise((resolve) => { releaseReport = resolve; }));
    runAssessment.mockResolvedValue(measured());

    const first = startSweep({ scope: 'unmeasured' });
    // Arrives before the report resolves — the first sweep object does not exist yet.
    const second = await startSweep({ scope: 'unmeasured' });
    expect(second.rejected).toBe('a sweep is already running');

    releaseReport(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    expect((await first).status).toBe('running');
    await settle();
    expect(runAssessment).toHaveBeenCalledTimes(1);
  });

  // A refused start must free the slot again, or one empty scope wedges the
  // feature until the process restarts.
  it('frees the slot when the start is refused', async () => {
    getAssessmentReport.mockResolvedValueOnce(report());
    expect((await startSweep({ scope: 'all' })).rejected).toBe('nothing to measure for that scope');

    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());
    expect((await startSweep({ scope: 'unmeasured' })).status).toBe('running');
  });

  // Every sample of an embedding-only model comes back
  // `400 "<model>" does not support chat`, so the selector drops it — which
  // would leave a NAMED one looking un-installed. Say what is actually wrong.
  it('refuses a named embedding model without calling it not installed', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'ollama', modelId: 'all-minilm:latest' }],
    }));
    const result = await startSweep({ backend: 'ollama', modelId: 'all-minilm:latest' });
    expect(result.rejected).toMatch(/embedding model/i);
    expect(result.rejected).not.toMatch(/not installed/i);
    expect(runAssessment).not.toHaveBeenCalled();
  });

  it('skips embedding models in a scope sweep and measures the rest', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [
        { backend: 'ollama', modelId: 'all-minilm:latest' },
        { backend: 'ollama', modelId: 'nomic-embed-text:latest' },
        { backend: 'ollama', modelId: 'qwen3.6:35b' },
      ],
    }));
    runAssessment.mockResolvedValue(measured());
    await startSweep({ scope: 'unmeasured' });
    await settle();
    expect(runAssessment.mock.calls.map((c) => c[0].modelId)).toEqual(['qwen3.6:35b']);
  });

  it('refuses a scope that covers nothing', async () => {
    getAssessmentReport.mockResolvedValue(report());
    const result = await startSweep({ scope: 'all' });
    expect(result.rejected).toBe('nothing to measure for that scope');
    expect(runAssessment).not.toHaveBeenCalled();
  });

  it('emits start and per-model progress frames under the sweep scope', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());
    const frames = [];

    await startSweep({ scope: 'unmeasured', onProgress: (f) => frames.push(f) });
    await settle();
    expect(frames.map((f) => f.event)).toEqual(['start', 'model-start', 'complete']);
    expect(frames.every((f) => f.scope === 'assessment-sweep')).toBe(true);
  });

  it('survives a progress listener that throws', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured', onProgress: () => { throw new Error('socket closed'); }, onChange: () => { throw new Error('socket closed'); } });
    await settle();
    expect(getSweepStatus().status).toBe('complete');
  });
});

describe('cancelSweep', () => {
  it('stops the queue and keeps what was already measured', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'ollama', modelId: 'a' }, { backend: 'ollama', modelId: 'b' }],
    }));
    // First model lands; the second reports itself cancelled, as runAssessment
    // does when it sees the abort signal.
    runAssessment
      .mockResolvedValueOnce(measured())
      .mockResolvedValueOnce({ cancelled: true });

    await startSweep({ scope: 'unmeasured' });
    await Promise.resolve();
    cancelSweep();
    await settle();

    const status = getSweepStatus();
    expect(status.status).toBe('cancelled');
    // The abandoned model is NOT a result — recording it would make a stopped
    // sweep look like it measured what it gave up on.
    expect(status.results.map((r) => r.modelId)).toEqual(['a']);
  });

  it('is a no-op when nothing is running', () => {
    expect(cancelSweep().status).toBe('idle');
  });

  // Cancelling frees the slot immediately, so a replacement sweep can start
  // while the abandoned one is still unwinding its last model. That loop must
  // not write its results (or its terminal frame) into the new queue.
  // A cancelled sweep is not finished with the machine: its last measurement is
  // still aborting, and a tuning sweep still has a launch configuration to put
  // back. A replacement started on top of that would be measuring through the
  // old sweep's relaunch.
  it('refuses a replacement while the cancelled sweep is still winding down', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'slow' }] }));
    let releaseFirst;
    runAssessment.mockImplementationOnce(() => new Promise((resolve) => { releaseFirst = resolve; }));

    await startSweep({ scope: 'unmeasured' });
    cancelSweep();

    const refused = await startSweep({ scope: 'unmeasured' });
    expect(refused.rejected).toMatch(/winding down/);

    releaseFirst(measured('fits', 9));
    await settle();
  });

  it('lets the next sweep start once the cancelled one has finished winding down', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'slow' }] }));
    let releaseFirst;
    runAssessment.mockImplementationOnce(() => new Promise((resolve) => { releaseFirst = resolve; }));

    await startSweep({ scope: 'unmeasured' });
    cancelSweep();
    releaseFirst({ cancelled: true });
    await settle();

    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'second' }] }));
    runAssessment.mockResolvedValue(measured());
    const started = await startSweep({ scope: 'unmeasured' });

    expect(started.rejected).toBeUndefined();
    await settle();
    expect(getSweepStatus().results.map((r) => r.modelId)).toEqual(['second']);
  });
});

describe('getSweepStatus', () => {
  it('reports idle before anything has been started', () => {
    expect(getSweepStatus()).toMatchObject({ status: 'idle', total: 0, completed: 0, current: null, results: [] });
  });

  it('never exposes the abort controller', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockImplementation(() => new Promise(() => {}));
    await startSweep({ scope: 'unmeasured' });
    expect(getSweepStatus().controller).toBeUndefined();
  });
});

describe('startSweep — the tuning dimension', () => {
  // The llama.cpp grid the service derives: backend defaults plus one variant
  // per sweepable knob. Asserted against the real catalog rather than a fixture,
  // because "the queue is the grid" is exactly the claim under test.
  const llamaGrid = tuningGridFor('llama');

  const onlyTarget = () => getAssessmentReport.mockResolvedValue(
    report({ unassessed: [{ backend: 'llama', modelId: 'target' }] })
  );

  it('measures one named model once per tuning, baseline first', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    expect(started.total).toBe(llamaGrid.length);
    expect(started.mode).toBe('tunings');
    expect(started.target).toEqual({ backend: 'llama', modelId: 'target' });

    await settle();
    expect(runAssessment.mock.calls.map((c) => c[0].tuning)).toEqual(llamaGrid.map((v) => v.tuning));
    // Every variant is the same model, so the label is the only thing that
    // distinguishes one store record from the next.
    expect(getSweepStatus().results.map((r) => r.tuningLabel)).toEqual(llamaGrid.map((v) => v.label));
  });

  // A separate store record per variant is what `compareTunings` groups on — two
  // variants sharing a key would overwrite each other and leave nothing to rank.
  it('gives every variant a distinct tuning, so each lands as its own record', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    const keys = runAssessment.mock.calls.map((c) => JSON.stringify(c[0].tuning));
    expect(new Set(keys).size).toBe(keys.length);
  });

  // The grid is a domain decision, not a request parameter: a caller that could
  // hand one over could ask for an arbitrary batch of provider calls, and the
  // count the consent gate named would stop being the count that runs.
  it('derives the grid itself rather than accepting one', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());

    await startSweep({ backend: 'llama', modelId: 'target', tunings: [{ ubatchSize: 99999 }] });
    await settle();
    expect(runAssessment.mock.calls.map((c) => c[0].tuning)).toEqual(llamaGrid.map((v) => v.tuning));
  });

  // Queuing a grid against a model that is not there would burn one doomed
  // measurement per variant and report the sweep as failed, not as a bad request.
  it('refuses a model no runtime has installed', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'llama', modelId: 'other' }] }));

    const result = await startSweep({ backend: 'llama', modelId: 'ghost', tunings: true });
    expect(result.rejected).toMatch(/not installed/);
    expect(runAssessment).not.toHaveBeenCalled();
  });

  // Naming a model leaves the scope nothing to decide. Without that, "sweep THIS
  // model" would silently do nothing whenever the model fell outside the scope —
  // and an already-measured model falls outside 'unmeasured' by definition.
  it('sweeps a named model the scope would have excluded', async () => {
    getAssessmentReport.mockResolvedValue(report({
      assessments: [{ backend: 'llama', modelId: 'measured-before', staleness: { stale: false } }],
    }));
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ scope: 'unmeasured', backend: 'llama', modelId: 'measured-before', tunings: true });
    expect(started.total).toBe(llamaGrid.length);
  });

  // A model with two stored tunings is still ONE model to cross with the grid;
  // measuring each variant twice would put the same configuration on both sides
  // of the comparison table.
  it('crosses the grid once for a model that holds several stored tunings', async () => {
    getAssessmentReport.mockResolvedValue(report({
      assessments: [
        { backend: 'llama', modelId: 'target', tuningKey: '', tuning: {} },
        { backend: 'llama', modelId: 'target', tuningKey: 'flashAttn=true', tuning: { flashAttn: true } },
      ],
    }));
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    expect(started.total).toBe(llamaGrid.length);
  });

  // A runtime PortOS cannot pass flags to yields the baseline alone — one
  // measurement with nothing to compare it to.
  it('refuses a runtime with no sweepable knob instead of measuring the baseline alone', async () => {
    isTuningSweepable.mockReturnValue(true);
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'mtplx', modelId: 'target' }] }));

    const result = await startSweep({ backend: 'mtplx', modelId: 'target', tunings: true });
    expect(result.rejected).toMatch(/no tuning knobs/);
    expect(runAssessment).not.toHaveBeenCalled();
  });

  // A runtime whose knobs survive the sweep would leave them set for good AND
  // measure every later "backend defaults" reading under them — a comparison
  // that reads as valid and is not (#4763).
  it('refuses a runtime it cannot reset and put back', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'target' }] }));

    const result = await startSweep({ backend: 'ollama', modelId: 'target', tunings: true });
    expect(result.rejected).toMatch(/put the runtime back/);
    expect(runAssessment).not.toHaveBeenCalled();
    expect(getAssessmentReport).not.toHaveBeenCalled();
  });

  // Each variant's launch line has to be exactly that variant's knobs, and the
  // baseline's none of them — that is what the manager's reset does, and a
  // sweep that did not ask for it would measure accumulating launch lines.
  it('asks for a complete tuning on every variant, so none can accumulate', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    for (const [args] of runAssessment.mock.calls) expect(args.resetTuning).toBe(true);
  });

  // A plain endpoint model sweep re-measures each model under the tuning it
  // already carries. Endpoint runtimes have no sweep-safe reset/restore path.
  it('does not reset an endpoint model sweep', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();
    expect(runAssessment.mock.calls[0][0].resetTuning).toBe(false);
    expect(captureLaunchState).not.toHaveBeenCalled();
  });

  it('resets each llama model to its recorded tuning and restores the daemon afterwards', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [
        { backend: 'llama', modelId: 'model-a' },
        { backend: 'llama', modelId: 'model-b' },
      ],
    }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();

    expect(captureLaunchState).toHaveBeenCalledWith('llama');
    expect(runAssessment.mock.calls.map(([args]) => args.resetTuning)).toEqual([true, true]);
    expect(restoreLaunchState).toHaveBeenCalledWith('llama', { model: '/models/example.gguf' });
  });

  // The running daemon is the only record of the launch flags the user chose, so
  // a sweep that rewrote it has to put it back — however the queue ended.
  it('captures the launch configuration first and restores it when the queue ends', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    expect(captureLaunchState).toHaveBeenCalledWith('llama');
    await settle();
    expect(restoreLaunchState).toHaveBeenCalledWith('llama', { model: '/models/example.gguf' });
  });

  it('restores the launch configuration after a cancelled sweep too', async () => {
    onlyTarget();
    runAssessment.mockImplementationOnce(() => { cancelSweep(); return Promise.resolve({ cancelled: true }); });

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    expect(restoreLaunchState).toHaveBeenCalledWith('llama', { model: '/models/example.gguf' });
  });

  // A restore that throws must not take the process down or leave the page
  // waiting on a sweep that never reports finished.
  it('finishes the sweep even when the restore fails', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());
    restoreLaunchState.mockRejectedValue(new Error('llama-server would not come back'));

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    expect(getSweepStatus().status).toBe('complete');
  });

  // A tuning sweep varies the configuration of ONE model, so without a model
  // there is nothing to hold constant.
  it('refuses a tuning sweep with no model to sweep', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'llama', modelId: 'a' }] }));

    const result = await startSweep({ scope: 'all', tunings: true });
    expect(result.rejected).toMatch(/needs a model/);
    expect(getAssessmentReport).not.toHaveBeenCalled();
  });

  // The queue is shared, so cancelling a tuning sweep has to behave exactly as
  // cancelling a model sweep does — stop after the measurement in flight, keep
  // what already landed.
  it('stops after the in-flight variant when cancelled mid-sweep', async () => {
    onlyTarget();
    runAssessment
      .mockResolvedValueOnce(measured())
      .mockImplementationOnce(() => { cancelSweep(); return Promise.resolve({ cancelled: true }); });

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    const status = getSweepStatus();
    expect(status.status).toBe('cancelled');
    expect(status.results).toHaveLength(1);
    expect(runAssessment).toHaveBeenCalledTimes(2);
  });

  // Every step of a tuning sweep names the same model, so a frame without the
  // label reads as one measurement repeating.
  it('names the tuning in each progress frame', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());
    const frames = [];

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true, onProgress: (f) => frames.push(f) });
    await settle();
    const starts = frames.filter((f) => f.event === 'model-start');
    expect(starts.map((f) => f.tuningLabel)).toEqual(llamaGrid.map((v) => v.label));
    expect(starts[1].message).toContain(llamaGrid[1].label);
  });

  // The report already refuses to RANK a reading whose tuning never reached the
  // daemon. The sweep's own results list said nothing, so a variant that failed
  // to launch read as a clean measurement of the tuning it was labelled with —
  // the exact row a tuning sweep is opened for.
  it('carries an unapplied tuning through to the result, not just to the store', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue({
      ...measured(),
      tuningApplied: false,
      tuningNotApplied: 'llama-server rejected that tuning',
    });

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    expect(getSweepStatus().results[0]).toMatchObject({
      tuningApplied: false,
      tuningNotApplied: 'llama-server rejected that tuning',
    });
  });

  // Nothing to capture means llama-server is stopped or someone else started it.
  // PortOS cannot put a tuning on its launch line, so every variant would record
  // "tuning not applied" and the comparison would rank configurations that never
  // ran — minutes of GPU for an answer that is not one.
  it('refuses when there is no launch configuration to vary', async () => {
    onlyTarget();
    captureLaunchState.mockResolvedValue(null);

    const result = await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    expect(result.rejected).toMatch(/start it from the LLMs page/);
    expect(runAssessment).not.toHaveBeenCalled();
  });

  // A restore stops and starts the daemon, which is exactly the disruption the
  // machine-wide claim exists to keep away from a measurement — and by then
  // `runAssessment` has released it and the page has re-enabled Measure.
  it('holds the machine-wide claim while it puts the launch configuration back', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());
    const release = vi.fn(async () => {});
    claimHeavyLocalJob.mockResolvedValue({ ok: true, release });

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    expect(claimHeavyLocalJob).toHaveBeenCalledWith(expect.objectContaining({ kind: 'local-model assessment' }));
    expect(release).toHaveBeenCalled();
  });

  // A claim it cannot get is not a reason to skip: the daemon would stay on the
  // sweep's last variant, which is the thing the restore exists to prevent.
  it('restores anyway when the claim cannot be had', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());
    claimHeavyLocalJob.mockResolvedValue({ ok: false, message: 'busy' });

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    expect(restoreLaunchState).toHaveBeenCalledWith('llama', { model: '/models/example.gguf' });
  });

  // A model sweep captured nothing, so there is nothing to put back and no
  // reason to take the machine for it.
  it('claims nothing when there is nothing to restore', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();
    expect(claimHeavyLocalJob).not.toHaveBeenCalled();
  });

  // `settled` outlasts `status`: the page keeps its per-model actions disabled
  // while a stopped sweep is still aborting and still restoring.
  it('pushes cancellation immediately and settlement only after restoring the runtime', async () => {
    onlyTarget();
    let releaseRun;
    let releaseRestore;
    const changes = [];
    runAssessment.mockImplementationOnce(() => new Promise((resolve) => { releaseRun = resolve; }));
    restoreLaunchState.mockImplementationOnce(() => new Promise((resolve) => { releaseRestore = resolve; }));

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true, onChange: (state) => changes.push(state) });
    cancelSweep();
    expect(changes.at(-1)).toMatchObject({ status: 'cancelled', settled: false, cancelRequested: true });
    releaseRun({ cancelled: true });
    await settle();
    expect(restoreLaunchState).toHaveBeenCalled();
    expect(changes.at(-1).settled).toBe(false);
    releaseRestore({ restored: true });
    await settle();
    expect(changes.at(-1)).toMatchObject({ status: 'cancelled', settled: true, current: null });
    expect(changes.filter((state) => state.settled)).toHaveLength(1);
  });

  // The measurements all landed, so the sweep did not fail — but the runtime is
  // now serving the last variant's launch line, and nothing else on the page
  // would say so.
  it('reports a restore that did not happen without calling the sweep failed', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());
    restoreLaunchState.mockResolvedValue({ restored: false, reason: 'llama-server is no longer PortOS-managed' });

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    const status = getSweepStatus();
    expect(status.status).toBe('complete');
    expect(status.error).toBeNull();
    expect(status.restoreError).toMatch(/no longer PortOS-managed/);
  });

  it('reports a restore that threw', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());
    restoreLaunchState.mockRejectedValue(new Error('llama-server would not come back'));

    await startSweep({ backend: 'llama', modelId: 'target', tunings: true });
    await settle();
    expect(getSweepStatus().restoreError).toMatch(/would not come back/);
  });

  // A model sweep captured nothing, so "not restored" is the honest no-op — not
  // something to alarm the user about.
  it('does not report a restore failure for a sweep that captured nothing', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();
    expect(getSweepStatus().restoreError).toBeNull();
  });

  it('reports the model sweep as mode "models" with no named target', async () => {
    getAssessmentReport.mockResolvedValue(report({ unassessed: [{ backend: 'ollama', modelId: 'a' }] }));
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ scope: 'unmeasured' });
    expect(started.mode).toBe('models');
    expect(started.target).toBeNull();
    expect(started.scope).toBe('unmeasured');
  });

  // A tuning sweep of one named model never consulted the scope, so reporting
  // one would tell the page it measured a set it never looked at.
  it('reports no scope for a sweep of one named model', async () => {
    onlyTarget();
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ scope: 'all', backend: 'llama', modelId: 'target', tunings: true });
    expect(started.scope).toBeNull();
  });
});

// A MODEL sweep may reset a runtime only where it can put the launch line back,
// and that permission is the CAPTURE, not the runtime's `sweepable` flag alone.
// llama-server started outside PortOS captures nothing, and a sweep that reset
// it anyway would clear flags the user chose with no record of what they were —
// while refusing outright would cancel every Ollama and LM Studio model queued
// alongside it.
describe('startSweep — a model sweep across runtimes', () => {
  const twoLlamaModels = () => getAssessmentReport.mockResolvedValue(report({
    assessments: [
      {
        backend: 'llama', modelId: 'first', tuningKey: 'batchSize=4096', tuningLabel: 'batch 4096',
        tuning: { batchSize: 4096 }, staleness: { stale: true },
      },
      {
        backend: 'llama', modelId: 'second', tuningKey: 'ubatchSize=1024', tuningLabel: 'ubatch 1024',
        tuning: { ubatchSize: 1024 }, staleness: { stale: true },
      },
    ],
  }));

  // The seam this module owns. What a reset then DOES — clear every sweepable
  // knob the new tuning does not name — is `llamaServerManager`'s contract, and
  // its own suite asserts the second launch line carries none of the first's.
  it('gives each model the complete tuning its record names', async () => {
    twoLlamaModels();
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'stale' });
    await settle();
    expect(runAssessment.mock.calls.map(([args]) => args)).toMatchObject([
      { modelId: 'first', tuning: { batchSize: 4096 }, resetTuning: true },
      { modelId: 'second', tuning: { ubatchSize: 1024 }, resetTuning: true },
    ]);
  });

  it('captures a runtime once for the whole queue, however many models it holds', async () => {
    twoLlamaModels();
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'stale' });
    expect(captureLaunchState).toHaveBeenCalledTimes(1);
    await settle();
    expect(restoreLaunchState).toHaveBeenCalledTimes(1);
    expect(restoreLaunchState).toHaveBeenCalledWith('llama', { model: '/models/example.gguf' });
  });

  it('resets only the sweepable runtimes in a mixed sweep', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'llama', modelId: 'gguf' }, { backend: 'ollama', modelId: 'pulled' }],
    }));
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();
    expect(captureLaunchState).toHaveBeenCalledWith('llama');
    expect(captureLaunchState).not.toHaveBeenCalledWith('ollama');
    expect(runAssessment.mock.calls.map(([args]) => args)).toMatchObject([
      { backend: 'llama', resetTuning: true },
      { backend: 'ollama', resetTuning: false },
    ]);
  });

  // The regression this guards: refusing the whole queue over one runtime PortOS
  // does not own throws away every measurement it could still have taken.
  it('measures the rest of the queue when a runtime cannot be captured', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'llama', modelId: 'gguf' }, { backend: 'ollama', modelId: 'pulled' }],
    }));
    captureLaunchState.mockResolvedValue(null);
    runAssessment.mockResolvedValue(measured());

    const started = await startSweep({ scope: 'unmeasured' });
    expect(started.rejected).toBeUndefined();
    expect(started.total).toBe(2);
    await settle();
    expect(getSweepStatus().results).toHaveLength(2);
  });

  // Nothing was captured, so nothing may be cleared — the measurement falls back
  // to the behavior it had before the reset existed rather than wiping launch
  // flags with no record of what they were.
  it('never resets a runtime whose launch line it could not capture', async () => {
    twoLlamaModels();
    captureLaunchState.mockResolvedValue(null);
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'stale' });
    await settle();
    for (const [args] of runAssessment.mock.calls) expect(args.resetTuning).toBe(false);
    expect(restoreLaunchState).not.toHaveBeenCalled();
    expect(getSweepStatus().restoreError).toBeNull();
  });

  // A TUNING sweep of the same uncapturable runtime still refuses — varying the
  // launch line is its entire job, so it is left with nothing to do. That is
  // "refuses when there is no launch configuration to vary" above; the asymmetry
  // between the two sweeps is deliberate.

  // Two runtimes can be left wrong at once, and one that will not come back must
  // not strand the next — so the restores are caught individually and the reason
  // names which runtime to go fix.
  it('restores every captured runtime even when one of them throws', async () => {
    getAssessmentReport.mockResolvedValue(report({
      unassessed: [{ backend: 'llama', modelId: 'gguf' }, { backend: 'mtplx', modelId: 'mlx' }],
    }));
    isTuningSweepable.mockImplementation((backend) => backend === 'llama' || backend === 'mtplx');
    captureLaunchState.mockImplementation(async (backend) => ({ model: `/models/${backend}.bin` }));
    restoreLaunchState.mockImplementation(async (backend) => {
      if (backend === 'llama') throw new Error('llama-server would not come back');
      return { restored: true, reason: null };
    });
    runAssessment.mockResolvedValue(measured());

    await startSweep({ scope: 'unmeasured' });
    await settle();
    expect(restoreLaunchState).toHaveBeenCalledWith('mtplx', { model: '/models/mtplx.bin' });
    expect(getSweepStatus().restoreError).toBe('llama: llama-server would not come back');
  });
});