Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

44.6 KB
Raw Blame
import { EventEmitter } from 'node:events';
import { registerModelObservationSocket } from '../services/modelObservation.js';
import { beforeEach, describe, expect, it, vi } from 'vitest';
import express from 'express';
import { request } from '../lib/testHelper.js';
import localLlmRoutes from './localLlm.js';
import { runLocalLlmTest, compareLocalLlmModels } from '../services/localLlmPlayground.js';
import { listModels, listVisionModels, listToolUseModels, installModel } from '../services/localLlm.js';
import { enrichCatalogWithVariants, applyMeasuredFit } from '../services/huggingFaceCatalog.js';
import { getMeasuredFits } from '../services/localModelAssessmentStore.js';
import { runAssessment } from '../services/localModelAssessments.js';
import { startSweep, getSweepStatus, cancelSweep } from '../services/localModelAssessmentSweep.js';
import { runOpenCodeAgentBenchmark } from '../services/localModelAgentBenchmark.js';
import { getLoadedModels, unloadModel } from '../services/ollamaManager.js';
import { getLoadedModels as getLoadedLmStudioModels, getLastLoadedModelsError as getLmStudioResidencyError } from '../services/lmStudioManager.js';
import { getSettings, readSettingsStrict, updateSettingsWith } from '../services/settings.js';
import { localLlmCompareSchema, localLlmTestSchema } from '../lib/validation.js';
import { errorEvents } from '../lib/errorHandler.js';

const getModelAbuseGuardStatus = vi.fn(async () => ({ ready: false, modelCached: false, runtimeReady: false }));
const installModelAbuseGuard = vi.fn(async () => ({ ok: true, ready: true }));
const cancelModelAbuseGuardInstall = vi.fn();

// asyncHandler emits to errorEvents on every route failure; with `io` set on
// the app it always fires. Swallow it so a validation-rejection test doesn't
// trip Node's "unhandled 'error' event" — assertions go through the response.
errorEvents.on('error', () => {});

vi.mock('../services/localLlm.js', () => ({
  getStatus: vi.fn(),
  listModels: vi.fn(async () => []),
  listVisionModels: vi.fn(async () => []),
  listToolUseModels: vi.fn(async () => []),
  installModel: vi.fn(),
  previewInstallModel: vi.fn(async () => ({
    kind: 'install', verdict: 'ok', destPath: '/tmp/models', expectedBytes: 0, freeBytes: 1e12, requiredBytes: 0, headroomBytes: 0,
  })),
  describeInstallProgress: vi.fn((p) => p?.status || null),
  deleteModel: vi.fn(),
  switchBackend: vi.fn(),
  migrateBackend: vi.fn(),
  installBackend: vi.fn(),
  upgradeBackend: vi.fn(),
  controlOllamaServer: vi.fn(),
}));

vi.mock('../services/modelAbuseGuard.js', () => ({
  getModelAbuseGuardStatus: (...args) => getModelAbuseGuardStatus(...args),
  installModelAbuseGuard: (...args) => installModelAbuseGuard(...args),
  cancelModelAbuseGuardInstall: (...args) => cancelModelAbuseGuardInstall(...args),
}));

vi.mock('../services/untrustedContent.js', () => ({
  getContributionAssessmentReadiness: vi.fn(async () => ({ ready: false, code: 'untrusted-content-model-required', sources: {} })),
}));

vi.mock('../services/localLlmPlayground.js', () => ({
  runLocalLlmTest: vi.fn(),
  compareLocalLlmModels: vi.fn(),
}));

vi.mock('../services/ollamaManager.js', () => ({
  getLastLoadedModelsError: vi.fn(() => null),
  getLoadedModels: vi.fn(async () => []),
  unloadModel: vi.fn(),
  // Reached via providerExecutionReadiness.js's managed-runtime table
  // (issue #8104) whenever an unmocked module transitively imports
  // aiProvider.js — this suite never drives that path, so a no-op stub keeps
  // the module graph loadable.
  ensureProviderReady: vi.fn(async () => ({ success: true })),
  isOllamaProvider: vi.fn(() => false),
}));

vi.mock('../services/lmStudioManager.js', () => ({
  getLastLoadedModelsError: vi.fn(() => null),
  getLoadedModels: vi.fn(async () => []),
}));

// Mock the HF catalog service so /catalog tests don't hit the network. The no-op
// enrichCatalogWithVariants simulates the offline/failed-enrichment case (it leaves
// the catalog's getCatalog overlay untouched), which is exactly when the route's
// raw-id installed overlay must be correct.
vi.mock('../services/huggingFaceCatalog.js', () => ({
  searchHuggingFaceModels: vi.fn(async () => []),
  enrichCatalogWithVariants: vi.fn(async (catalog) => catalog),
  applyMeasuredFit: vi.fn((models) => models),
}));

// Measured assessments are folded into the catalog fit badge. Disk-only in
// production; mocked here so a catalog listing test never touches the store.
vi.mock('../services/localModelAssessmentStore.js', () => ({
  getMeasuredFits: vi.fn(async () => ({})),
}));

vi.mock('../services/localModelAssessments.js', () => ({
  getAssessmentReport: vi.fn(async () => ({ ranked: [], excluded: [] })),
  runAssessment: vi.fn(async () => ({ verdict: 'fits' })),
  deleteAssessment: vi.fn(async () => ({ deleted: true })),
}));

vi.mock('../services/localModelAssessmentSweep.js', () => ({
  startSweep: vi.fn(async () => ({ status: 'running', total: 3, completed: 0 })),
  getSweepStatus: vi.fn(() => ({ status: 'idle', total: 0, completed: 0, results: [] })),
  cancelSweep: vi.fn(() => ({ status: 'cancelled', total: 3, completed: 1, results: [] })),
}));

vi.mock('../services/localModelAgentBenchmark.js', () => ({
  runOpenCodeAgentBenchmark: vi.fn(async ({ backend, modelId, timeoutMs }) => ({
    backend, modelId, timeoutMs, completed: true, taskCharsPerSecond: 100,
  })),
}));

vi.mock('../services/llamaServerManager.js', () => ({
  getLlamaServerStatus: vi.fn(async () => ({ installed: true, running: false })),
  getLlamaServerUpdateStatus: vi.fn(async () => ({
    version: null,
    latestVersion: null,
    updateAvailable: false,
    canUpgrade: false,
    downloadUrl: 'https://github.com/ggml-org/llama.cpp/releases',
  })),
  startLlamaServer: vi.fn(async () => ({ success: true, pid: 123 })),
  stopLlamaServer: vi.fn(async () => ({ success: true })),
  installLlamaServer: vi.fn(async () => ({ success: true, message: 'installed' })),
  upgradeLlamaServer: vi.fn(async () => ({ success: true, note: 'updated' })),
}));

// The launcher's curated presets (and their weights' on-disk state) ride along
// on the status response so the card can offer a Download button per GGUF.
vi.mock('../services/specDecodeModels.js', () => ({
  getSpecDecodePresetStatus: vi.fn(async () => ([{ id: 'test-preset', label: 'Test', specType: 'draft-dspark', model: null, draftModel: null }])),
  downloadSpecDecodeModel: vi.fn(async () => ({ success: true, path: 'models/base.gguf', file: 'base.gguf' })),
  previewSpecDecodeDownload: vi.fn(async () => ({
    kind: 'spec-decode', verdict: 'ok', destPath: 'models/base.gguf', expectedBytes: 6, freeBytes: 1e12, requiredBytes: 6, headroomBytes: 0, alreadyDownloaded: false,
  })),
  cancelSpecDecodeModelDownload: vi.fn(() => true),
  removeSpecDecodeModel: vi.fn(async () => ({ success: true, deleted: true, path: 'models/base.gguf' })),
}));

// /loaded reads getSettings() to honor a user's intentionally-disabled backends,
// so mock it (defaults to no backends disabled; the disabled-case test flips it).
vi.mock('../services/settings.js', () => ({
  getSettings: vi.fn(async () => ({})),
  readSettingsStrict: vi.fn(),
  updateSettingsWith: vi.fn(),
  // localPersistentMindSetup → cos → cosTaskStore → codeReview listens on boot.
  settingsEvents: { on: vi.fn(), emit: vi.fn(), setMaxListeners: vi.fn() },
}));

function makeApp() {
  const app = express();
  app.use(express.json());
  app.set('io', { emit: vi.fn() });
  app.use('/api/local-llm', localLlmRoutes);
  return app;
}

describe('local LLM playground routes', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('GET /catalog?variants=1 keeps an installed LM Studio recommendation installed (raw-id overlay)', async () => {
    // The route appends LM Studio's quantization to the enrichment installed list,
    // but getCatalog's normalizer can't parse `@quant` — so the overlay must get the
    // RAW ids, or an already-installed model wrongly shows Install. (Enrichment is
    // mocked to a no-op, simulating HF being unreachable.)
    listModels.mockResolvedValue([{ id: 'lmstudio-community/gemma-4-12B-it-GGUF', quantization: 'Q4_K_M' }]);

    const res = await request(makeApp()).get('/api/local-llm/catalog?backend=lmstudio&variants=1');

    expect(res.status).toBe(200);
    expect(enrichCatalogWithVariants).toHaveBeenCalled();
    const entry = res.body.models.find((m) => m.id === 'lmstudio-community/gemma-4-12B-it-GGUF');
    expect(entry).toBeTruthy();
    expect(entry.installed).toBe(true);
  });

  it('GET /catalog skips HF enrichment unless variants=1 (fast local path for the playground)', async () => {
    const res = await request(makeApp()).get('/api/local-llm/catalog?backend=ollama');

    expect(res.status).toBe(200);
    expect(Array.isArray(res.body.models)).toBe(true);
    // The playground only needs catalog metadata — it must not pay for HF probes.
    expect(enrichCatalogWithVariants).not.toHaveBeenCalled();
  });

  it('GET /catalog exposes the managed security classifier separately from chat models', async () => {
    const res = await request(makeApp()).get('/api/local-llm/catalog?backend=ollama');

    expect(res.status).toBe(200);
    expect(res.body.securityGuards).toEqual([expect.objectContaining({
      id: 'llama-prompt-guard-2-86m',
      pipelineTag: 'text-classification',
      runtime: 'python-transformers',
    })]);
    expect(res.body.models.some((model) => model.id === 'llama-prompt-guard-2-86m')).toBe(false);
  });

  // Same guarantee for the jev scorer: a descriptor that leaked into `models`
  // would be offered by every provider/model picker that reads the catalog.
  it('GET /catalog exposes the jev decision scorer separately from chat models', async () => {
    const res = await request(makeApp()).get('/api/local-llm/catalog?backend=ollama');

    expect(res.status).toBe(200);
    expect(res.body.decisionScorers).toEqual([expect.objectContaining({
      id: 'openjev-qwen3.5-4b-nli',
      pipelineTag: 'text-classification',
      runtime: 'python-transformers',
    })]);
    expect(res.body.models.some((model) => model.id === 'openjev-qwen3.5-4b-nli')).toBe(false);
  });

  it('keeps the Prompt Guard lifecycle out of the general model installer', async () => {
    const status = await request(makeApp()).get('/api/local-llm/security-guard/status');
    expect(status.status).toBe(200);
    expect(getModelAbuseGuardStatus).toHaveBeenCalled();
    // A ready classifier must not mask a contribution assessment with no usable provider/model.
    expect(status.body.contributionAssessment).toEqual(expect.objectContaining({ ready: false, code: 'untrusted-content-model-required' }));

    const install = await request(makeApp()).post('/api/local-llm/security-guard/install').send({});
    expect(install.status).toBe(200);
    expect(installModelAbuseGuard).toHaveBeenCalledWith({ onEvent: expect.any(Function) });
    expect(installModel).not.toHaveBeenCalled();

    const cancel = await request(makeApp()).post('/api/local-llm/security-guard/install/cancel').send({});
    expect(cancel.status).toBe(200);
    expect(cancel.body).toEqual({ cancelled: true });
    expect(cancelModelAbuseGuardInstall).toHaveBeenCalled();
  });

  it('returns actionable Prompt Guard access prerequisites from the install route', async () => {
    installModelAbuseGuard.mockResolvedValueOnce({
      ok: false,
      code: 'security-guard-huggingface-token-required',
    });
    const missingToken = await request(makeApp()).post('/api/local-llm/security-guard/install').send({});
    expect(missingToken.status).toBe(502);
    expect(missingToken.body).toMatchObject({
      code: 'security-guard-huggingface-token-required',
      error: 'Add a Hugging Face read token before installing Prompt Guard.',
    });

    installModelAbuseGuard.mockResolvedValueOnce({
      ok: false,
      code: 'security-guard-huggingface-access-required',
    });
    const missingAccess = await request(makeApp()).post('/api/local-llm/security-guard/install').send({});
    expect(missingAccess.status).toBe(502);
    expect(missingAccess.body).toMatchObject({
      code: 'security-guard-huggingface-access-required',
      error: 'Hugging Face has not granted Prompt Guard access yet. Submit the usage request on its model card, then retry.',
    });
  });

  it('POST /install forwards force so a redownload can replace on-disk weights', async () => {
    installModel.mockResolvedValue({ success: true, modelId: 'unsloth/Qwen3.8-27B-GGUF@UD-Q4_K_M' });

    const res = await request(makeApp())
      .post('/api/local-llm/install')
      .send({ backend: 'lmstudio', modelId: 'unsloth/Qwen3.8-27B-GGUF@UD-Q4_K_M', force: true });

    expect(res.status).toBe(200);
    expect(installModel).toHaveBeenCalledWith(
      'lmstudio',
      'unsloth/Qwen3.8-27B-GGUF@UD-Q4_K_M',
      expect.any(Function),
      { force: true }
    );
  });

  it('POST /install without force still pulls as a regular install', async () => {
    installModel.mockResolvedValue({ success: true, modelId: 'llama3.2' });

    const res = await request(makeApp())
      .post('/api/local-llm/install')
      .send({ backend: 'ollama', modelId: 'llama3.2' });

    expect(res.status).toBe(200);
    expect(installModel).toHaveBeenCalledWith('ollama', 'llama3.2', expect.any(Function), { force: false });
  });

  it('runs a single local model test with validated defaults', async () => {
    runLocalLlmTest.mockResolvedValue({
      backend: 'ollama',
      modelId: 'llama3.2',
      text: 'hello',
      runId: 'run-1',
      timings: { totalMs: 25, ttftMs: 10, chars: 5, charsPerSecond: 200 },
    });

    const res = await request(makeApp())
      .post('/api/local-llm/test')
      .send({ backend: 'ollama', modelId: 'llama3.2', prompt: 'Say hello' });

    expect(res.status).toBe(200);
    expect(res.body.text).toBe('hello');
    expect(runLocalLlmTest).toHaveBeenCalledWith({
      backend: 'ollama',
      modelId: 'llama3.2',
      prompt: 'Say hello',
      systemPrompt: '',
      temperature: 0.3,
      maxTokens: 1000,
      timeoutMs: 300000,
      // Derived from res — lets a client disconnect tear down the upstream stream.
      signal: expect.any(AbortSignal),
    });
  });

  it('streams tokens then a terminal result frame as NDJSON', async () => {
    runLocalLlmTest.mockImplementation(async ({ onToken }) => {
      onToken('Hel');
      onToken('lo');
      return { backend: 'ollama', modelId: 'llama3.2', text: 'Hello', runId: 'run-1' };
    });

    const res = await request(makeApp())
      .post('/api/local-llm/test/stream')
      .send({ backend: 'ollama', modelId: 'llama3.2', prompt: 'Say hello' });

    expect(res.status).toBe(200);
    expect(res.headers['content-type']).toMatch(/application\/x-ndjson/);
    const frames = res.text.trim().split('\n').map((l) => JSON.parse(l));
    expect(frames).toEqual([
      { type: 'token', delta: 'Hel', kind: 'content' },
      { type: 'token', delta: 'lo', kind: 'content' },
      { type: 'result', result: { backend: 'ollama', modelId: 'llama3.2', text: 'Hello', runId: 'run-1' } },
    ]);
  });

  it('tags reasoning tokens with kind:reasoning so the client can render them separately', async () => {
    runLocalLlmTest.mockImplementation(async ({ onToken }) => {
      onToken('thinking…', 'reasoning');
      onToken('Answer.', 'content');
      return { backend: 'ollama', modelId: 'deepseek-r1', text: 'Answer.', runId: 'run-2' };
    });

    const res = await request(makeApp())
      .post('/api/local-llm/test/stream')
      .send({ backend: 'ollama', modelId: 'deepseek-r1', prompt: 'Think then answer' });

    expect(res.status).toBe(200);
    const frames = res.text.trim().split('\n').map((l) => JSON.parse(l));
    expect(frames).toEqual([
      { type: 'token', delta: 'thinking…', kind: 'reasoning' },
      { type: 'token', delta: 'Answer.', kind: 'content' },
      { type: 'result', result: { backend: 'ollama', modelId: 'deepseek-r1', text: 'Answer.', runId: 'run-2' } },
    ]);
  });

  it('emits exactly one terminal result frame (no extra 500) when the run resolves an in-stream error', async () => {
    // A timed-out/aborted run resolves an { error, text } result rather than
    // throwing — the route must surface it as the single terminal frame.
    runLocalLlmTest.mockResolvedValue({
      backend: 'ollama', modelId: 'llama3.2', error: 'Timed out after 5000ms', text: 'partial',
    });

    const res = await request(makeApp())
      .post('/api/local-llm/test/stream')
      .send({ backend: 'ollama', modelId: 'llama3.2', prompt: 'Say hello' });

    expect(res.status).toBe(200);
    const frames = res.text.trim().split('\n').map((l) => JSON.parse(l));
    expect(frames).toEqual([
      { type: 'result', result: { backend: 'ollama', modelId: 'llama3.2', error: 'Timed out after 5000ms', text: 'partial' } },
    ]);
  });

  it('converts a pre-stream provider throw into a terminal error result frame (no 500 after headers)', async () => {
    runLocalLlmTest.mockRejectedValue(new Error('Local provider "ollama" is not configured'));

    const res = await request(makeApp())
      .post('/api/local-llm/test/stream')
      .send({ backend: 'ollama', modelId: 'llama3.2', prompt: 'Say hello' });

    expect(res.status).toBe(200); // headers flushed before the throw — never a JSON 500
    const frames = res.text.trim().split('\n').map((l) => JSON.parse(l));
    expect(frames).toHaveLength(1);
    expect(frames[0]).toMatchObject({ type: 'result', result: { error: 'Local provider "ollama" is not configured', text: '' } });
  });

  it('compares models in the requested execution mode', async () => {
    compareLocalLlmModels.mockResolvedValue({
      mode: 'parallel',
      results: [
        { backend: 'ollama', modelId: 'a', text: 'A' },
        { backend: 'lmstudio', modelId: 'b', text: 'B' },
      ],
    });

    const targets = [
      { backend: 'ollama', modelId: 'a' },
      { backend: 'lmstudio', modelId: 'b' },
    ];
    const res = await request(makeApp())
      .post('/api/local-llm/compare')
      .send({ mode: 'parallel', targets, prompt: 'Compare this', options: { maxTokens: 64 } });

    expect(res.status).toBe(200);
    expect(res.body.results).toHaveLength(2);
    expect(compareLocalLlmModels).toHaveBeenCalledWith({
      mode: 'parallel',
      targets,
      prompt: 'Compare this',
      options: {
        systemPrompt: '',
        temperature: 0.3,
        maxTokens: 64,
        timeoutMs: 300000,
      },
      signal: expect.any(AbortSignal),
    });
  });

  it('rejects empty prompts before running a model', () => {
    const parsed = localLlmTestSchema.safeParse({ backend: 'ollama', modelId: 'llama3.2', prompt: '   ' });

    expect(parsed.success).toBe(false);
  });

  it('limits comparisons to six targets', () => {
    const targets = Array.from({ length: 7 }, (_, i) => ({ backend: 'ollama', modelId: `model-${i}` }));
    const parsed = localLlmCompareSchema.safeParse({ targets, prompt: 'too many' });

    expect(parsed.success).toBe(false);
  });
});

describe('local LLM memory-management routes', () => {
  beforeEach(() => {
    vi.clearAllMocks();
  });

  it('shares residency samples across socket viewers and fences the unload response', async () => {
    getLoadedModels.mockResolvedValue([{ id: 'example-model' }]);
    getLoadedLmStudioModels.mockResolvedValue([]);
    const first = new EventEmitter();
    const second = new EventEmitter();
    first.connected = second.connected = true;
    registerModelObservationSocket(first);
    registerModelObservationSocket(second);
    first.emit('loaded-models:subscribe');
    second.emit('loaded-models:subscribe');
    try {
      const app = makeApp();
      const [a, b] = await Promise.all([
        request(app).get('/api/local-llm/loaded'),
        request(app).get('/api/local-llm/loaded'),
      ]);
      expect(a.body.ollama).toEqual([{ id: 'example-model' }]);
      expect(b.body).toEqual(a.body);
      expect(getLoadedModels).toHaveBeenCalledTimes(1);
      expect(getLoadedLmStudioModels).toHaveBeenCalledTimes(1);
      unloadModel.mockImplementationOnce(async () => {
        getLoadedModels.mockResolvedValue([]);
        return { unloaded: true, model: 'example-model' };
      });
      await request(app).post('/api/local-llm/unload').send({ backend: 'ollama', modelId: 'example-model' });
      const after = await request(app).get('/api/local-llm/loaded');
      expect(after.body.ollama).toEqual([]);
    } finally {
      first.emit('disconnect');
      second.emit('disconnect');
    }
  });

  it('GET /loaded reports models both local backends currently have resident', async () => {
      // Mirror the real getLoadedModels() field set so the fixture documents the
      // pass-through contract and would catch any future field-stripping.
    const resident = { id: 'llama3.2', name: 'llama3.2', size: 4096, sizeVram: 4096, expiresAt: null };
    const lmStudioResident = { id: 'example/lmstudio', state: 'loaded' };
    getLoadedModels.mockResolvedValue([resident]);
    getLoadedLmStudioModels.mockResolvedValue([lmStudioResident]);

    const res = await request(makeApp()).get('/api/local-llm/loaded');

    expect(res.status).toBe(200);
    expect(res.body).toEqual({ ollama: [resident], lmstudio: [lmStudioResident], sourceErrors: [], disabled: [] });
    expect(getLoadedModels).toHaveBeenCalledTimes(1);
    expect(getLoadedLmStudioModels).toHaveBeenCalledWith(true);
     });

  it('GET /loaded keeps a failed disabled backend in sourceErrors but names it disabled', async () => {
      // "Mark disabled" only silences the availability NAG — it is not evidence the
      // backend holds no memory. So /loaded must still probe a disabled backend AND
      // still surface its failed residency in sourceErrors (the panel's
      // "Free everything" guard keys off that), while separately naming it in
      // `disabled` so the banner can stay quiet about it.
    getSettings.mockResolvedValueOnce({ localLlm: { lmstudio: { disabled: true } } });
    getLmStudioResidencyError.mockReturnValueOnce('LM Studio is unavailable');
    getLoadedLmStudioModels.mockResolvedValue([{ id: 'example/lmstudio', state: 'loaded' }]);

    const res = await request(makeApp()).get('/api/local-llm/loaded');

    expect(res.status).toBe(200);
       // Residency is honored — the backend is still probed…
    expect(getLoadedLmStudioModels).toHaveBeenCalledWith(true);
    expect(res.body.lmstudio).toEqual([{ id: 'example/lmstudio', state: 'loaded' }]);
       // …and its failed probe keeps the "unknown residency" status sourceErrors so
       // "Free everything" can't claim it freed a model it never saw.
    expect(res.body.sourceErrors).toContain('lmstudio');
       // The disabled flag is the signal the panel uses to hold the banner.
    expect(res.body.disabled).toEqual(['lmstudio']);
    expect(getSettings).toHaveBeenCalled();
      });

  it('GET /loaded still reports an enabled backend whose residency probe fails', async () => {
      // An enabled backend that fails its residency probe surfaces in sourceErrors
      // and is NOT in `disabled`, so the panel both shows the nag and keeps its
      // "excluded from Free everything" guard.
    getSettings.mockResolvedValueOnce({});
    getLmStudioResidencyError.mockReturnValueOnce('LM Studio is unavailable');

    const res = await request(makeApp()).get('/api/local-llm/loaded');

    expect(res.status).toBe(200);
    expect(res.body.sourceErrors).toContain('lmstudio');
       // An enabled backend is not in the disabled list.
    expect(res.body.disabled).not.toContain('lmstudio');
    expect(getLoadedLmStudioModels).toHaveBeenCalledWith(true);
      });

  it('POST /unload evicts a resident model and echoes the service result', async () => {
    // Real unloadModel() success shape is { unloaded: true, model } — NOT modelId
    // (ollamaManager.js); the handler spreads it into the response verbatim.
    unloadModel.mockResolvedValue({ unloaded: true, model: 'llama3.2' });

    const res = await request(makeApp())
      .post('/api/local-llm/unload')
      .send({ backend: 'ollama', modelId: 'llama3.2' });

    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true, unloaded: true, model: 'llama3.2' });
    expect(unloadModel).toHaveBeenCalledWith('llama3.2');
  });

  it('POST /unload treats an already-evicted model as an idempotent 200 no-op', async () => {
    unloadModel.mockResolvedValue({ unloaded: false, reason: 'not loaded' });

    const res = await request(makeApp())
      .post('/api/local-llm/unload')
      .send({ backend: 'ollama', modelId: 'llama3.2' });

    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true, unloaded: false, reason: 'not loaded', modelId: 'llama3.2' });
  });

  it('POST /unload surfaces a genuine unload failure as 502', async () => {
    unloadModel.mockResolvedValue({ unloaded: false, reason: 'Ollama unreachable' });

    const res = await request(makeApp())
      .post('/api/local-llm/unload')
      .send({ backend: 'ollama', modelId: 'llama3.2' });

    expect(res.status).toBe(502);
    // Standard error envelope (errorHandler): message in `error`, machine code
    // derived from status, the modelId carried in `context` for diagnostics.
    expect(res.body.error).toBe('Ollama unreachable');
    expect(res.body.code).toBe('BAD_GATEWAY');
    expect(res.body.context).toEqual({ modelId: 'llama3.2' });
    expect(unloadModel).toHaveBeenCalledWith('llama3.2');
  });

  it('POST /unload refuses a non-ollama backend before calling the service', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/unload')
      .send({ backend: 'lmstudio', modelId: 'some-model' });

    expect(res.status).toBe(400);
    expect(res.body.error).toMatch(/use \/api\/lmstudio\/unload/);
    expect(unloadModel).not.toHaveBeenCalled();
  });

  it('POST /unload rejects a flag-like modelId via Zod validation', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/unload')
      .send({ backend: 'ollama', modelId: '-rf' });

    expect(res.status).toBe(400);
    expect(res.body.code).toBe('VALIDATION_ERROR');
    expect(unloadModel).not.toHaveBeenCalled();
  });
});

describe('local LLM capability routes', () => {
  beforeEach(() => { vi.clearAllMocks(); });

  it('GET /tool-use-models returns the service list under `models`', async () => {
    listToolUseModels.mockResolvedValueOnce([
      { providerId: 'ollama', backend: 'ollama', id: 'phi4-mini:latest', name: 'phi4-mini:latest', toolUse: true },
    ]);

    const res = await request(makeApp()).get('/api/local-llm/tool-use-models');

    expect(res.status).toBe(200);
    expect(res.body).toEqual({
      models: [
        { providerId: 'ollama', backend: 'ollama', id: 'phi4-mini:latest', name: 'phi4-mini:latest', toolUse: true },
      ],
    });
    expect(listToolUseModels).toHaveBeenCalledTimes(1);
  });

  it('GET /tool-use-models is a distinct list from /vision-models', async () => {
    // The two endpoints answer different questions; folding tool-use rows into
    // /vision-models would break its "every row is vision-capable" contract,
    // which the LoRA caption picker lists verbatim.
    listVisionModels.mockResolvedValueOnce([
      { providerId: 'ollama', backend: 'ollama', id: 'qwen2.5-vl:7b', name: 'qwen2.5-vl:7b', vision: true },
    ]);
    listToolUseModels.mockResolvedValueOnce([
      { providerId: 'ollama', backend: 'ollama', id: 'phi4-mini:latest', name: 'phi4-mini:latest', toolUse: true },
    ]);
    const app = makeApp();

    const vision = await request(app).get('/api/local-llm/vision-models');
    const tools = await request(app).get('/api/local-llm/tool-use-models');

    expect(vision.body.models.map((m) => m.id)).toEqual(['qwen2.5-vl:7b']);
    expect(tools.body.models.map((m) => m.id)).toEqual(['phi4-mini:latest']);
  });
});

describe('measured assessments wiring', () => {
  beforeEach(() => { vi.clearAllMocks(); });

  it('folds measured evidence into the catalog fit badge', async () => {
    getMeasuredFits.mockResolvedValue({ 'example-model:14b': { fit: 'too-large', verdict: 'does-not-fit', stale: false } });
    const res = await request(makeApp()).get('/api/local-llm/catalog?backend=ollama');

    expect(res.status).toBe(200);
    expect(getMeasuredFits).toHaveBeenCalledWith('ollama');
    // The overlay runs against the SAME backend the listing was built for —
    // an LM Studio measurement must never decorate an Ollama card.
    expect(applyMeasuredFit).toHaveBeenCalledWith(res.body.models, {
      backend: 'ollama',
      measured: { 'example-model:14b': { fit: 'too-large', verdict: 'does-not-fit', stale: false } },
    });
  });

  it('still serves the catalog when the assessments store cannot be read', async () => {
    // A broken measurement file must not take the install picker down with it.
    getMeasuredFits.mockRejectedValue(new Error('disk on fire'));
    const res = await request(makeApp()).get('/api/local-llm/catalog?backend=ollama');
    expect(res.status).toBe(200);
    expect(applyMeasuredFit).toHaveBeenCalledWith(expect.anything(), { backend: 'ollama', measured: {} });
  });

  it('forwards assessment progress to the shared localLlm:progress socket event', async () => {
    runAssessment.mockImplementation(async ({ onProgress }) => {
      onProgress({ scope: 'assessment', backend: 'ollama', modelId: 'example-model:14b', event: 'start', sampleIndex: 1, sampleCount: 3 });
      return { verdict: 'fits' };
    });
    const app = makeApp();
    const res = await request(app)
      .post('/api/local-llm/assessments/run')
      .send({ backend: 'ollama', modelId: 'example-model:14b' });

    expect(res.status).toBe(200);
    expect(app.get('io').emit).toHaveBeenCalledWith('localLlm:progress', {
      scope: 'assessment', backend: 'ollama', modelId: 'example-model:14b', event: 'start', sampleIndex: 1, sampleCount: 3,
    });
  });

  it('runs the explicit OpenCode agent benchmark with a bounded timeout', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/agent-benchmark')
      .send({ backend: 'ollama', modelId: 'qwen3.8:27b-mlx', timeoutMs: 120000 });

    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({
      backend: 'ollama', modelId: 'qwen3.8:27b-mlx', timeoutMs: 120000, completed: true,
    });
    expect(runOpenCodeAgentBenchmark).toHaveBeenCalledWith({
      backend: 'ollama', modelId: 'qwen3.8:27b-mlx', timeoutMs: 120000,
    });
  });

  it('rejects an agent benchmark backend outside the configured local targets', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/agent-benchmark')
      .send({ backend: 'vllm', modelId: 'qwen3.8-27b-dflash2' });

    expect(res.status).toBe(400);
    expect(runOpenCodeAgentBenchmark).not.toHaveBeenCalled();
  });

  // The sweep is the overnight path: it must return as soon as the queue exists,
  // because the run outlives the request that started it.
  it('POST /assessments/sweep starts a queue and returns its snapshot', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/sweep')
      .send({ scope: 'all' });

    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({ status: 'running', total: 3 });
    expect(startSweep).toHaveBeenCalledWith(expect.objectContaining({ scope: 'all' }));
  });

  it('POST /assessments/sweep defaults to the unmeasured scope', async () => {
    const res = await request(makeApp()).post('/api/local-llm/assessments/sweep').send({});
    expect(res.status).toBe(200);
    expect(startSweep).toHaveBeenCalledWith(expect.objectContaining({ scope: 'unmeasured' }));
  });

  it('POST /assessments/sweep rejects an unknown scope rather than silently narrowing it', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/sweep')
      .send({ scope: 'everything-please' });

    expect(res.status).toBe(400);
    expect(startSweep).not.toHaveBeenCalled();
  });

  // A refused start has to be visible: a 200 with no queue behind it would leave
  // the page waiting on progress that never comes.
  it('POST /assessments/sweep 409s when the service refuses to start', async () => {
    startSweep.mockResolvedValueOnce({ status: 'running', rejected: 'a sweep is already running' });
    const res = await request(makeApp()).post('/api/local-llm/assessments/sweep').send({ scope: 'all' });
    expect(res.status).toBe(409);
  });

  it('forwards sweep progress and public queue snapshots to their socket events', async () => {
    startSweep.mockImplementation(async ({ onProgress, onChange }) => {
      onProgress({ scope: 'assessment-sweep', event: 'start', total: 2, completed: 0 });
      onChange({ status: 'running', settled: false, total: 2, completed: 0, results: [], cancelRequested: false });
      return { status: 'running', total: 2, completed: 0 };
    });
    const app = makeApp();
    await request(app).post('/api/local-llm/assessments/sweep').send({ scope: 'unmeasured' });

    expect(app.get('io').emit).toHaveBeenCalledWith('localLlm:progress', {
      scope: 'assessment-sweep', event: 'start', total: 2, completed: 0,
    });
    expect(app.get('io').emit).toHaveBeenCalledWith('localLlm:sweep:changed', {
      status: 'running', settled: false, total: 2, completed: 0, results: [], cancelRequested: false,
    });
  });

  // `tunings` is the ASK, not the grid — the service decides which knob sets a
  // "tuning sweep of this model" means. A client that could post the grid could
  // ask for an arbitrary batch of provider calls.
  it('POST /assessments/sweep forwards a tuning-sweep request without a grid', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/sweep')
      .send({ backend: 'llama', modelId: 'example-model.gguf', tunings: true });

    expect(res.status).toBe(200);
    expect(startSweep).toHaveBeenCalledWith(expect.objectContaining({
      backend: 'llama', modelId: 'example-model.gguf', tunings: true,
    }));
  });

  // The service's refusals ride the existing rejected → 409 path rather than a
  // second, hand-rolled one in the handler.
  it('POST /assessments/sweep 409s when the service refuses the tuning grid', async () => {
    startSweep.mockResolvedValueOnce({ status: 'idle', rejected: 'mtplx has no tuning knobs PortOS can sweep' });
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/sweep')
      .send({ backend: 'mtplx', modelId: 'example-model', tunings: true });

    expect(res.status).toBe(409);
  });

  // Half a model reference would fall through to the scope and measure every
  // installed model — hours of work nobody asked for.
  it('POST /assessments/sweep rejects a backend with no modelId', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/sweep')
      .send({ backend: 'llama', tunings: true });

    expect(res.status).toBe(400);
    expect(startSweep).not.toHaveBeenCalled();
  });

  // No wire knob shrinks the grid: the consent gate names its count from the
  // report's grid, so a request that could shrink it would run a different
  // number than the user agreed to.
  it('POST /assessments/sweep rejects an attempt to size the grid from the wire', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/assessments/sweep')
      .send({ backend: 'llama', modelId: 'example-model.gguf', tunings: true, maxVariants: 2 });

    expect(res.status).toBe(200);
    expect(startSweep.mock.calls[0][0].maxVariants).toBeUndefined();
  });

  it('POST /assessments/sweep leaves a plain model sweep untuned', async () => {
    await request(makeApp()).post('/api/local-llm/assessments/sweep').send({ scope: 'all' });
    expect(startSweep.mock.calls[0][0].tunings).toBe(false);
  });

  it('GET /assessments/sweep reports the queue without touching a provider', async () => {
    const res = await request(makeApp()).get('/api/local-llm/assessments/sweep');
    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({ status: 'idle' });
    expect(getSweepStatus).toHaveBeenCalled();
    expect(startSweep).not.toHaveBeenCalled();
  });

  it('POST /assessments/sweep/cancel stops the queue', async () => {
    const res = await request(makeApp()).post('/api/local-llm/assessments/sweep/cancel');
    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({ status: 'cancelled', completed: 1 });
    expect(cancelSweep).toHaveBeenCalled();
  });
});

describe('llama-server routes', () => {
  it('GET /api/local-llm/llama-server/status returns status with the weight presets', async () => {
    const res = await request(makeApp()).get('/api/local-llm/llama-server/status');
    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({
      installed: true,
      running: false,
      presets: [{ id: 'test-preset', label: 'Test', specType: 'draft-dspark', model: null, draftModel: null }],
    });
  });

  it('GET /api/local-llm/llama-server/status keeps Homebrew probes out of lifecycle status', async () => {
    const { getLlamaServerUpdateStatus } = await import('../services/llamaServerManager.js');
    const res = await request(makeApp()).get('/api/local-llm/llama-server/status');

    expect(res.status).toBe(200);
    expect(getLlamaServerUpdateStatus).not.toHaveBeenCalled();
    expect(res.body).not.toHaveProperty('updateAvailable');
  });

  it('GET /api/local-llm/llama-server/update-status returns llama.cpp update metadata', async () => {
    const { getLlamaServerUpdateStatus } = await import('../services/llamaServerManager.js');
    getLlamaServerUpdateStatus.mockResolvedValueOnce({
      version: '0.1.1-dev',
      latestVersion: '0.3.0',
      updateAvailable: true,
      canUpgrade: true,
      downloadUrl: 'https://github.com/ggml-org/llama.cpp/releases',
    });

    const res = await request(makeApp()).get('/api/local-llm/llama-server/update-status');

    expect(res.body).toMatchObject({
      version: '0.1.1-dev',
      latestVersion: '0.3.0',
      updateAvailable: true,
      canUpgrade: true,
    });
  });

  it('GET /api/local-llm/llama-server/status publishes the drafter-free spec types', async () => {
    // The card renders the picker from this list, so a launch with no drafter
    // GGUF (`ngram-map-k`) has to be reachable without the user knowing the
    // llama.cpp vocabulary by heart.
    const res = await request(makeApp()).get('/api/local-llm/llama-server/status');
    expect(res.body.specTypes.map((entry) => entry.id)).toEqual(
      expect.arrayContaining(['ngram-map-k', 'draft-dflash', 'none']),
    );
  });

  it('POST /api/local-llm/llama-server/download-model fetches one preset GGUF', async () => {
    const { downloadSpecDecodeModel } = await import('../services/specDecodeModels.js');
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/download-model')
      .send({ presetId: 'test-preset', role: 'model' });

    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({ success: true, path: 'models/base.gguf' });
    expect(downloadSpecDecodeModel).toHaveBeenCalledWith(expect.objectContaining({ presetId: 'test-preset', role: 'model' }));
  });

  it('POST /api/local-llm/llama-server/download-model rejects an unknown role', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/download-model')
      .send({ presetId: 'test-preset', role: 'sneaky' });

    expect(res.status).toBe(400);
  });

  it('POST /api/local-llm/download-preflight returns size, dest, and free disk without starting a transfer', async () => {
    const { previewSpecDecodeDownload, downloadSpecDecodeModel } = await import('../services/specDecodeModels.js');
    downloadSpecDecodeModel.mockClear();
    const res = await request(makeApp())
      .post('/api/local-llm/download-preflight')
      .send({ kind: 'spec-decode', presetId: 'test-preset', role: 'model' });

    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({
      kind: 'spec-decode',
      destPath: 'models/base.gguf',
      expectedBytes: 6,
      freeBytes: 1e12,
      verdict: 'ok',
    });
    expect(previewSpecDecodeDownload).toHaveBeenCalledWith({ presetId: 'test-preset', role: 'model' });
    expect(downloadSpecDecodeModel).not.toHaveBeenCalled();
  });

  it('POST /api/local-llm/llama-server/download-model/cancel cancels one active preset download', async () => {
    const { cancelSpecDecodeModelDownload } = await import('../services/specDecodeModels.js');
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/download-model/cancel')
      .send({ presetId: 'test-preset', role: 'model' });

    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true, cancelled: true });
    expect(cancelSpecDecodeModelDownload).toHaveBeenCalledWith({ presetId: 'test-preset', role: 'model' });
  });

  it('POST /api/local-llm/llama-server/download-model/remove deletes one preset GGUF', async () => {
    const { removeSpecDecodeModel } = await import('../services/specDecodeModels.js');
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/download-model/remove')
      .send({ presetId: 'test-preset', role: 'model' });

    expect(res.status).toBe(200);
    expect(res.body).toMatchObject({ success: true, deleted: true, path: 'models/base.gguf' });
    expect(removeSpecDecodeModel).toHaveBeenCalledWith({ presetId: 'test-preset', role: 'model' });
  });

  it('POST /api/local-llm/llama-server/download-model/remove rejects an unknown role', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/download-model/remove')
      .send({ presetId: 'test-preset', role: 'sneaky' });

    expect(res.status).toBe(400);
  });

  it('POST /api/local-llm/llama-server/start launches server with valid payload', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/start')
      .send({ model: 'models/Qwen3.8-27B.gguf', draftModel: 'models/DFlash.gguf' });
    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true, pid: 123 });
  });

  it('POST /api/local-llm/llama-server/start rejects invalid payload', async () => {
    const res = await request(makeApp())
      .post('/api/local-llm/llama-server/start')
      .send({});
    expect(res.status).toBe(400);
  });

  it('POST /api/local-llm/llama-server/stop stops server', async () => {
    const res = await request(makeApp()).post('/api/local-llm/llama-server/stop');
    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true });
  });

  it('POST /api/local-llm/llama-server/install triggers install', async () => {
    const res = await request(makeApp()).post('/api/local-llm/llama-server/install');
    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true, message: 'installed' });
  });

  it('POST /api/local-llm/llama-server/upgrade emits progress and returns the result', async () => {
    const { upgradeLlamaServer } = await import('../services/llamaServerManager.js');
    upgradeLlamaServer.mockResolvedValueOnce({ success: true, note: 'updated and restarted' });
    const app = makeApp();

    const res = await request(app).post('/api/local-llm/llama-server/upgrade');

    expect(res.status).toBe(200);
    expect(res.body).toEqual({ success: true, note: 'updated and restarted' });
    expect(upgradeLlamaServer).toHaveBeenCalledWith({ onProgress: expect.any(Function) });
    expect(app.get('io').emit).toHaveBeenCalledWith(
      'localLlm:progress',
      { event: 'complete', message: 'llama.cpp updated — updated and restarted' },
    );
  });
});


describe('Jev integration policies', () => {
  it('resolves inherited modes and changes only scorer fields, preserving other policies', async () => {
    let persisted = { unrelated: true, untrustedContent: {
      defaults: { jevMode: 'prefer', classifierMode: 'required' },
      sources: { messages: { jevMode: 'only' }, email: { model: 'example-model' },
        'github-issue': { providerId: 'example-provider', jevMinMargin: 0.4 } },
    } };
    readSettingsStrict.mockImplementation(async () => ({ corrupt: false, settings: persisted }));
    updateSettingsWith.mockImplementation(async mutate => (persisted = mutate(persisted)));
    const app = makeApp();
    const before = await request(app).get('/api/local-llm/jev/policy');
    expect(before.body.sources.email.jevMode).toBe('only');
    expect(before.body.sources['github-issue']).toEqual({ jevMode: 'prefer', jevMinMargin: 0.4 });
    const changed = await request(app).put('/api/local-llm/jev/policy').send({
      sources: { email: { jevMode: 'disabled' } }, scopeAdherenceEnabled: false,
    });
    expect(changed.status).toBe(200);
    expect(changed.body.scopeAdherenceEnabled).toBe(false);
    expect(changed.body.sources.email.jevMode).toBe('disabled');
    expect(persisted.untrustedContent.sources.email.model).toBe('example-model');
    expect(persisted.untrustedContent.sources['github-issue'].providerId).toBe('example-provider');
    expect(persisted.unrelated).toBe(true);
    const invalid = await request(app).put('/api/local-llm/jev/policy').send({ sources: { email: { classifierMode: 'optional' } } });
    expect(invalid.status).toBe(400);
  });
});