Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

35.9 KB
Raw Blame
/**
 * Local-model assessment scoring — pure, dependency-free.
 *
 * PortOS already guesses whether a local model fits from catalog metadata alone
 * (weight bytes × a fixed overhead multiplier, in `huggingFaceCatalog.js`). That
 * guess never touches the machine: it can't tell a model that loads and streams
 * fast from one that loads, thrashes, and crawls, and it says
 * nothing at all about how a model behaves as the prompt gets long.
 *
 * This module turns *measured* evidence — one bounded generation per context
 * length, recorded by `services/localModelAssessments.js` — into an explainable
 * verdict and an intent-specific ranking.
 *
 * ## The sentinel contract (read before touching anything here)
 *
 * `null` means NOT MEASURED. It never means zero, never means failed, and never
 * collapses into a fallback. `[]` / `0` mean measured-and-empty. Concretely:
 *
 *   - `sample.charsPerSecond === null`  → the sample never produced timings
 *   - `sample.ok === false`              → the sample RAN and FAILED (has `error`)
 *   - `verdict === 'unknown'`            → nothing usable was measured
 *   - `scores.speed === null`            → speed was never observed; a ranking
 *                                          must not treat that as "slow"
 *
 * Anything that conflates those makes an unassessed model look worse than a
 * measured-bad one, which is the exact failure this feature exists to remove.
 *
 * (Dependency-free apart from `localModelHeuristics.js`, which is itself pure —
 * a sweep has to know which installed models are even *assessable*.)
 */

import { isEmbeddingModel } from './localModelHeuristics.js';

/**
 * Fit verdicts. Deliberately four values, not a boolean:
 *   - `fits`         — at least one context length generated successfully
 *   - `does-not-fit` — every attempted context length failed for a resource
 *                      reason (OOM / load failure / timeout)
 *   - `incompatible` — the backend refused the model outright (wrong format,
 *                      unsupported architecture) — retrying with less context
 *                      or more RAM will not help
 *   - `unknown`      — nothing was measured
 */
export const FIT_VERDICTS = ['fits', 'does-not-fit', 'incompatible', 'unknown'];

/** Ranking intents, mirroring how people actually pick a local model. */
export const ASSESSMENT_INTENTS = ['balanced', 'smartest', 'fastest', 'lightweight'];

// Error text that means "this model can never run here", as opposed to "this
// model ran out of room at this context length". Matched case-insensitively
// against the sample error. Kept narrow on purpose: a false `incompatible`
// permanently hides a model that a smaller context would have run.
const INCOMPATIBLE_RE =
  /unsupported|unknown model architecture|unrecognized|not supported|invalid model file|no such file|unsupported model format|does not support/i;

// Error text that means "resource exhaustion at this size" — the model itself is
// fine, this machine could not hold it as configured.
const RESOURCE_RE =
  /out of memory|oom|cannot allocate|insufficient memory|not enough memory|resource exhausted|model requires more system memory|timed out|context (?:length|window) exceed/i;

/**
 * Classify one recorded sample's failure. Returns `null` for a sample that
 * succeeded or that carries no error text (unknown cause — never guessed at).
 *
 * @param {{ ok?: boolean, error?: string }} sample
 * @returns {'incompatible'|'resource'|null}
 */
export function classifySampleFailure(sample) {
  if (!sample || sample.ok !== false) return null;
  const error = typeof sample.error === 'string' ? sample.error : '';
  if (!error) return null;
  if (INCOMPATIBLE_RE.test(error)) return 'incompatible';
  if (RESOURCE_RE.test(error)) return 'resource';
  return null;
}

const isMeasured = (value) => typeof value === 'number' && Number.isFinite(value);

/** Samples that actually produced output. */
const okSamples = (samples) => (Array.isArray(samples) ? samples : []).filter((s) => s?.ok === true);

/**
 * Derive the fit verdict from the recorded samples.
 *
 * One success anywhere is enough for `fits` — a model that runs at 4k but dies
 * at 32k still fits this machine, and `maxWorkingContextTokens` (below) is what
 * carries the nuance. `incompatible` wins over `does-not-fit` because it is the
 * actionable one: no amount of freed memory changes it.
 *
 * @param {Array<{ ok?: boolean, error?: string }>} samples
 * @returns {'fits'|'does-not-fit'|'incompatible'|'unknown'}
 */
export function classifyFitVerdict(samples) {
  const list = Array.isArray(samples) ? samples : [];
  if (list.length === 0) return 'unknown';
  if (list.some((s) => s?.ok === true)) return 'fits';
  const failures = list.map(classifySampleFailure);
  if (failures.some((f) => f === 'incompatible')) return 'incompatible';
  if (failures.some((f) => f === 'resource')) return 'does-not-fit';
  // Ran, failed, and we can't say why (network blip, backend restart). That is
  // NOT evidence the model doesn't fit — leave it unknown so a re-run decides.
  return 'unknown';
}

/**
 * Largest context length that produced a successful generation.
 * `null` when nothing succeeded — distinct from `0`, which never occurs here.
 *
 * @param {Array<{ ok?: boolean, contextTokens?: number }>} samples
 * @returns {number|null}
 */
export function maxWorkingContextTokens(samples) {
  const working = okSamples(samples)
    .map((s) => s.contextTokens)
    .filter(isMeasured);
  return working.length ? Math.max(...working) : null;
}

/**
 * Mean of a numeric field across successful samples, or `null` when the field
 * was never measured. Rounded to two decimals so persisted records stay stable.
 */
function meanOf(samples, field) {
  const values = okSamples(samples).map((s) => s[field]).filter(isMeasured);
  if (!values.length) return null;
  const mean = values.reduce((sum, v) => sum + v, 0) / values.length;
  return Number(mean.toFixed(2));
}

/**
 * Collapse the per-context samples into one performance summary.
 *
 * `contextDegradation` is the ratio of the throughput at the LARGEST working
 * context to the peak throughput across all contexts (0..1]; 1 means the longest
 * context ran as fast as the fastest one did, 0.25 means it ran at a quarter of
 * that.
 *
 * It is deliberately anchored to the longest context rather than to the slowest
 * sample. Results are not always monotonic — a mid-range context can dip below
 * both its neighbours from an unrelated hiccup — and a min-based ratio would
 * both contradict what this number is reported as ("held X% of peak throughput
 * at its longest context") and penalize the ranking for a one-off intermediate
 * outlier instead of for the long-context behavior the axis is about.
 *
 * `null` when fewer than two context lengths produced throughput: a single
 * sample cannot describe a trend, and reporting `1` there would fake perfect
 * scaling.
 *
 * @param {Array<object>} samples
 */
export function summarizePerformance(samples) {
  const ok = okSamples(samples);
  const measured = ok
    .filter((s) => isMeasured(s.charsPerSecond) && s.charsPerSecond > 0 && isMeasured(s.contextTokens));
  const fastest = measured.length ? Math.max(...measured.map((s) => s.charsPerSecond)) : null;
  const atLongest = measured.length
    ? measured.reduce((longest, s) => (s.contextTokens > longest.contextTokens ? s : longest)).charsPerSecond
    : null;
  const tokenRates = ok.map((s) => s.tokensPerSecond).filter(isMeasured);
  return {
    samplesRun: Array.isArray(samples) ? samples.length : 0,
    samplesOk: ok.length,
    meanCharsPerSecond: meanOf(samples, 'charsPerSecond'),
    meanTtftMs: meanOf(samples, 'ttftMs'),
    peakCharsPerSecond: fastest,
    // Token-denominated throughput, present only when the daemon reported token
    // counts. `null` here means the runtime told us nothing — a reader must show
    // "not measured", not fall back to chars/s under a tokens/s label.
    meanTokensPerSecond: meanOf(samples, 'tokensPerSecond'),
    peakTokensPerSecond: tokenRates.length ? Math.max(...tokenRates) : null,
    meanPromptTokensPerSecond: meanOf(samples, 'promptTokensPerSecond'),
    timingSource: (() => {
      const sources = [...new Set(ok.map((s) => s.timingSource).filter(Boolean))];
      return sources.length === 1 ? sources[0] : (sources.length ? 'mixed' : null);
    })(),
    // `true` when ANY contributing sample counted streamed frames instead of
    // reading the daemon's own count. One estimated sample makes the mean an
    // estimate, so this is deliberately pessimistic. `null` = no token evidence.
    tokensEstimated: tokenRates.length
      ? ok.some((s) => isMeasured(s.tokensPerSecond) && s.tokensEstimated === true)
      : null,
    maxWorkingContextTokens: maxWorkingContextTokens(samples),
    // Clamped: the longest context can measure marginally ABOVE the peak of the
    // others only through noise, and a >1 "degradation" is meaningless.
    contextDegradation:
      measured.length >= 2 && fastest > 0 ? Number(Math.min(1, atLongest / fastest).toFixed(3)) : null,
  };
}

/**
 * Flatten every measurement into one tokens-per-second table, fastest first.
 *
 * This is the "how fast is each model, really" view: one row per measured
 * model+tuning, carrying the per-context readings and elapsed test time so
 * falloff as the prompt grows is visible rather than averaged away. It is a
 * REPORT, not a ranking — no scoring, no intent, no exclusions — so a model that
 * only ran at 512 tokens still appears, with one column filled and the rest
 * honestly blank.
 *
 * Rows whose runtime reported no token counts are still included (with `null`
 * rates) rather than dropped: "this runtime does not report tokens" is the
 * answer to why a model is missing from a tokens/s table, and silently omitting
 * it would make the table look complete when it isn't.
 *
 * @param {Array<object>} assessments stored assessment records
 * @returns {{ rows: Array<object>, contexts: number[], modelsWithTokenRates: number }}
 */
export function buildThroughputReport(assessments) {
  const list = Array.isArray(assessments) ? assessments : [];
  // Every rate on a row is null-or-measured; nothing is defaulted to 0.
  const num = (value) => (isMeasured(value) ? value : null);
  const contexts = new Set();
  const rows = [];

  for (const assessment of list) {
    const samples = Array.isArray(assessment?.samples) ? assessment.samples : [];
    const perf = assessment?.performance || {};
    const points = [];
    for (const sample of samples) {
      if (!isMeasured(sample?.contextTokens)) continue;
      contexts.add(sample.contextTokens);
      points.push({
        contextTokens: sample.contextTokens,
        ok: sample?.ok === true,
        // A failed sample carries its error so the cell can say WHY it is blank
        // instead of just being blank.
        tokensPerSecond: num(sample.tokensPerSecond),
        promptTokensPerSecond: num(sample.promptTokensPerSecond),
        charsPerSecond: num(sample.charsPerSecond),
        ttftMs: num(sample.ttftMs),
        // Keep the end-to-end duration with the context column. Unlike the
        // aggregate mean TTFT, this answers how long THIS individual test took,
        // including prompt processing and generation.
        totalMs: num(sample.totalMs),
        completionTokens: num(sample.completionTokens),
        timingSource: sample.timingSource || null,
        error: sample?.error || null,
      });
    }
    rows.push({
      backend: assessment?.backend || null,
      modelId: assessment?.modelId || null,
      tuningKey: assessment?.tuningKey || '',
      tuningLabel: assessment?.tuningLabel || null,
      // A reading taken under a tuning that never reached the daemon describes
      // some OTHER configuration. It stays in the table (the numbers are real)
      // but has to carry the caveat, or the row credits the wrong config.
      tuningApplied: assessment?.tuningApplied ?? null,
      tuningNotApplied: assessment?.tuningNotApplied || null,
      verdict: assessment?.verdict || 'unknown',
      assessedAt: assessment?.assessedAt || null,
      staleness: assessment?.staleness || null,
      meanTokensPerSecond: num(perf.meanTokensPerSecond),
      peakTokensPerSecond: num(perf.peakTokensPerSecond),
      meanPromptTokensPerSecond: num(perf.meanPromptTokensPerSecond),
      meanCharsPerSecond: num(perf.meanCharsPerSecond),
      meanTtftMs: num(perf.meanTtftMs),
      timingSource: perf.timingSource || null,
      tokensEstimated: typeof perf.tokensEstimated === 'boolean' ? perf.tokensEstimated : null,
      points: points.sort((a, b) => a.contextTokens - b.contextTokens),
    });
  }

  // Fastest first, but a row with NO token rate sorts last rather than as zero —
  // unmeasured is not slow. Ties fall back to model id so the order is stable.
  rows.sort((a, b) => {
    const left = a.meanTokensPerSecond;
    const right = b.meanTokensPerSecond;
    if (left === null && right === null) return String(a.modelId).localeCompare(String(b.modelId));
    if (left === null) return 1;
    if (right === null) return -1;
    return right - left || String(a.modelId).localeCompare(String(b.modelId));
  });

  return {
    rows,
    contexts: [...contexts].sort((a, b) => a - b),
    // How many rows actually carry a tokens/s figure, so the UI can explain an
    // empty table ("no local runtime here reports token counts") instead of
    // rendering a blank grid.
    modelsWithTokenRates: rows.filter((r) => r.meanTokensPerSecond !== null).length,
  };
}

// Speed normalization anchor, in CHARACTERS per second.
//
// Chars/s, not tokens/s, and deliberately so. PortOS now RECORDS tokens/s when
// the daemon reports token counts (`summarizeTimings`), and reports it — but it
// cannot SCORE on it: a runtime that reports no usage block would score `null`
// on speed and drop out of the ranking entirely, and records written before
// token counts existed carry none. Chars/s is the one figure every runtime and
// every stored record has, so the ranking stays comparable across both.
// Still no tokenizer in the repo: nothing here ever derives a token count from
// a character count.
//
// 240 chars/s (~60 tok/s at the conventional ~4 chars/token) is a comfortable
// interactive rate on consumer hardware; at or above it the axis scores 1, below
// it the score is linear so a crawling model reads as clearly worse rather than
// both bottoming out.
const SPEED_CEILING_CHARS_PER_SECOND = 240;

// Parameter-count anchor for the capability score. Local models above ~70B are
// rare on consumer hardware and already score at the top of the curve.
const PARAMS_CEILING_B = 70;

const clamp01 = (v) => Math.min(1, Math.max(0, v));

/**
 * Parse a parameter count in billions out of a params string or model id
 * (`"32B"`, `"qwen3:14b"`). `null` when the id carries no size marker — never
 * defaulted, so an unsized model doesn't masquerade as a small one.
 *
 * @param {{ params?: string, modelId?: string }} model
 * @returns {number|null}
 */
export function parseParamsBillions({ params, modelId } = {}) {
  const match = `${params || ''} ${modelId || ''}`.match(/(\d+(?:\.\d+)?)\s*[bB]\b/);
  return match ? Number(match[1]) : null;
}

/**
 * Score one assessment on the four axes the ranking uses. Every axis is `null`
 * when its evidence is missing — callers must handle that rather than
 * substituting 0.
 *
 * - `capability` — parameter count, the only capability proxy available offline
 * - `speed`      — measured mean throughput against the interactive anchor
 * - `fidelity`   — how well throughput held up as context grew (needs ≥2 samples)
 * - `memory`     — measured headroom left after the model's resident footprint
 *
 * @param {object} assessment
 * @returns {{capability:number|null, speed:number|null, fidelity:number|null, memory:number|null}}
 */
export function scoreAssessment(assessment) {
  const perf = assessment?.performance || {};
  const paramsB = parseParamsBillions({ params: assessment?.params, modelId: assessment?.modelId });
  const residentGb = assessment?.residentGb;
  const budgetGb = assessment?.environment?.memoryBudgetGb;

  const memory =
    isMeasured(residentGb) && isMeasured(budgetGb) && budgetGb > 0
      ? clamp01(1 - residentGb / budgetGb)
      : null;

  return {
    capability: paramsB == null ? null : clamp01(Math.log10(1 + paramsB) / Math.log10(1 + PARAMS_CEILING_B)),
    speed: isMeasured(perf.meanCharsPerSecond) ? clamp01(perf.meanCharsPerSecond / SPEED_CEILING_CHARS_PER_SECOND) : null,
    fidelity: isMeasured(perf.contextDegradation) ? clamp01(perf.contextDegradation) : null,
    memory,
  };
}

// Per-intent axis weights. Weights of axes with no measurement are dropped and
// the rest renormalized, so a partly-measured model is scored on what IS known
// instead of being penalized for the gap. `coverage` reports how much of the
// intended weight was actually backed by evidence, and the caller surfaces it.
const INTENT_WEIGHTS = {
  balanced: { capability: 0.3, speed: 0.3, fidelity: 0.2, memory: 0.2 },
  smartest: { capability: 0.6, speed: 0.1, fidelity: 0.2, memory: 0.1 },
  fastest: { capability: 0.1, speed: 0.6, fidelity: 0.2, memory: 0.1 },
  lightweight: { capability: 0.1, speed: 0.25, fidelity: 0.15, memory: 0.5 },
};

/**
 * Weighted score for one intent.
 *
 * @param {object} scores from `scoreAssessment`
 * @param {string} intent one of ASSESSMENT_INTENTS
 * @returns {{ score: number|null, coverage: number }} `score` is null when NO
 *   axis was measured — an unassessed model must not be rankable as "0".
 */
export function scoreForIntent(scores, intent) {
  const weights = INTENT_WEIGHTS[intent] || INTENT_WEIGHTS.balanced;
  let weighted = 0;
  let used = 0;
  for (const [axis, weight] of Object.entries(weights)) {
    const value = scores?.[axis];
    if (!isMeasured(value)) continue;
    weighted += value * weight;
    used += weight;
  }
  if (used === 0) return { score: null, coverage: 0 };
  return { score: Number((weighted / used).toFixed(4)), coverage: Number(used.toFixed(2)) };
}

const INTENT_LABEL = {
  balanced: 'balanced',
  smartest: 'strongest',
  fastest: 'fastest',
  lightweight: 'lightest',
};

/**
 * One-sentence, evidence-only explanation for a ranked candidate. Every clause
 * names a measured number; unmeasured axes are simply omitted rather than
 * described with a guess.
 */
export function explainAssessment(assessment, intent) {
  const perf = assessment?.performance || {};
  const parts = [];
  // Tokens/s leads when the runtime reported token counts — it is the figure
  // people actually compare local models on. Chars/s is the fallback, never both:
  // two throughput numbers in one sentence read as a contradiction.
  if (isMeasured(perf.meanTokensPerSecond)) {
    parts.push(`${perf.meanTokensPerSecond} tok/s${perf.tokensEstimated ? ' (estimated)' : ''} measured`);
  } else if (isMeasured(perf.meanCharsPerSecond)) {
    parts.push(`${perf.meanCharsPerSecond} chars/s measured`);
  }
  if (isMeasured(perf.maxWorkingContextTokens)) {
    parts.push(`ran at up to ${perf.maxWorkingContextTokens.toLocaleString('en-US')} tokens of context`);
  }
  if (isMeasured(perf.contextDegradation)) {
    parts.push(`held ${Math.round(perf.contextDegradation * 100)}% of peak throughput at its longest context`);
  }
  if (isMeasured(assessment?.residentGb)) parts.push(`${assessment.residentGb} GB resident`);
  if (!parts.length) return `No measurements recorded yet — run an assessment to rank it for ${INTENT_LABEL[intent] || intent} use.`;
  return `${parts.join(', ')}.`;
}

/**
 * Rank assessed models for one intent.
 *
 * Only `fits` candidates are ranked — a model that did not run is not a
 * recommendation, however good its metadata looks. Everything else is returned
 * under `excluded` with its verdict so the UI can explain the absence instead
 * of silently dropping it.
 *
 * @param {Array<object>} assessments
 * @param {string} intent
 * @returns {{ intent: string, ranked: Array<object>, excluded: Array<object> }}
 */
export function rankByIntent(assessments, intent = 'balanced') {
  const resolvedIntent = ASSESSMENT_INTENTS.includes(intent) ? intent : 'balanced';
  const list = Array.isArray(assessments) ? assessments : [];

  const ranked = [];
  const excluded = [];
  for (const assessment of list) {
    const verdict = assessment?.verdict || 'unknown';
    if (verdict !== 'fits') {
      excluded.push({
        backend: assessment?.backend || null,
        modelId: assessment?.modelId || null,
        tuningKey: assessment?.tuningKey || '',
        tuningLabel: assessment?.tuningLabel || null,
        verdict,
        reason: assessment?.verdictReason || null,
      });
      continue;
    }
    const scores = scoreAssessment(assessment);
    const { score, coverage } = scoreForIntent(scores, resolvedIntent);
    if (score == null) {
      excluded.push({
        backend: assessment?.backend || null,
        modelId: assessment?.modelId || null,
        tuningKey: assessment?.tuningKey || '',
        tuningLabel: assessment?.tuningLabel || null,
        verdict,
        reason: 'ran, but no axis of this intent was measured',
      });
      continue;
    }
    ranked.push({
      backend: assessment?.backend || null,
      modelId: assessment?.modelId || null,
      verdict,
      score,
      coverage,
      scores,
      performance: assessment?.performance || null,
      // The measured footprint feeds the memory axis, so it has to travel WITH
      // the ranked entry — a consumer that renders it from the entry alone would
      // otherwise show "not measured" for every model /api/ps actually measured.
      residentGb: Number.isFinite(assessment?.residentGb) ? assessment.residentGb : null,
      params: assessment?.params ?? null,
      assessedAt: assessment?.assessedAt || null,
      // Travels with the entry so the UI can mark a reading taken before a RAM
      // upgrade / backend update WITHOUT re-deriving the comparison client-side.
      // `null` = the caller did not annotate staleness, which is not the same as
      // "compared and current".
      staleness: assessment?.staleness || null,
      // The tuning IS part of this row's identity, not decoration. A model can
      // hold several measurements, one per launch configuration, and the
      // consumer uses these to key the row, to delete THIS measurement rather
      // than the backend-defaults one, and to pre-fill a re-measure with the
      // configuration that produced it. Dropping them collapsed every variant
      // onto the default record.
      tuningKey: assessment?.tuningKey || '',
      tuning: assessment?.tuning || {},
      tuningLabel: assessment?.tuningLabel || null,
      // `null` = nothing was settable (see runAssessment); `false` means these
      // numbers describe some OTHER configuration, which the row has to say.
      tuningApplied: assessment?.tuningApplied ?? null,
      tuningNotApplied: assessment?.tuningNotApplied || null,
      explanation: explainAssessment(assessment, resolvedIntent),
    });
  }

  // FRESH evidence outranks stale evidence outright, before score is even
  // considered: a reading taken before a RAM upgrade or a backend update
  // describes a machine that no longer exists, so it must never be the top
  // recommendation — the same rule `reconcileFit` and the editorial recommender
  // apply. Stale rows stay in the list (labelled) rather than vanishing, because
  // this panel is where the user goes to see and re-run them.
  // Then: higher score first; ties broken by evidence coverage so a
  // fully-measured model outranks one scored on a single axis, then by model id
  // for stability.
  const isStale = (entry) => (entry.staleness?.stale ? 1 : 0);
  ranked.sort((a, b) =>
    (isStale(a) - isStale(b))
    || (b.score - a.score)
    || (b.coverage - a.coverage)
    || String(a.modelId).localeCompare(String(b.modelId))
    // Two tunings of ONE model tie on model id, so the signature is what makes
    // their order stable across reloads.
    || String(a.tuningKey).localeCompare(String(b.tuningKey)));
  return { intent: resolvedIntent, ranked, excluded };
}

// ---- batch measurement ------------------------------------------------------

/** What a "measure everything" run can cover. */
export const SWEEP_SCOPES = ['unmeasured', 'stale', 'all'];

const targetKey = (t) => `${t.backend}:${t.modelId}@${t.tuningKey || ''}`;

/**
 * Expand a sweep scope into the exact list of measurements to run.
 *
 * Pure and deterministic so the consent modal can be told the real count before
 * anything runs, and so the sweep and the count can never disagree about what
 * "all" means — they call this same function.
 *
 * A re-measure reuses the tuning that produced the existing record, matching
 * what the single-model Measure-again button does: "run it again" has to
 * reproduce the same configuration, or the sweep would quietly replace every
 * tuned reading with a backend-defaults one.
 *
 * ## The second dimension
 *
 * `tunings` swaps that rule out: instead of each model keeping the tuning it
 * carries, every model is crossed with the supplied grid — one measurement per
 * (model, tuning) pair. That is what makes a TUNING sweep a case of this
 * function rather than a queue of its own, and it is why the dedupe below is
 * keyed on model AND tuning: two variants sharing a key are one measurement, and
 * running it twice would put the same configuration on both sides of the
 * comparison table.
 *
 * @param {object} options
 * @param {Array<object>} options.assessments stored records, already filtered to
 *   models that are STILL INSTALLED — a sweep must not queue a model the runtime
 *   can no longer load.
 * @param {Array<{backend:string, modelId:string}>} options.unassessed models with
 *   no evidence at all
 * @param {'unmeasured'|'stale'|'all'} options.scope
 * @param {{backend:string, modelId:string}|null} [options.only] restrict to one
 *   model. Naming a model leaves the scope nothing to decide, so it implies
 *   `all` — otherwise "sweep THIS model" would silently do nothing whenever the
 *   model happened to fall outside the scope.
 * @param {Array<{tuning:object, key:string, label:string|null}>|null} [options.tunings]
 *   knob sets to measure every selected model under. Entries are taken as
 *   already-normalized (they come from `localModelTuning.js#tuningGridFor`), so
 *   this module stays dependency-free.
 * @returns {Array<{backend:string, modelId:string, tuning:object, tuningKey:string, tuningLabel:string|null, reason:string}>}
 */
export function selectSweepTargets({ assessments = [], unassessed = [], scope = 'unmeasured', only = null, tunings = null } = {}) {
  const resolved = only ? 'all' : (SWEEP_SCOPES.includes(scope) ? scope : 'unmeasured');
  const wanted = (m) => !only || (m?.backend === only.backend && m?.modelId === only.modelId);
  const stored = Array.isArray(assessments) ? assessments : [];
  const fresh = Array.isArray(unassessed) ? unassessed : [];
  const targets = [];
  const seen = new Set();

  const add = (target) => {
    const key = targetKey(target);
    if (!target.backend || !target.modelId || seen.has(key)) return;
    // An assessment measures a model by GENERATING with it, so an embedding-only
    // model has no measurement to take: the daemon answers every sample with
    // `400 "<model>" does not support chat`, the record lands as `incompatible`
    // (which reads as "this machine can't run it" — it can, that is just not
    // what it does), and each doomed sample raises an AI-provider investigation
    // task. Excluded from BOTH branches, so a bogus record written before this
    // existed is not re-queued forever by the `stale`/`all` scopes.
    if (isEmbeddingModel(target.modelId)) return;
    seen.add(key);
    targets.push(target);
  };

  // Never-measured models come first in every scope that includes them: they are
  // the ones with no answer at all, so if an overnight run is cut short they are
  // the measurements worth having got.
  if (resolved === 'unmeasured' || resolved === 'all') {
    for (const model of fresh) {
      if (!wanted(model)) continue;
      add({ backend: model?.backend, modelId: model?.modelId, tuning: {}, tuningKey: '', tuningLabel: null, reason: 'never measured' });
    }
  }

  if (resolved === 'stale' || resolved === 'all') {
    for (const assessment of stored) {
      if (resolved === 'stale' && !assessment?.staleness?.stale) continue;
      if (!wanted(assessment)) continue;
      add({
        backend: assessment?.backend,
        modelId: assessment?.modelId,
        tuning: assessment?.tuning && typeof assessment.tuning === 'object' ? assessment.tuning : {},
        tuningKey: assessment?.tuningKey || '',
        tuningLabel: assessment?.tuningLabel || null,
        reason: assessment?.staleness?.stale ? 'measured on a different machine state' : 're-measure',
      });
    }
  }

  if (!Array.isArray(tunings) || tunings.length === 0) return targets;

  // Cross the selected models with the grid. Re-running `add` rather than
  // pushing directly keeps ONE dedupe rule for both dimensions — a model that
  // reached this point under two stored tunings collapses back to one entry per
  // grid variant instead of measuring each variant twice.
  const models = targets.splice(0, targets.length);
  seen.clear();
  for (const model of models) {
    for (const variant of tunings) {
      add({
        backend: model.backend,
        modelId: model.modelId,
        tuning: variant?.tuning && typeof variant.tuning === 'object' ? variant.tuning : {},
        tuningKey: variant?.key || '',
        tuningLabel: variant?.label ?? null,
        // `''` is the baseline — the reading every other variant's throughput is
        // reported relative to, so it is named rather than left blank.
        reason: variant?.key ? 'tuning variant' : 'backend defaults',
      });
    }
  }

  return targets;
}

/**
 * How many measurements each scope would run, for the consent gate.
 * Derived from `selectSweepTargets` so a displayed count is always the count
 * that would actually execute.
 */
export function summarizeSweepScopes({ assessments, unassessed } = {}) {
  return Object.fromEntries(
    SWEEP_SCOPES.map((scope) => [scope, selectSweepTargets({ assessments, unassessed, scope }).length])
  );
}

// ---- staleness ---------------------------------------------------------------

/**
 * Machine facts a measurement is only valid FOR. These are the durable ones: a
 * RAM upgrade, a CPU swap, an OS/arch move, or a backend update all invalidate a
 * recorded reading, and none of them announce themselves to the user.
 *
 * Transient state is deliberately NOT here. `availableMemoryGb` swings by
 * gigabytes minute to minute on any working machine, so comparing it would flag
 * essentially every assessment as stale and the badge would mean nothing.
 */
const ENVIRONMENT_FIELDS = [
  { field: 'platform', label: 'operating system' },
  { field: 'arch', label: 'CPU architecture' },
  { field: 'cpuCount', label: 'CPU count' },
  { field: 'totalMemoryGb', label: 'installed memory', unit: ' GB', toleranceGb: 0.5 },
  { field: 'backendVersion', label: 'backend version' },
];

// Absent on EITHER side is not-comparable, never a change: an older record
// predates a captured field (`backendVersion` landed after the first
// assessments), and LM Studio reports no version at all. Reading either as
// "changed" would mark every legacy record stale on sight.
const present = (v) => v !== null && v !== undefined && v !== '';

/**
 * Compare a recorded assessment environment against the live one.
 *
 * @param {object|null} recorded environment stored with the assessment
 * @param {object|null} live environment captured now
 * @returns {{ comparable: boolean, stale: boolean, changes: Array<{field:string,label:string,from:*,to:*}> }}
 *   `comparable: false` means no field could be compared — that is UNKNOWN, not
 *   fresh, and callers must not render it as a clean bill of health.
 */
export function compareEnvironments(recorded, live) {
  const changes = [];
  let compared = 0;
  for (const { field, label, toleranceGb } of ENVIRONMENT_FIELDS) {
    const from = recorded?.[field];
    const to = live?.[field];
    if (!present(from) || !present(to)) continue;
    compared += 1;
    const changed = typeof toleranceGb === 'number' && typeof from === 'number' && typeof to === 'number'
      ? Math.abs(from - to) > toleranceGb
      : from !== to;
    if (changed) changes.push({ field, label, from, to });
  }
  return { comparable: compared > 0, stale: changes.length > 0, changes };
}

/**
 * One human sentence naming what changed since the measurement, or `null` when
 * nothing did (or nothing could be compared).
 */
export function describeStaleness(staleness) {
  const changes = staleness?.changes;
  if (!Array.isArray(changes) || changes.length === 0) return null;
  const parts = changes.map(({ label, from, to }) => `${label} ${from} → ${to}`);
  return `Measured on a different machine state — ${parts.join('; ')}.`;
}

// ---- folding measured evidence into the estimated fit badge ------------------

// The catalog's estimate calls a model "tight" once its projected resident size
// passes 60% of the usable budget (`huggingFaceCatalog.js#classifyFit`). The
// measured verdict uses the SAME threshold against the measured footprint so the
// two vocabularies stay comparable — a badge that flips from `comfortable` to
// `tight` must mean the same thing whichever source produced it.
const TIGHT_FRACTION = 0.6;

/**
 * Translate a stored assessment into the catalog's fit vocabulary.
 *
 * `null` means the measurement says nothing about fit (`unknown` verdict, or no
 * assessment at all) — it must NOT collapse into `'unknown'`, which is a real
 * badge value the estimate also produces.
 *
 * @param {object|null} assessment
 * @returns {'comfortable'|'tight'|'too-large'|'incompatible'|null}
 */
export function measuredFitVerdict(assessment) {
  switch (assessment?.verdict) {
    case 'fits': {
      const resident = assessment?.residentGb;
      const budget = assessment?.environment?.memoryBudgetGb;
      // Footprint unknown (LM Studio reports none) — it RAN, so `comfortable`
      // is the honest read; we just can't say how close to the edge it was.
      if (!Number.isFinite(resident) || !Number.isFinite(budget) || budget <= 0) return 'comfortable';
      return resident > budget * TIGHT_FRACTION ? 'tight' : 'comfortable';
    }
    case 'does-not-fit': return 'too-large';
    case 'incompatible': return 'incompatible';
    default: return null;
  }
}

/**
 * Reconcile the size ESTIMATE with the MEASUREMENT for one model.
 *
 * Rules, in order:
 *   1. No usable measurement → the estimate stands, unchanged.
 *   2. A STALE measurement never overrides the estimate. It describes a machine
 *      that no longer exists (RAM changed, backend updated), so trusting it over
 *      a fresh estimate would be worse than the estimate alone — but it is still
 *      reported, because "we measured this before your upgrade" is information.
 *   3. Otherwise the measurement wins. It ran the model; the estimate multiplied
 *      a file size by 1.2.
 *
 * `disagrees` is the point of the whole exercise: it marks the models where
 * running the thing contradicted the arithmetic, which is the most useful thing
 * this data can say.
 *
 * @param {string|null} estimatedFit the catalog's `classifyFit` result
 * @param {object|null} measurement compact record from the assessments store:
 *   `{ fit, verdict, assessedAt, stale, staleReason }`
 */
export function reconcileFit(estimatedFit, measurement) {
  const estimated = estimatedFit || null;
  const measuredFit = measurement?.fit || null;
  const base = {
    fit: estimated,
    fitSource: 'estimated',
    estimatedFit: estimated,
    measuredFit,
    verdict: measurement?.verdict || null,
    assessedAt: measurement?.assessedAt || null,
    // `disagrees` is only meaningful when BOTH sides have an opinion; an
    // unmeasured or unestimated model disagrees with nothing.
    disagrees: Boolean(measuredFit && estimated && estimated !== 'unknown' && measuredFit !== estimated),
    stale: Boolean(measurement?.stale),
    staleReason: measurement?.staleReason || null,
  };
  if (!measuredFit || measurement?.stale) return base;
  return { ...base, fit: measuredFit, fitSource: 'measured' };
}