Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

20.7 KB
Raw Blame
/**
 * Capability test catalog and scoring — pure.
 *
 * The Performance page answers "how FAST is this model here". This module backs
 * the other half: "can it actually DO the thing its badges claim". Each test is
 * bound to the capability badges the install catalog already shows
 * (`chat`/`code`/`reasoning`/`vision`/`tools`), so a test runs exactly where the
 * model made a claim worth checking.
 *
 * ## Two rules this file exists to hold
 *
 * 1. **An unclaimed capability is NOT a failure.** A model with no `vision`
 *    badge is `not-applicable` on the image test — never `failed`. That is what
 *    keeps the results matrix meaningful: a red cell always means "it said it
 *    could, and it could not".
 * 2. **Unknown is not empty.** A runtime that reports no capability list at all
 *    (a bare llama.cpp endpoint) yields `null`, which resolves to `unknown` —
 *    distinct from `[]`, which is a runtime that answered and named none. The
 *    UI offers an unknown-capability test rather than hiding or failing it.
 *
 * Scoring lives here rather than in the runner so every verdict is reproducible
 * from a stored transcript with no provider call — re-scoring an old run after
 * a keyword change costs nothing.
 */

import { escapeRegExp } from './textUtils.js';
import {
  buildRhetoricReferencePrompt,
  parseRhetoricJson,
  RHETORIC_REFERENCE_SET,
  scoreRhetoricReference,
} from './postRhetoric.js';

/** Verdicts every scorer returns. `partial` is real evidence, not a soft fail. */
export const TEST_VERDICTS = Object.freeze(['passed', 'partial', 'failed']);

/**
 * The prompt each generative test sends, verbatim.
 *
 * Exported because the consent gate shows it before the first provider call —
 * a user agreeing to a run should be able to read what is about to be asked.
 * Fixed wording is also what makes two models' results comparable.
 */
export const CAPABILITY_TEST_PROMPTS = Object.freeze({
  'image-analysis': 'Describe everything you can see in this image, including colours, objects, and any text or numbers.',
  'story-outline': [
    'Premise: a tidal-marsh oyster farmer finds that the water killing her beds is coming from the sea wall she campaigned to build.',
    '',
    "Outline this as a hero's journey in twelve beats. Give each beat its standard name as a heading, followed by one short paragraph.",
  ].join('\n'),
  'fiction-scene': [
    'Premise: a tidal-marsh oyster farmer finds that the water killing her beds is coming from the sea wall she campaigned to build.',
    '',
    'Write a 350–500 word opening scene in close third person.',
    'Keep the prose concrete and sensory. Include one spoken line of dialogue, the oyster farmer taking a consequential action, and a final turn that makes the sea wall the source of the dying water.',
  ].join('\n'),
  'rhetoric-reference': buildRhetoricReferencePrompt(),
});

/**
 * The task the sandbox agent is given. The sandbox already contains the broken
 * module, its test, and the data file the test reads — the model has to find
 * and read them, not be handed their contents.
 */
export const SANDBOX_TASK_PROMPT = [
  'This is a disposable PortOS capability check in the current directory.',
  'The test file cart-totals.test.mjs is failing.',
  'Read cart-totals.test.mjs, cart-totals.mjs and orders.json, then fix cart-totals.mjs on disk so the test passes.',
  'Run `node cart-totals.test.mjs` to check your work, and keep going until it exits 0.',
  'Do not edit cart-totals.test.mjs or orders.json, and do not create files outside this directory.',
].join(' ');

/**
 * One entry per test.
 *
 * `capabilities` is the badge set a model must ALREADY claim for the test to
 * apply — the gate, not a wish list. `prefers` is advisory only: it colours the
 * explanation ("a tool-caller without `code` usually narrates instead of
 * acting") and never decides applicability, because a model that clears the
 * gate deserves to be measured rather than pre-judged.
 */
export const CAPABILITY_TESTS = Object.freeze([
  Object.freeze({
    id: 'sandbox-repair',
    label: 'Sandbox repair',
    kind: 'agent',
    capabilities: Object.freeze(['tools']),
    prefers: Object.freeze(['code']),
    blurb: 'A broken module and its failing test are copied into a throwaway sandbox. The model must read the files, write a fix to disk, and make the test pass.',
    // Watching it work is the point of this one, so the runner drives a real
    // OpenCode TUI rather than an in-process tool loop.
    driver: 'tui',
  }),
  Object.freeze({
    id: 'image-analysis',
    label: 'Image analysis',
    kind: 'vision',
    capabilities: Object.freeze(['vision']),
    prefers: Object.freeze([]),
    blurb: 'A fixture image with known contents goes in; the description comes back. Scored on required and bonus keywords, so the check is deterministic.',
    driver: 'chat',
  }),
  Object.freeze({
    id: 'story-outline',
    label: 'Story outline',
    kind: 'text',
    capabilities: Object.freeze(['chat']),
    prefers: Object.freeze(['reasoning']),
    blurb: "Outline a hero's-journey story from one premise. Scored on how many of the twelve beats are present and in order.",
    driver: 'chat',
  }),
  Object.freeze({
    id: 'fiction-scene',
    label: 'Fiction scene',
    kind: 'text',
    capabilities: Object.freeze(['chat']),
    prefers: Object.freeze(['reasoning']),
    blurb: 'Write a short opening scene from a fixed premise. Scored on required story facts plus minimum scene craft signals; read the saved prose for quality.',
    driver: 'chat',
  }),
  Object.freeze({
    id: 'rhetoric-reference',
    label: 'Rhetoric evaluator',
    kind: 'text',
    capabilities: Object.freeze(['chat']),
    prefers: Object.freeze(['reasoning']),
    blurb: `Score ${RHETORIC_REFERENCE_SET.length} fictional, human-labelled rhetoric attempts. The result reports calibration error and agreement against the committed gold scores.`,
    driver: 'chat',
  }),
]);

export const CAPABILITY_TEST_IDS = Object.freeze(CAPABILITY_TESTS.map((t) => t.id));

export const getCapabilityTest = (id) => CAPABILITY_TESTS.find((t) => t.id === id) || null;

/**
 * Does this test apply to a model with these badges?
 *
 * @param {object} test one of CAPABILITY_TESTS
 * @param {string[]|null|undefined} capabilities `null`/`undefined` = the runtime
 *   reported none at all (unknown); `[]` = it answered and the model claims none.
 * @returns {{ state: 'applicable'|'not-applicable'|'unknown', missing: string[], reason: string|null }}
 */
export function applicabilityFor(test, capabilities) {
  if (!test) return { state: 'not-applicable', missing: [], reason: 'unknown test' };
  if (!Array.isArray(capabilities)) {
    return {
      state: 'unknown',
      missing: [],
      // Naming the runtime's silence rather than the model's: the model may well
      // be capable, and the UI offers the test anyway with this caveat attached.
      reason: 'this runtime reports no capability list, so the claim is unverified',
    };
  }
  const missing = test.capabilities.filter((c) => !capabilities.includes(c));
  if (missing.length) {
    return {
      state: 'not-applicable',
      missing,
      reason: `no ${missing.join(' or ')} badge`,
    };
  }
  return { state: 'applicable', missing: [], reason: null };
}

/** Every test with its applicability for one model. Order follows the catalog. */
export const applicableTests = (capabilities) =>
  CAPABILITY_TESTS.map((test) => ({ test, ...applicabilityFor(test, capabilities) }));

// ---- keyword scoring --------------------------------------------------------

/**
 * The vision fixture's known contents.
 *
 * REQUIRED are the four objects any usable vision model must name; the pass
 * turns on those alone. BONUS is detail work — two colours and a number painted
 * on a sign — which separates "captions the scene" from "reads the frame". A
 * model that gets every required term passes with bonus misses recorded, so the
 * score says what it was good at rather than collapsing to one number.
 */
export const VISION_FIXTURE_KEYWORDS = Object.freeze({
  required: Object.freeze([
    Object.freeze({ id: 'bicycle', label: 'Bicycle', any: Object.freeze(['bicycle', 'bike']) }),
    Object.freeze({ id: 'bench', label: 'Bench', any: Object.freeze(['bench']) }),
    Object.freeze({ id: 'lamp', label: 'Street lamp', any: Object.freeze(['lamp', 'lamppost', 'lamp post', 'streetlight', 'street light']) }),
    Object.freeze({ id: 'dog', label: 'Dog', any: Object.freeze(['dog', 'puppy']) }),
  ]),
  bonus: Object.freeze([
    Object.freeze({ id: 'red-bicycle', label: 'Bicycle is red', any: Object.freeze(['red', 'crimson', 'scarlet']) }),
    Object.freeze({ id: 'blue-bench', label: 'Bench is blue', any: Object.freeze(['blue']) }),
    Object.freeze({ id: 'sign-number', label: 'Reads the number 3 on the sign', any: Object.freeze(['3', 'three']) }),
  ]),
});

/**
 * The fiction fixture's content anchors. This is deliberately a modest
 * structural rubric, not a claim that keywords can judge literary quality.
 * The complete scene is retained so a human can read the prose after the
 * deterministic checks have separated a coherent attempt from a refusal or a
 * generic paragraph.
 */
export const FICTION_SCENE_KEYWORDS = Object.freeze({
  required: Object.freeze([
    Object.freeze({ id: 'tidal-marsh', label: 'Tidal-marsh setting', any: Object.freeze(['tidal marsh', 'marsh', 'estuary']) }),
    Object.freeze({ id: 'oyster-farmer', label: 'Oyster farmer', any: Object.freeze(['oyster farmer', 'oyster grower']) }),
    Object.freeze({ id: 'dying-beds', label: 'Dying oyster beds', any: Object.freeze(['dying', 'dead', 'die', 'killing', 'rot']) }),
    Object.freeze({ id: 'sea-wall', label: 'Sea wall', any: Object.freeze(['sea wall', 'seawall']) }),
    Object.freeze({ id: 'water', label: 'Water as the immediate problem', any: Object.freeze(['water', 'brine', 'tide', 'salt']) }),
  ]),
  bonus: Object.freeze([
    Object.freeze({ id: 'sensory-detail', label: 'Concrete sensory detail', any: Object.freeze(['mud', 'silt', 'stench', 'stink', 'cold', 'wind', 'grit', 'slick']) }),
    Object.freeze({ id: 'spoken-line', label: 'Spoken line', any: Object.freeze(['said', 'asked', 'whispered', 'shouted', 'murmured']) }),
    Object.freeze({ id: 'consequence', label: 'Consequential action', any: Object.freeze(['cut', 'climb', 'break', 'open', 'pull', 'wade', 'run', 'turn']) }),
  ]),
});

// A phrase is matched on word boundaries so "bike" does not fire on "bikeshed"
// and "3" does not fire inside "1024x3". Multi-word phrases tolerate any run of
// whitespace, because a wrapped transcript can break "lamp post" across lines.
const phrasePattern = (phrase) =>
  escapeRegExp(String(phrase).trim()).replace(/\\?\s+/g, '\\s+');

const keywordRegExp = (entry) =>
  new RegExp(`(?<![\\p{L}\\p{N}_])(?:${entry.any.map(phrasePattern).join('|')})(?![\\p{L}\\p{N}_])`, 'giu');

// "there is no dog" must not score as a dog. Only the few words immediately
// before the match are considered: a negation further back than that belongs to
// a different clause, and treating it as one here would start losing real hits.
const NEGATED_BEFORE = /\b(?:no|not|without|isn'?t|aren'?t|lacks?|lacking|missing|absent|cannot|can'?t)\b[^.;!?]{0,24}$/i;

const isNegated = (text, matchIndex) => NEGATED_BEFORE.test(text.slice(Math.max(0, matchIndex - 60), matchIndex));

/**
 * Is one keyword entry present in the text, ignoring negated mentions?
 * @returns {{ hit: boolean, matched: string|null }}
 */
export function findKeyword(text, entry) {
  const haystack = String(text || '');
  const re = keywordRegExp(entry);
  for (const match of haystack.matchAll(re)) {
    if (!isNegated(haystack, match.index)) return { hit: true, matched: match[0] };
  }
  return { hit: false, matched: null };
}

/**
 * Score a description against a required/bonus keyword spec.
 *
 * The pass turns on REQUIRED alone. Bonus terms are reported either way — they
 * are what makes two passing models distinguishable — but a bonus miss can
 * never fail a run, or "bonus" would just be a second required tier.
 */
export function scoreKeywords(text, spec = VISION_FIXTURE_KEYWORDS) {
  const score = (entries) => (entries || []).map((entry) => ({
    id: entry.id,
    label: entry.label,
    any: [...entry.any],
    ...findKeyword(text, entry),
  }));

  const required = score(spec.required);
  const bonus = score(spec.bonus);
  const requiredHit = required.filter((r) => r.hit).length;
  const bonusHit = bonus.filter((b) => b.hit).length;
  const verdict = requiredHit === required.length
    ? 'passed'
    : (requiredHit >= Math.ceil(required.length / 2) ? 'partial' : 'failed');

  return {
    verdict,
    required,
    bonus,
    requiredHit,
    requiredTotal: required.length,
    bonusHit,
    bonusTotal: bonus.length,
    summary: `${requiredHit} of ${required.length} required`
      + (bonus.length ? `, ${bonusHit} of ${bonus.length} bonus` : ''),
  };
}

// ---- story-beat scoring -----------------------------------------------------

/**
 * The twelve beats, with the wordings a model actually uses for each.
 *
 * Matching is on the beat NAME, not on the prose: the test asks for named
 * headings, so a missing heading is a missing beat rather than a scoring
 * failure. Alternates cover the common synonyms ("Call to adventure" /
 * "The call"), not every possible paraphrase — a model that renames the
 * structure has not produced the structure that was asked for.
 */
export const HEROS_JOURNEY_BEATS = Object.freeze([
  Object.freeze({ id: 'ordinary-world', label: 'Ordinary world', any: Object.freeze(['ordinary world', 'everyday world', 'status quo', 'the known world']) }),
  Object.freeze({ id: 'call-to-adventure', label: 'Call to adventure', any: Object.freeze(['call to adventure', 'the call', 'inciting incident']) }),
  Object.freeze({ id: 'refusal', label: 'Refusal of the call', any: Object.freeze(['refusal of the call', 'refusal', 'refuses the call']) }),
  Object.freeze({ id: 'mentor', label: 'Meeting the mentor', any: Object.freeze(['meeting the mentor', 'meeting with the mentor', 'the mentor', 'supernatural aid']) }),
  Object.freeze({ id: 'threshold', label: 'Crossing the threshold', any: Object.freeze(['crossing the threshold', 'crossing the first threshold', 'the threshold']) }),
  Object.freeze({ id: 'tests-allies', label: 'Tests, allies, enemies', any: Object.freeze(['tests, allies', 'tests allies', 'allies and enemies', 'the road of trials', 'trials']) }),
  Object.freeze({ id: 'approach', label: 'Approach to the inmost cave', any: Object.freeze(['approach to the inmost cave', 'inmost cave', 'innermost cave', 'the approach']) }),
  Object.freeze({ id: 'ordeal', label: 'The ordeal', any: Object.freeze(['the ordeal', 'ordeal', 'the central crisis']) }),
  Object.freeze({ id: 'reward', label: 'The reward', any: Object.freeze(['the reward', 'reward', 'seizing the sword']) }),
  Object.freeze({ id: 'road-back', label: 'The road back', any: Object.freeze(['the road back', 'road back', 'the return journey']) }),
  Object.freeze({ id: 'resurrection', label: 'Resurrection', any: Object.freeze(['resurrection', 'the final test', 'climax']) }),
  Object.freeze({ id: 'elixir', label: 'Return with the elixir', any: Object.freeze(['return with the elixir', 'the elixir', 'elixir', 'freedom to live']) }),
]);

/**
 * Score an outline for beat coverage and ordering.
 *
 * Coverage is structure, NOT quality — it can only say the model held a
 * twelve-part shape across a long generation. Whether the outline is any good is
 * a reading job, which is why the runner always keeps the full text.
 *
 * @returns {{verdict:string, beats:Array, found:number, total:number, inOrder:boolean, summary:string}}
 */
export function scoreStoryBeats(text, beats = HEROS_JOURNEY_BEATS) {
  const haystack = String(text || '');
  const scored = beats.map((beat) => {
    const re = keywordRegExp(beat);
    const match = re.exec(haystack);
    return {
      id: beat.id,
      label: beat.label,
      hit: Boolean(match),
      // Where the beat first appears, used for the ordering check. `null` for a
      // miss — never -1, which would sort ahead of every real position.
      at: match ? match.index : null,
    };
  });

  const hits = scored.filter((b) => b.hit);
  // Ordering is judged on the beats that ARE present: an outline missing beat 3
  // is not out of order, it is short one beat, and conflating the two would
  // punish the same gap twice.
  const inOrder = hits.every((beat, i) => i === 0 || beat.at > hits[i - 1].at);
  const found = hits.length;
  const verdict = found === beats.length && inOrder
    ? 'passed'
    : (found >= Math.ceil(beats.length * 0.66) ? 'partial' : 'failed');

  return {
    verdict,
    beats: scored,
    found,
    total: beats.length,
    inOrder,
    summary: `${found} of ${beats.length} beats${found > 1 && !inOrder ? ' — out of order' : ''}`,
  };
}

/**
 * Score a short fiction scene without pretending that a lexical rubric is an
 * aesthetic judge. Required anchors establish that the model followed the
 * premise; length, paragraphing, and dialogue establish that it attempted a
 * scene rather than returning notes. The prose itself remains the evidence a
 * human should use for voice, originality, and sentence quality.
 *
 * @returns {{verdict:string, summary:string, required:Array, bonus:Array,
 *   requiredHit:number, requiredTotal:number, bonusHit:number, bonusTotal:number,
 *   wordCount:number, paragraphCount:number, hasDialogue:boolean}}
 */
export function scoreFictionScene(text, spec = FICTION_SCENE_KEYWORDS) {
  const keyword = scoreKeywords(text, spec);
  const source = String(text || '').trim();
  const words = source ? source.split(/\s+/u).filter(Boolean) : [];
  const paragraphs = source ? source.split(/\n\s*\n/u).map((p) => p.trim()).filter(Boolean) : [];
  const hasDialogue = /["“][^"”\n]{2,}["”]/u.test(source)
    || /\b(?:said|asked|whispered|shouted|murmured)\s*:\s*\S/iu.test(source)
    || /(?:^|\n)\s*—\s*\S(?:.*\S)?$/mu.test(source);
  const craft = {
    minimumWords: words.length >= 350 && words.length <= 500,
    paragraphs: paragraphs.length >= 3,
    dialogue: hasDialogue,
  };
  const craftHit = Object.values(craft).filter(Boolean).length;
  const requiredComplete = keyword.requiredHit === keyword.requiredTotal;
  const verdict = requiredComplete && craftHit === Object.keys(craft).length
    ? 'passed'
    : (keyword.requiredHit >= Math.ceil(keyword.requiredTotal * 0.6) && craftHit >= 1 ? 'partial' : 'failed');
  return {
    ...keyword,
    verdict,
    wordCount: words.length,
    paragraphCount: paragraphs.length,
    hasDialogue,
    craft,
    summary: `${keyword.requiredHit} of ${keyword.requiredTotal} story anchors, ${keyword.bonusHit} of ${keyword.bonusTotal} scene signals, ${craftHit} of ${Object.keys(craft).length} minimum craft checks, ${words.length} words`,
  };
}

// ---- sandbox-repair scoring -------------------------------------------------

/**
 * Turn the observed facts of a sandbox run into a verdict.
 *
 * Every input is read from DISK after the agent stops — never from what the
 * agent said it did. A model that reports success without touching the file
 * scores nothing.
 *
 * @param {object} facts
 * @param {boolean} facts.moduleChanged the module under repair differs from the fixture
 * @param {boolean} facts.fixturesIntact the test and data files are byte-identical to the fixture
 * @param {boolean} facts.testsPass the sandbox test command exited 0
 * @param {number} facts.toolCalls tool calls observed in the agent stream
 */
export function scoreSandboxRepair({ moduleChanged, fixturesIntact, testsPass, toolCalls = 0 }) {
  const checks = [
    { id: 'module-written', label: 'Wrote a fix to the module on disk', ok: Boolean(moduleChanged) },
    { id: 'fixtures-intact', label: 'Left the test and data files untouched', ok: Boolean(fixturesIntact) },
    { id: 'tests-pass', label: 'The sandbox test command exits 0', ok: Boolean(testsPass) },
  ];
  // Editing the test into submission is the one way to "pass" without fixing
  // anything, so it fails outright rather than scoring partial — a partial there
  // would read as progress toward a solution it is moving away from.
  const verdict = !fixturesIntact
    ? 'failed'
    : (testsPass ? 'passed' : (moduleChanged ? 'partial' : 'failed'));
  const summary = verdict === 'passed'
    ? `fixed in ${toolCalls} tool call${toolCalls === 1 ? '' : 's'}`
    : (!fixturesIntact
      ? 'edited the test or data file instead of the module'
      : (moduleChanged ? 'wrote a fix, but the test still fails' : 'never wrote a fix'));

  return { verdict, checks, summary, toolCalls };
}

/** Roll several test verdicts into the one a model's row shows. */
export function rollUpVerdict(verdicts) {
  const list = (verdicts || []).filter((v) => TEST_VERDICTS.includes(v));
  if (!list.length) return null;
  if (list.includes('failed')) return 'failed';
  if (list.includes('partial')) return 'partial';
  return 'passed';
}