Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
17.3 KB
Raw Blame
/**
 * SkillOpt models plan: which model every touchpoint of a run will call and
 * which configuration chose it.
 *
 * `resolveSkillOptModels` is the ONE resolution of the optimizer / target /
 * judge roles (tiers deep / subagent / reasoning) shared by the CLI, the
 * `run_skillopt` MCP op, the cycle phase and the background job handler.
 * `buildModelsPlan` adds the engine-internal touchpoints a run can reach:
 * per-task `judge.model` overrides (benchmark + held-out), the gateway's
 * expansion and chat models (provenance retained by
 * `reconfigureGatewayWithEngine`), the embedding model of the column search
 * actually reads, and the reranker when search enables it.
 *
 * Strict mode (`--models-strict` / `skillopt.models_strict`) guarantees that
 * every ACTIVE model was chosen by explicit, touchpoint-specific
 * configuration: a touchpoint resolved through `models.default`, a built-in
 * default, a capability substitution or unknown provenance aborts the run
 * before any spend with a copy-paste fix per touchpoint. Aliases are allowed;
 * the guarantee is about who chose the model, not a fixed identity.
 *
 * Every echoed string (model ids, origins, raw config values) passes
 * `sanitizeEcho`: control characters stripped, capped at 200 chars.
 */

import { getChatModel, getExpansionModel } from '../ai/gateway.ts';
import { getGatewayModelSource } from '../ai/gateway-model-sources.ts';
import { loadConfig, loadConfigFileOnly, loadConfigWithEngine } from '../config.ts';
import type { ConfigReader } from '../config-snapshot.ts';
import type { BrainEngine } from '../engine.ts';
import { errorFor } from '../errors.ts';
import {
  describeResolveOrigin,
  resolveModelDetailed,
  TIER_DEFAULTS,
  type ModelTier,
  type ResolveSource,
} from '../model-config.ts';
import { DEFAULT_COLUMN_NAME, resolveEmbeddingColumn } from '../search/embedding-column.ts';
import { loadSearchModeConfig, resolveSearchMode } from '../search/mode.ts';
import type { ReflectCap } from './output-cap.ts';
import type { BenchmarkTask } from './types.ts';

export type ModelSource = ResolveSource | 'file_config' | 'benchmark' | 'unknown';

export interface ModelResolution {
  model: string;
  source: ModelSource;
  /** Flag / config key / env var that supplied the model, or `built-in default`. */
  origin: string;
  substituted_from?: string;
  substitution_reason?: string;
}

export type SkillOptRole = 'optimizer' | 'target' | 'judge';

export type SkillOptModels = Record<SkillOptRole, ModelResolution>;

export type PlanTouchpoint = SkillOptRole | 'expansion' | 'chat' | 'embedding' | 'reranker';

export interface ModelsPlanEntry extends ModelResolution {
  touchpoint: PlanTouchpoint;
  active: boolean;
  inactive_reason?: string;
}

export interface StrictViolation {
  touchpoint: PlanTouchpoint;
  model: string;
  source: ModelSource;
  origin: string;
  /** Copy-paste remediation (a `gbrain config set` line, or the embedding config path). */
  fix: string;
}

export interface StrictVerdict {
  enabled: boolean;
  ok: boolean;
  violations: StrictViolation[];
  /** Raw `skillopt.models_strict` value that was not recognized (treated as on). */
  unrecognized_value?: string;
}

export const MODELS_STRICT_CONFIG_KEY = 'skillopt.models_strict';

const ECHO_MAX_CHARS = 200;

/** Strip control characters / newlines and cap at 200 chars. */
export function sanitizeEcho(value: string): string {
  return value.replace(/[\u0000-\u001f\u007f-\u009f]/g, '').slice(0, ECHO_MAX_CHARS);
}

const ROLE_RESOLUTION: Record<SkillOptRole, { tier: ModelTier; flag: string }> = {
  optimizer: { tier: 'deep', flag: '--optimizer-model' },
  target: { tier: 'subagent', flag: '--target-model' },
  judge: { tier: 'reasoning', flag: '--judge-model' },
};

export interface SkillOptModelFlags {
  optimizerModel?: string;
  targetModel?: string;
  judgeModel?: string;
}

/**
 * Resolve the three skillopt roles: flag > role config chain (tier config,
 * `models.default`, env, key-aware tier default). Reports the source, the
 * origin key and any subagent capability substitution.
 */
export async function resolveSkillOptModels(
  engine: ConfigReader | null,
  flags: SkillOptModelFlags = {},
): Promise<SkillOptModels> {
  const flagFor: Record<SkillOptRole, string | undefined> = {
    optimizer: flags.optimizerModel,
    target: flags.targetModel,
    judge: flags.judgeModel,
  };
  const out = {} as SkillOptModels;
  for (const role of ['optimizer', 'target', 'judge'] as const) {
    const { tier, flag } = ROLE_RESOLUTION[role];
    const r = await resolveModelDetailed(engine, { cliFlag: flagFor[role], tier, fallback: TIER_DEFAULTS[tier] });
    out[role] = {
      model: sanitizeEcho(r.model),
      source: r.source,
      origin: describeResolveOrigin(r.source, { tier, flag }),
      ...(r.substituted_from ? { substituted_from: sanitizeEcho(r.substituted_from), substitution_reason: r.substitution_reason } : {}),
    };
  }
  return out;
}

/** Role provenance for models handed over without one (programmatic callers, legacy jobs): fail-closed `unknown`. */
export function unknownProvenanceModels(
  models: { optimizerModel: string; targetModel: string; judgeModel: string },
  origin: string,
): SkillOptModels {
  const entry = (model: string): ModelResolution => ({ model: sanitizeEcho(model), source: 'unknown', origin });
  return { optimizer: entry(models.optimizerModel), target: entry(models.targetModel), judge: entry(models.judgeModel) };
}

/** Spread into SkillOptOpts: the role models plus their provenance. */
export function skillOptModelOpts(models: SkillOptModels): {
  optimizerModel: string; targetModel: string; judgeModel: string; models: SkillOptModels;
} {
  return { optimizerModel: models.optimizer.model, targetModel: models.target.model, judgeModel: models.judge.model, models };
}

function judgeUsage(tasks: readonly BenchmarkTask[] | undefined): {
  defaultActive: boolean; inactiveReason?: string; overrides: Map<string, number>;
} {
  const overrides = new Map<string, number>();
  if (!tasks) return { defaultActive: true, overrides };
  let llmTasks = 0;
  let defaultActive = false;
  for (const t of tasks) {
    const judge = t.judge as { kind?: string; model?: unknown };
    if (judge.kind === 'rule' || judge.kind === 'qrels') continue;
    llmTasks += 1;
    const override = judge.kind === 'llm' && typeof judge.model === 'string' ? judge.model.trim() : '';
    if (override) overrides.set(override, (overrides.get(override) ?? 0) + 1);
    else defaultActive = true;
  }
  if (defaultActive) return { defaultActive, overrides };
  return { defaultActive, inactiveReason: llmTasks === 0 ? 'rule/qrels benchmark' : 'every LLM task sets judge.model', overrides };
}

function gatewayEntry(touchpoint: 'expansion' | 'chat'): ModelsPlanEntry {
  let model: string;
  try {
    model = touchpoint === 'expansion' ? getExpansionModel() : getChatModel();
  } catch {
    return { touchpoint, model: '(gateway unconfigured)', source: 'unknown', origin: 'gateway unconfigured', active: true };
  }
  const src = getGatewayModelSource(touchpoint, model);
  return {
    touchpoint,
    model: sanitizeEcho(model),
    source: src?.source ?? 'unknown',
    origin: src?.origin ?? 'unrecorded: gateway not resolved against this brain',
    active: true,
  };
}

/**
 * Embedding provenance from the resolver search itself uses: the active
 * column (`search_embedding_column` / `embedding_columns`), else the
 * file-plane `embedding_model` or `GBRAIN_EMBEDDING_MODEL`. DB-plane
 * `embedding_model` rows are never merged into search config, so they are
 * never reported as the source.
 */
async function embeddingEntry(engine: BrainEngine): Promise<ModelsPlanEntry> {
  const unknown = (origin: string): ModelsPlanEntry =>
    ({ touchpoint: 'embedding', model: '(unresolved)', source: 'unknown', origin, active: true });
  let column;
  let explicitDefaultColumn = false;
  try {
    const cfg = (await loadConfigWithEngine(engine).catch(() => null)) ?? loadConfig() ?? { engine: engine.kind };
    column = resolveEmbeddingColumn(undefined, cfg);
    const userColumns: unknown = cfg.embedding_columns;
    explicitDefaultColumn = typeof userColumns === 'object' && userColumns !== null && Object.hasOwn(userColumns, DEFAULT_COLUMN_NAME);
  } catch (err) {
    return unknown(sanitizeEcho(`embedding column unresolved: ${err instanceof Error ? err.message : String(err)}`));
  }
  if (!column.embeddingModel) return unknown('unverified embedding identity (gbrain migrate embeddings --status)');
  const model = sanitizeEcho(column.embeddingModel);
  if (column.name !== DEFAULT_COLUMN_NAME) {
    return { touchpoint: 'embedding', model, source: 'config_key', origin: sanitizeEcho(`search_embedding_column=${column.name} (embedding_columns)`), active: true };
  }
  if (explicitDefaultColumn) {
    return { touchpoint: 'embedding', model, source: 'config_key', origin: `embedding_columns.${DEFAULT_COLUMN_NAME}`, active: true };
  }
  if (process.env.GBRAIN_EMBEDDING_MODEL?.trim()) {
    return { touchpoint: 'embedding', model, source: 'env', origin: 'GBRAIN_EMBEDDING_MODEL', active: true };
  }
  let fileModel: string | undefined;
  try { fileModel = loadConfigFileOnly()?.embedding_model?.trim(); } catch { /* unreadable file plane */ }
  return fileModel
    ? { touchpoint: 'embedding', model, source: 'file_config', origin: 'embedding_model (config.json)', active: true }
    : { touchpoint: 'embedding', model, source: 'fallback', origin: 'built-in default', active: true };
}

/** Reranker entry, only when search enables reranking. */
async function rerankerEntry(engine: BrainEngine): Promise<ModelsPlanEntry | null> {
  let input;
  try { input = await loadSearchModeConfig(engine); } catch { return null; }
  const knobs = resolveSearchMode(input);
  if (!knobs.reranker_enabled) return null;
  const explicit = input.overrides?.reranker_model !== undefined;
  return {
    touchpoint: 'reranker',
    model: sanitizeEcho(knobs.reranker_model),
    source: explicit ? 'config_key' : 'fallback',
    origin: explicit ? 'search.reranker.model' : `built-in default, search.mode ${knobs.resolved_mode}`,
    active: true,
  };
}

/**
 * Full plan for one run. `tasks` are every task the run can judge
 * (benchmark + held-out); omitted, the default judge is assumed active.
 */
export async function buildModelsPlan(
  engine: BrainEngine,
  models: SkillOptModels,
  tasks?: readonly BenchmarkTask[],
): Promise<ModelsPlanEntry[]> {
  const judges = judgeUsage(tasks);
  const plan: ModelsPlanEntry[] = [
    { touchpoint: 'optimizer', ...models.optimizer, active: true },
    { touchpoint: 'target', ...models.target, active: true },
    {
      touchpoint: 'judge',
      ...models.judge,
      active: judges.defaultActive,
      ...(judges.inactiveReason ? { inactive_reason: judges.inactiveReason } : {}),
    },
  ];
  for (const [model, count] of judges.overrides) {
    plan.push({
      touchpoint: 'judge',
      model: sanitizeEcho(model),
      source: 'benchmark',
      origin: `benchmark judge.model, ${count} task${count === 1 ? '' : 's'}`,
      active: true,
    });
  }
  plan.push(gatewayEntry('expansion'), gatewayEntry('chat'), await embeddingEntry(engine));
  const reranker = await rerankerEntry(engine);
  if (reranker) plan.push(reranker);
  return plan;
}

const FALLBACK_SOURCES: ReadonlySet<ModelSource> = new Set(['models_default', 'tier_default', 'fallback', 'unknown']);

/** Origin as echoed: the key, plus the substitution when the capability gate replaced the configured model. */
function originText(e: ModelResolution): string {
  return e.substituted_from
    ? `${e.origin} -> substituted: ${e.substitution_reason ?? 'unsupported'}; configured ${e.substituted_from}`
    : e.origin;
}

/** True when strict mode counts this entry's choice as a fallback (not explicit configuration). */
export function isFallbackChoice(entry: ModelResolution): boolean {
  return FALLBACK_SOURCES.has(entry.source) || entry.substituted_from !== undefined;
}

const FIX_KEY: Record<Exclude<PlanTouchpoint, 'embedding'>, string> = {
  optimizer: 'models.tier.deep',
  target: 'models.tier.subagent',
  judge: 'models.tier.reasoning',
  expansion: 'models.tier.utility',
  chat: 'models.chat',
  reranker: 'search.reranker.model',
};

function fixFor(entry: ModelsPlanEntry): string {
  if (entry.touchpoint === 'embedding') {
    return `set embedding_model in ~/.gbrain/config.json or export GBRAIN_EMBEDDING_MODEL=${entry.model} (changing the embedding model needs a migration: gbrain migrate embeddings --status)`;
  }
  return `gbrain config set ${FIX_KEY[entry.touchpoint]} ${entry.model}`;
}

export function strictVerdict(
  plan: readonly ModelsPlanEntry[],
  strict: { on: boolean; unrecognized?: string },
): StrictVerdict {
  const violations = plan
    .filter((e) => e.active && isFallbackChoice(e))
    .map((e) => ({ touchpoint: e.touchpoint, model: e.model, source: e.source, origin: originText(e), fix: fixFor(e) }));
  return {
    enabled: strict.on,
    ok: violations.length === 0,
    violations,
    ...(strict.unrecognized !== undefined ? { unrecognized_value: strict.unrecognized } : {}),
  };
}

const STRICT_ON = new Set(['true', '1', 'yes', 'on']);
const STRICT_OFF = new Set(['false', '0', 'no', 'off', '']);
const warnedStrictValues = new Set<string>();

/** `true|1|yes|on` on; `false|0|no|off|empty` off; anything else warns once and counts as ON. */
export function parseModelsStrict(raw: string | null | undefined): { on: boolean; unrecognized?: string } {
  if (raw === null || raw === undefined) return { on: false };
  const v = raw.trim().toLowerCase();
  if (STRICT_ON.has(v)) return { on: true };
  if (STRICT_OFF.has(v)) return { on: false };
  const unrecognized = sanitizeEcho(raw);
  if (!warnedStrictValues.has(unrecognized)) {
    warnedStrictValues.add(unrecognized);
    process.stderr.write(`[skillopt] ${MODELS_STRICT_CONFIG_KEY}='${unrecognized}' is not a recognized value (true/false); treating it as on\n`);
  }
  return { on: true, unrecognized };
}

/** `--models-strict` (flag) wins; otherwise `skillopt.models_strict` config. */
export async function resolveModelsStrict(
  engine: ConfigReader,
  flag?: boolean,
): Promise<{ on: boolean; unrecognized?: string }> {
  if (flag === true) return { on: true };
  let raw: string | null | undefined = null;
  try {
    raw = await engine.getConfig(MODELS_STRICT_CONFIG_KEY);
  } catch {
    // Fail closed: an unreadable policy must not silently disable enforcement.
    process.stderr.write(`[skillopt] could not read ${MODELS_STRICT_CONFIG_KEY}; treating strict mode as on\n`);
    return { on: true };
  }
  return parseModelsStrict(raw);
}

/** Multi-line verdict: pass, or each offending touchpoint with its copy-paste fix. */
export function describeStrictVerdict(verdict: StrictVerdict): string {
  if (verdict.ok) return 'models strict check: pass (every active model was chosen by explicit configuration)';
  const lines = [
    `models strict check: ${verdict.violations.length} active model touchpoint(s) were not chosen by touchpoint-specific configuration:`,
  ];
  for (const v of verdict.violations) {
    lines.push(`  ${v.touchpoint}  ${v.model}  (${v.origin})`, `    ${v.fix}`);
  }
  if (verdict.unrecognized_value !== undefined) {
    lines.push(`(${MODELS_STRICT_CONFIG_KEY}='${verdict.unrecognized_value}' treated as on)`);
  }
  return lines.join('\n');
}

/** Pre-spend strict abort listing each offending touchpoint with its fix. */
export function modelsStrictError(verdict: StrictVerdict): Error {
  return errorFor({
    class: 'ModelsStrict',
    code: 'models_strict_violation',
    message: describeStrictVerdict(verdict),
    hint: `run the listed commands, or turn strict mode off (drop --models-strict; gbrain config set ${MODELS_STRICT_CONFIG_KEY} false)`,
  });
}

const FALLBACK_FOOTNOTE = '* not chosen by touchpoint-specific configuration (models.default, a built-in default, a substitution or unknown provenance); --models-strict aborts on these';

function bannerRow(e: ModelsPlanEntry): string {
  const origin = e.active ? originText(e) : `inactive: ${e.inactive_reason ?? 'unused'}`;
  const mark = e.active && isFallbackChoice(e) ? ' *' : '';
  return `${e.touchpoint.padEnd(9)}  ${e.model}  (${origin})${mark}`;
}

const sameEntry = (a: ModelsPlanEntry, b: ModelsPlanEntry): boolean =>
  a.touchpoint === b.touchpoint && a.model === b.model && a.source === b.source && a.origin === b.origin && a.active === b.active;

/**
 * Stderr banner. With `baseline` (batch / cycle runs after the invocation
 * banner) only rows that differ from it print, and nothing when none do.
 */
export function formatModelsBanner(
  plan: readonly ModelsPlanEntry[],
  opts: { skill?: string; baseline?: readonly ModelsPlanEntry[]; reflectCap?: ReflectCap } = {},
): string {
  const rows = opts.baseline ? plan.filter((e) => !opts.baseline!.some((b) => sameEntry(b, e))) : plan;
  if (rows.length === 0) return '';
  const scope = opts.skill ? ` for ${sanitizeEcho(opts.skill)}` : '';
  const lines = [opts.baseline ? `[skillopt] Models${scope} (differs from the run banner):` : `[skillopt] Models${scope}:`];
  for (const e of rows) lines.push(`[skillopt]   ${bannerRow(e)}`);
  if (opts.reflectCap && !opts.baseline) {
    lines.push(`[skillopt]   optimizer output cap ${opts.reflectCap.maxTokens} tokens (${opts.reflectCap.source === 'flag' ? '--reflect-max-tokens' : opts.reflectCap.source === 'config' ? 'skillopt.reflect_max_tokens' : 'default'})`);
  }
  if (rows.some((e) => e.active && isFallbackChoice(e))) lines.push(`[skillopt]   ${FALLBACK_FOOTNOTE}`);
  return lines.join('\n') + '\n';
}