Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.7 KB
Raw Blame
/**
 * Ranker wave, Phase E2 (Cat 13 conceptual recall) — pure pins for
 * arm-confidence.ts, its composition through fusion-lists.ts, and the
 * `keyword_arm_confidence_floor` knob plane (bundle, config parse, resolution
 * chain, knobs hash `kacf=`, registry).
 *
 * The measured defect (Voyage space, reranker off, autocut off): hybrid nDCG@5
 * 53.0 on the held-out concepts vs bare vector 60.5 — the keyword arm's noise
 * on paraphrase probes drags the fused result below the vector arm. The ONE
 * pre-registered mechanism: when the keyword arm's scale-free margin ratio
 * `top / (top + second)` is below the floor, the keyword AND title lists fuse
 * at weight 0.5. Every bundle lands with the floor OFF (null).
 */

import { describe, expect, test } from 'bun:test';
import {
  KEYWORD_ARM_CONFIDENCE_FLOOR_MAX,
  KEYWORD_ARM_WEAK_WEIGHT,
  decideKeywordArmWeight,
  keywordArmConfidence,
  normalizeKeywordArmConfidenceFloor,
  type KeywordArmConfidenceDecision,
} from '../../src/core/search/arm-confidence.ts';
import { composeFusionLists, pushVectorList, type VectorArm } from '../../src/core/search/fusion-lists.ts';
import {
  MODE_BUNDLES,
  SEARCH_MODES,
  SEARCH_MODE_CONFIG_KEYS,
  attributeKnob,
  knobsHash,
  loadOverridesFromConfig,
  resolveSearchMode,
  type ResolvedSearchKnobs,
} from '../../src/core/search/mode.ts';
import { KNOB_DESCRIPTIONS, KNOB_NULL_LABELS, formatKnobValue } from '../../src/core/search/modes-report.ts';
import { KNOWN_CONFIG_KEYS } from '../../src/core/config.ts';
import { rrfFusionWeighted } from '../../src/core/search/hybrid.ts';
import type { SearchResult } from '../../src/core/types.ts';

function row(slug: string, score: number, chunk_id = 1): SearchResult {
  return {
    slug,
    page_id: 1,
    title: slug,
    type: 'note',
    chunk_text: `${slug} body`,
    chunk_source: 'timeline',
    chunk_id,
    chunk_index: 0,
    score,
    stale: false,
  } as SearchResult;
}

const KS = { vectorK: 60, textRrfK: 50, imageRrfK: 75, keywordK: 66, baseRrfK: 60 };

describe('keywordArmConfidence — scale-free margin ratio over the returned rows', () => {
  test('empty arm → margin 0, top 0', () => {
    expect(keywordArmConfidence([])).toEqual({ margin_ratio: 0, top_score: 0 });
  });

  test('a single row is uncontested → margin 1, top = its score', () => {
    expect(keywordArmConfidence([row('a', 0.31)])).toEqual({ margin_ratio: 1, top_score: 0.31 });
  });

  test('two rows: top / (top + second); an exact tie is 0.5', () => {
    expect(keywordArmConfidence([row('a', 0.2), row('b', 0.2)]).margin_ratio).toBeCloseTo(0.5, 12);
    const c = keywordArmConfidence([row('a', 0.3), row('b', 0.1)]);
    expect(c.margin_ratio).toBeCloseTo(0.75, 12);
    expect(c.top_score).toBe(0.3);
  });

  test('order-independent: takes the two LARGEST scores, not positions 0/1', () => {
    const asc = keywordArmConfidence([row('c', 0.05), row('b', 0.1), row('a', 0.3)]);
    const desc = keywordArmConfidence([row('a', 0.3), row('b', 0.1), row('c', 0.05)]);
    expect(asc).toEqual(desc);
    expect(asc.margin_ratio).toBeCloseTo(0.75, 12);
  });

  test('scale invariance: multiplying every score by a constant leaves the margin unchanged', () => {
    const base = [row('a', 0.3), row('b', 0.1), row('c', 0.02)];
    const scaled = base.map((r) => ({ ...r, score: r.score * 1000 }));
    expect(keywordArmConfidence(scaled).margin_ratio).toBeCloseTo(keywordArmConfidence(base).margin_ratio, 12);
    expect(keywordArmConfidence(scaled).top_score).toBeCloseTo(300, 9);
  });

  test('non-finite / non-positive scores count as 0 evidence', () => {
    expect(keywordArmConfidence([row('a', 0)]).margin_ratio).toBe(0);
    expect(keywordArmConfidence([row('a', Number.NaN), row('b', -1)]).margin_ratio).toBe(0);
    // A real top over a zero second is uncontested.
    expect(keywordArmConfidence([row('a', 0.2), row('b', 0)]).margin_ratio).toBe(1);
    expect(keywordArmConfidence([row('a', 0.2), row('b', Number.POSITIVE_INFINITY)]).margin_ratio).toBe(1);
  });

  test('margin lives in [0, 1]', () => {
    for (const rows of [[], [row('a', 1)], [row('a', 1), row('b', 1)], [row('a', 5), row('b', 1), row('c', 4)]]) {
      const m = keywordArmConfidence(rows).margin_ratio;
      expect(m).toBeGreaterThanOrEqual(0);
      expect(m).toBeLessThanOrEqual(1);
    }
  });
});

describe('normalizeKeywordArmConfidenceFloor — the ONE range contract for keyword_arm_confidence_floor', () => {
  test('finite numbers in (0, 1] pass through (boundary 1 inclusive)', () => {
    expect(normalizeKeywordArmConfidenceFloor(0.6)).toBe(0.6);
    expect(normalizeKeywordArmConfidenceFloor(1)).toBe(1);
    expect(normalizeKeywordArmConfidenceFloor(Number.MIN_VALUE)).toBe(Number.MIN_VALUE);
    expect(KEYWORD_ARM_CONFIDENCE_FLOOR_MAX).toBe(1);
  });

  test('null / false and the literals off / null / false → null (knob off)', () => {
    expect(normalizeKeywordArmConfidenceFloor(null)).toBeNull();
    expect(normalizeKeywordArmConfidenceFloor(false)).toBeNull();
    expect(normalizeKeywordArmConfidenceFloor('off')).toBeNull();
    expect(normalizeKeywordArmConfidenceFloor(' OFF ')).toBeNull();
    expect(normalizeKeywordArmConfidenceFloor('null')).toBeNull();
    expect(normalizeKeywordArmConfidenceFloor('false')).toBeNull();
  });

  test('numeric strings parse (config-key plane); garbage does not', () => {
    expect(normalizeKeywordArmConfidenceFloor('0.6')).toBe(0.6);
    expect(normalizeKeywordArmConfidenceFloor(' 1 ')).toBe(1);
    expect(normalizeKeywordArmConfidenceFloor('')).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor('   ')).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor('x')).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor('0.6abc')).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor('NaN')).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor('Infinity')).toBeUndefined();
  });

  test('out-of-range / non-finite / wrong-typed → undefined (unset: fall through)', () => {
    expect(normalizeKeywordArmConfidenceFloor(0)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(-0)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(-0.5)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(1.0001)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(2)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(Number.NaN)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(Number.POSITIVE_INFINITY)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(undefined)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor(true)).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor({})).toBeUndefined();
    expect(normalizeKeywordArmConfidenceFloor([0.6])).toBeUndefined();
  });
});

describe('decideKeywordArmWeight — the weight decision', () => {
  const weak = [row('d1', 0.2), row('d2', 0.2)]; // margin 0.5
  const strong = [row('d1', 0.9)]; // margin 1
  const base = { vectorArmVoted: true, relationalQuery: false };

  test('floor null → weight undefined (no key emitted), decision still carries the margin', () => {
    const r = decideKeywordArmWeight({ keywordList: weak, floor: null, ...base });
    expect(r.weight).toBeUndefined();
    expect(r.decision).toEqual({ margin_ratio: 0.5, top_score: 0.2, downweighted: false });
  });

  test('floor set + weak arm + vector voted + non-relational → weight 0.5', () => {
    const r = decideKeywordArmWeight({ keywordList: weak, floor: 0.6, ...base });
    expect(r.weight).toBe(KEYWORD_ARM_WEAK_WEIGHT);
    expect(r.weight).toBe(0.5);
    expect(r.decision.downweighted).toBe(true);
  });

  test('strong arm → unchanged', () => {
    const r = decideKeywordArmWeight({ keywordList: strong, floor: 0.6, ...base });
    expect(r.weight).toBeUndefined();
    expect(r.decision).toEqual({ margin_ratio: 1, top_score: 0.9, downweighted: false });
  });

  test('relational query → unchanged even when weak', () => {
    const r = decideKeywordArmWeight({ keywordList: weak, floor: 0.6, vectorArmVoted: true, relationalQuery: true });
    expect(r.weight).toBeUndefined();
    expect(r.decision.downweighted).toBe(false);
  });

  test('no vector vote (keyword-only fallback shape) → unchanged even when weak', () => {
    const r = decideKeywordArmWeight({ keywordList: weak, floor: 0.6, vectorArmVoted: false, relationalQuery: false });
    expect(r.weight).toBeUndefined();
    expect(r.decision.downweighted).toBe(false);
  });

  test('empty keyword arm → never down-weighted (nothing to demote)', () => {
    const r = decideKeywordArmWeight({ keywordList: [], floor: 0.6, ...base });
    expect(r.weight).toBeUndefined();
    expect(r.decision).toEqual({ margin_ratio: 0, top_score: 0, downweighted: false });
  });

  test('the comparison is strict: margin == floor is NOT weak; floor 1 demotes every contested arm', () => {
    expect(decideKeywordArmWeight({ keywordList: weak, floor: 0.5, ...base }).weight).toBeUndefined();
    expect(decideKeywordArmWeight({ keywordList: weak, floor: 0.500001, ...base }).weight).toBe(0.5);
    expect(decideKeywordArmWeight({ keywordList: [row('a', 0.9), row('b', 0.01)], floor: 1, ...base }).weight).toBe(0.5);
    expect(decideKeywordArmWeight({ keywordList: strong, floor: 1, ...base }).weight).toBeUndefined();
  });
});

describe('composeFusionLists — arm-confidence weighting of the keyword + title entries', () => {
  const gold = row('notes/gold', 0, 1);
  const d1 = row('notes/decoy-1', 0.2, 2);
  const d2 = row('notes/decoy-2', 0.2, 3);
  const rel = row('people/investor', 0, 4);

  function arms(): VectorArm[] {
    const a: VectorArm[] = [];
    pushVectorList(a, [gold], 'original');
    return a;
  }

  function compose(over: {
    floor?: number | null;
    arms?: VectorArm[];
    keyword?: SearchResult[];
    relationalQuery?: boolean;
    onDecision?: (d: KeywordArmConfidenceDecision) => void;
  } = {}) {
    return composeFusionLists({
      arms: over.arms ?? arms(),
      keywordFusionList: over.keyword ?? [d1, d2],
      titleFusionList: [d1, d2],
      relationalList: [rel],
      includeRelational: true,
      relationalQuery: over.relationalQuery,
      onKeywordArmConfidence: over.onDecision,
      ks: KS,
      knobs: { expansionVariantBudget: null, keywordArmConfidenceFloor: over.floor },
    });
  }

  test('floor null / unset → byte-identical entries: no `weight` key anywhere', () => {
    for (const floor of [null, undefined]) {
      const lists = compose({ floor });
      // `arm` is the explain-only instance label; fusion never reads it.
      expect(lists).toEqual([
        { list: [gold], k: KS.vectorK, arm: 'vector' },
        { list: [d1, d2], k: KS.keywordK, arm: 'keyword' },
        { list: [d1, d2], k: KS.keywordK, arm: 'title' },
        { list: [rel], k: KS.baseRrfK, arm: 'relational' },
      ]);
      for (const e of lists) expect(e).not.toHaveProperty('weight');
    }
  });

  test('floor 0.6 + weak arm → keyword AND title entries weight 0.5; vector + relational untouched', () => {
    let decision: KeywordArmConfidenceDecision | undefined;
    const lists = compose({ floor: 0.6, onDecision: (d) => { decision = d; } });
    expect(lists).toEqual([
      { list: [gold], k: KS.vectorK, arm: 'vector' },
      { list: [d1, d2], k: KS.keywordK, weight: 0.5, arm: 'keyword' },
      { list: [d1, d2], k: KS.keywordK, weight: 0.5, arm: 'title' },
      { list: [rel], k: KS.baseRrfK, arm: 'relational' },
    ]);
    expect(lists[0]).not.toHaveProperty('weight');
    expect(lists[3]).not.toHaveProperty('weight');
    expect(decision).toEqual({ margin_ratio: 0.5, top_score: 0.2, downweighted: true });
  });

  test('strong arm (single keyword row) → unchanged; callback reports downweighted false', () => {
    let decision: KeywordArmConfidenceDecision | undefined;
    const lists = compose({ floor: 0.6, keyword: [d1], onDecision: (d) => { decision = d; } });
    expect(lists[1]).toEqual({ list: [d1], k: KS.keywordK, arm: 'keyword' });
    expect(lists[2]).toEqual({ list: [d1, d2], k: KS.keywordK, arm: 'title' });
    expect(decision).toEqual({ margin_ratio: 1, top_score: 0.2, downweighted: false });
  });

  test('relational query → unchanged (the relational arm owns that shape)', () => {
    let decision: KeywordArmConfidenceDecision | undefined;
    const lists = compose({ floor: 0.6, relationalQuery: true, onDecision: (d) => { decision = d; } });
    expect(lists).toEqual(compose({ floor: null }));
    expect(decision?.downweighted).toBe(false);
    expect(decision?.margin_ratio).toBe(0.5);
  });

  test('no text vector vote (keyword-only fallback shape / image-only arm) → unchanged', () => {
    const empty: VectorArm[] = [];
    pushVectorList(empty, [], 'original');
    expect(compose({ floor: 0.6, arms: empty })).toEqual(compose({ floor: null, arms: empty }));
    const imageOnly: VectorArm[] = [];
    pushVectorList(imageOnly, [gold], 'image');
    expect(compose({ floor: 0.6, arms: imageOnly })).toEqual(compose({ floor: null, arms: imageOnly }));
    // ...but a surviving expansion VARIANT is a real text vote.
    const variantOnly: VectorArm[] = [];
    pushVectorList(variantOnly, [], 'original');
    pushVectorList(variantOnly, [gold], 'variant');
    expect(compose({ floor: 0.6, arms: variantOnly })[2]).toEqual({ list: [d1, d2], k: KS.keywordK, weight: 0.5, arm: 'keyword' });
  });

  test('empty keyword list → unchanged; the empty title list is still omitted', () => {
    const lists = composeFusionLists({
      arms: arms(),
      keywordFusionList: [],
      titleFusionList: [],
      relationalList: [],
      includeRelational: true,
      ks: KS,
      knobs: { expansionVariantBudget: null, keywordArmConfidenceFloor: 0.6 },
    });
    expect(lists).toEqual([{ list: [gold], k: KS.vectorK, arm: 'vector' }, { list: [], k: KS.keywordK, arm: 'keyword' }]);
  });

  test('the callback fires exactly once per composition, even with the knob off; a throwing callback never breaks fusion', () => {
    let fired = 0;
    compose({ floor: null, onDecision: () => { fired++; } });
    compose({ floor: 0.6, onDecision: () => { fired++; } });
    expect(fired).toBe(2);
    expect(() => compose({ floor: 0.6, onDecision: () => { throw new Error('boom'); } })).not.toThrow();
  });

  test('end-to-end RRF: floor off → the lexical decoy wins top-1; floor 0.6 → gold wins (k 60 vec / 66 lexical)', () => {
    const off = rrfFusionWeighted(compose({ floor: null }), false);
    expect(off[0].slug).toBe('notes/decoy-1');
    const on = rrfFusionWeighted(compose({ floor: 0.6 }), false);
    expect(on[0].slug).toBe('notes/gold');
    // Decoy kept its votes at half weight: 0.5/66 + 0.5/66 < 1/60.
    const decoy = on.find((r) => r.slug === 'notes/decoy-1')!;
    expect(decoy.score).toBeLessThan(on[0].score);
  });
});

describe('keyword_arm_confidence_floor knob plane (bundle, parse, chain, hash, registry)', () => {
  test('every bundle lands at null (behavior-preserving; the Phase E2 receipt decides the flip)', () => {
    for (const m of SEARCH_MODES) {
      expect(MODE_BUNDLES[m].keyword_arm_confidence_floor).toBeNull();
    }
  });

  test('loadOverridesFromConfig: (0, 1] and the off/null literals parse; everything else falls through', () => {
    const via = (v: string) => loadOverridesFromConfig({ 'search.keyword_arm_confidence_floor': v });
    expect(via('0.6').keyword_arm_confidence_floor).toBe(0.6);
    expect(via('1').keyword_arm_confidence_floor).toBe(1);
    expect(via('off').keyword_arm_confidence_floor).toBeNull();
    expect(via('OFF').keyword_arm_confidence_floor).toBeNull();
    expect(via('null').keyword_arm_confidence_floor).toBeNull();
    expect(via('false').keyword_arm_confidence_floor).toBeNull();
    for (const bad of ['0', '1.5', '-1', 'x', '', 'NaN', 'Infinity', '0.6abc']) {
      expect(via(bad)).not.toHaveProperty('keyword_arm_confidence_floor');
    }
    expect(loadOverridesFromConfig({})).not.toHaveProperty('keyword_arm_confidence_floor');
  });

  test('the config parser IS the normalizer (one range contract, two seams)', () => {
    for (const v of ['0.6', '1', '0', '1.5', '-1', 'NaN', '', 'x', 'off', 'null', 'false', 'OFF']) {
      const expected = normalizeKeywordArmConfidenceFloor(v);
      const parsed = loadOverridesFromConfig({ 'search.keyword_arm_confidence_floor': v });
      if (expected === undefined) expect(parsed).not.toHaveProperty('keyword_arm_confidence_floor');
      else expect(parsed.keyword_arm_confidence_floor).toBe(expected as number | null);
    }
  });

  test('resolution chain: per-call > config override > bundle; a null override is honored, not skipped', () => {
    expect(resolveSearchMode({ mode: 'balanced' }).keyword_arm_confidence_floor).toBeNull();
    expect(resolveSearchMode({ mode: 'balanced', overrides: { keyword_arm_confidence_floor: 0.6 } }).keyword_arm_confidence_floor).toBe(0.6);
    expect(
      resolveSearchMode({ mode: 'balanced', overrides: { keyword_arm_confidence_floor: 0.6 }, perCall: { keyword_arm_confidence_floor: 0.3 } })
        .keyword_arm_confidence_floor,
    ).toBe(0.3);
    expect(
      resolveSearchMode({ mode: 'balanced', overrides: { keyword_arm_confidence_floor: 0.6 }, perCall: { keyword_arm_confidence_floor: null } })
        .keyword_arm_confidence_floor,
    ).toBeNull();
    const input = { mode: 'balanced', overrides: { keyword_arm_confidence_floor: null } } as const;
    expect(attributeKnob('keyword_arm_confidence_floor', input, resolveSearchMode(input)).source).toBe('override');
    // An out-of-range config value resolves to the bundle (null).
    expect(
      resolveSearchMode({ mode: 'balanced', overrides: loadOverridesFromConfig({ 'search.keyword_arm_confidence_floor': '1.5' }) })
        .keyword_arm_confidence_floor,
    ).toBeNull();
  });

  test('an invalid per-call value normalized to undefined falls through and hashes as off (never kacf=NaN)', () => {
    const off = resolveSearchMode({ mode: 'balanced' });
    for (const bad of [Number.NaN, 0, -1, 1.5, Number.POSITIVE_INFINITY]) {
      const r = resolveSearchMode({ mode: 'balanced', perCall: { keyword_arm_confidence_floor: normalizeKeywordArmConfidenceFloor(bad) } });
      expect(r.keyword_arm_confidence_floor).toBeNull();
      expect(knobsHash(r)).toBe(knobsHash(off));
    }
    const r = resolveSearchMode({
      mode: 'balanced',
      overrides: { keyword_arm_confidence_floor: 0.6 },
      perCall: { keyword_arm_confidence_floor: normalizeKeywordArmConfidenceFloor(Number.NaN) },
    });
    expect(r.keyword_arm_confidence_floor).toBe(0.6);
  });

  test('knobsHash folds the floor (kacf=): off vs 0.6 vs 0.5 all differ; null is stable; a partial literal hashes as off', () => {
    const base = resolveSearchMode({ mode: 'balanced' });
    const off = knobsHash(base);
    const offExplicit = knobsHash({ ...base, keyword_arm_confidence_floor: null });
    const six = knobsHash({ ...base, keyword_arm_confidence_floor: 0.6 });
    const five = knobsHash({ ...base, keyword_arm_confidence_floor: 0.5 });
    expect(off).toBe(offExplicit);
    expect(new Set([off, six, five]).size).toBe(3);
    const { keyword_arm_confidence_floor: _drop, ...partial } = base;
    expect(knobsHash(partial as ResolvedSearchKnobs)).toBe(off);
    // Same-value floors hash identically (toFixed(3) is the canonical form).
    expect(knobsHash({ ...base, keyword_arm_confidence_floor: 0.6 })).toBe(six);
  });

  test('registered everywhere the knob plane needs it (reset key, config registry, dashboard row + null label)', () => {
    expect(SEARCH_MODE_CONFIG_KEYS).toContain('search.keyword_arm_confidence_floor');
    expect(KNOWN_CONFIG_KEYS).toContain('search.keyword_arm_confidence_floor');
    expect(typeof KNOB_DESCRIPTIONS.keyword_arm_confidence_floor).toBe('string');
    expect(KNOB_NULL_LABELS.keyword_arm_confidence_floor).toBe('off (null)');
    expect(formatKnobValue('keyword_arm_confidence_floor', null)).toBe('off (null)');
    expect(formatKnobValue('keyword_arm_confidence_floor', 0.6)).toBe('0.6');
  });
});