Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.3 KB
Raw Blame
/**
 * W7 (wave 4b): the private forecast journal. Every active take that is not
 * a fact is a prediction; the journal gives each one its prediction time
 * (certified or not), exact claim, horizon (src/core/takes-horizon.ts),
 * state and resolution, and keeps the unresolved denominator. The grouped
 * scorecard (`takes_scorecard {group_by}`) reads the same predictions.
 *
 * Trusted local only: the CLI (`gbrain takes journal`, `gbrain takes
 * scorecard --group-by`) and the local op path read it; a remote caller that
 * asks for groups gets `scorecard_group_by_local_only`
 * (src/core/ops/takes.ts).
 *
 * Prediction time: when the belief ledger recorded the exact claim (claim,
 * kind, holder) on the page, either its first live `take.asserted` or, for a
 * take the page was created with, the creation snapshot (see
 * recordedPredictionTimes), certified only where the ledger's coverage
 * certifies that instant (a managed source with no drift or truncated purge
 * walk); without a recorded time, the take's since_date, never certified.
 */
import type { BrainEngine } from './engine.ts';
import { coverageAt, coverageIntervals } from './belief/coverage-report.ts';
import { takeClaimHash } from './belief/keys.ts';
import { gradeJudgeMeasured } from './cycle/grade-takes-measured.ts';
import { readTakeHorizons, type HorizonBasis } from './takes-horizon.ts';

export type JournalState = 'resolved' | 'overdue' | 'open' | 'undated';
export interface PredictionTime { at: string | null; basis: 'recorded' | 'since_date' | null; certified: boolean }
export interface JournalEntry {
  take_id: number; source_id: string; page_slug: string; row_num: number; claim: string; kind: string; holder: string; weight: number | null;
  prediction: PredictionTime;
  horizon: { date: string; basis: HorizonBasis; current: true } | null;
  state: JournalState;
  resolution: { quality: string; resolved_at: string | null; resolved_by: string | null; resolved_source: string | null } | null;
  /** The newest cached grade_takes verdict (W6), applied or not; judge_measured says whether the W6 eval measured its judge and prompt. */
  grade: { verdict: string; confidence: number; applied: boolean; judge_model_id: string; judge_measured: boolean; graded_at: string } | null;
}
export interface JournalTotals { predictions: number; resolved: number; unresolved: number; open: number; overdue: number; undated: number; certified_prediction_times: number }
export interface JournalQuery {
  sourceId?: string; sourceIds?: string[]; holder?: string; about?: string; since?: string; until?: string;
  state?: JournalState; asOf?: Date; limit?: number;
}
export interface ScorecardGroup {
  key: string; n: number; correct: number; incorrect: number; partial: number; unresolvable: number;
  accuracy: number | null; accuracy_shrunk: number | null; accuracy_wilson95: [number, number] | null; accuracy_recent: number | null;
  brier: number | null; brier_skill: number | null; citations: number;
  open: number; overdue: number; undated: number; unresolved: number; low_n: boolean;
}

/** Pseudo-observations of the Beta prior centred on the pooled accuracy. */
export const SCORECARD_PRIOR_STRENGTH = 10;
/** Half-life, in days since resolution, of the recency-weighted accuracy. */
export const SCORECARD_RECENCY_HALF_LIFE_DAYS = 365;
export const SCORECARD_DEFAULT_MIN_N = 10;

const iso = (v: string | Date | null) => v == null ? null : new Date(v).toISOString();

interface TakeRow {
  id: number | string; source_id: string; page_slug: string; row_num: number; claim: string; kind: string; holder: string; weight: number | string | null;
  since_date: string | null; resolved_quality: string | null; resolved_at: string | Date | null; resolved_by: string | null; resolved_source: string | null;
}

async function readPredictions(engine: BrainEngine, q: JournalQuery): Promise<TakeRow[]> {
  const sources = q.sourceIds?.length ? q.sourceIds : q.sourceId ? [q.sourceId] : null;
  return engine.executeRaw<TakeRow>(
    `SELECT t.id, p.source_id, p.slug AS page_slug, t.row_num, t.claim, t.kind, t.holder, t.weight, t.since_date,
            t.resolved_quality, t.resolved_at, t.resolved_by, t.resolved_source
       FROM takes t JOIN pages p ON p.id = t.page_id
      WHERE p.deleted_at IS NULL AND t.active AND t.kind <> 'fact'
        AND ($1::text[] IS NULL OR p.source_id = ANY($1::text[]))
        AND ($2::text IS NULL OR t.holder = $2)
        AND ($3::text IS NULL OR left(p.slug, length($3)) = $3)
        AND ($4::text IS NULL OR t.since_date >= $4)
        AND ($5::text IS NULL OR t.since_date <= $5)
      ORDER BY p.source_id, p.slug, t.row_num`,
    [sources, q.holder ?? null, q.about ?? null, q.since ?? null, q.until ?? null]);
}

interface LedgerRow { source_id: string; slug: string; event_type: string; subject_key: string | null; at: string | Date }

/**
 * Recorded prediction times from the belief ledger, keyed by (source, slug,
 * claim hash), over each page's current lifetime. A take added after its
 * page was created has a live `take.asserted`; its earliest is the time.
 * A take present when the page was created has no per-take creation event
 * (the create is one `page.published` snapshot, BELIEF_LEDGER.md section 4),
 * so it takes the snapshot's time, but only when coverage has run without a
 * stop, restore or drift mark on the page since then: otherwise the take
 * could have been added while the ledger was not watching. A take whose first
 * event is `take.transferred` came from another page and gets no recorded
 * time. Certified only where coverageAt certifies the instant.
 */
async function recordedPredictionTimes(engine: BrainEngine, rows: TakeRow[]): Promise<Map<string, PredictionTime>> {
  const out = new Map<string, PredictionTime>();
  if (!rows.length) return out;
  const sources = [...new Set(rows.map(r => r.source_id))];
  const events = await engine.executeRaw<LedgerRow>(
    `SELECT l.source_id, l.slug, e.event_type, e.subject_key, e.recorded_at AS at
       FROM belief_page_lifetimes l
       JOIN sources s ON s.id = l.source_id AND s.incarnation = l.source_incarnation
       JOIN belief_events e ON e.source_id = l.source_id AND e.source_incarnation = l.source_incarnation AND e.page_lifetime = l.page_lifetime
      WHERE l.ended_at IS NULL AND l.source_id = ANY($1::text[]) AND l.slug = ANY($2::text[]) AND e.origin <> 'backfill'
        AND (e.subject_kind = 'take' OR (e.event_type = 'page.published' AND e.value->>'revision_before' IS NULL))
      ORDER BY e.id`,
    [sources, [...new Set(rows.map(r => r.page_slug))]]);
  const created = new Map<string, string>();
  const firstTakeEvent = new Map<string, { type: string; at: string }>();
  for (const e of events) {
    const page = `${e.source_id}\u0000${e.slug}`;
    if (e.event_type === 'page.published') { if (!created.has(page)) created.set(page, iso(e.at)!); continue; }
    const key = `${page}\u0000${String((JSON.parse(e.subject_key!) as unknown[])[2])}`;
    if (!firstTakeEvent.has(key)) firstTakeEvent.set(key, { type: e.event_type, at: iso(e.at)! });
  }
  const intervals = new Map<string, Awaited<ReturnType<typeof coverageIntervals>>>();
  for (const id of sources) intervals.set(id, await coverageIntervals(engine, id));
  const watchedSince = (sourceId: string, slug: string, at: string) => {
    const open = intervals.get(sourceId)!.at(-1);
    return !!open && !open.stopped_at && open.started_at <= at
      && !open.restores.some(r => r.at > at) && !open.drift.some(d => d.slug === slug && d.to > at);
  };
  for (const r of rows) {
    const page = `${r.source_id}\u0000${r.page_slug}`;
    const key = `${page}\u0000${takeClaimHash(r.claim, r.kind, r.holder)}`;
    if (out.has(key)) continue;
    const first = firstTakeEvent.get(key);
    const pageCreated = created.get(page);
    const at = first?.type === 'take.asserted' ? first.at
      : first?.type === 'take.transferred' || !pageCreated || !watchedSince(r.source_id, r.page_slug, pageCreated) ? null
      : pageCreated;
    if (!at) continue;
    const coverage = await coverageAt(engine, r.source_id, at, r.page_slug);
    if (!coverage.ok) continue;
    out.set(key, { at, basis: 'recorded', certified: coverage.certified });
  }
  return out;
}

/** The journal: entries (filtered by `state`, then capped by `limit`) and totals over every prediction the other filters match. */
export async function readForecastJournal(engine: BrainEngine, q: JournalQuery = {}): Promise<{ as_of: string; entries: JournalEntry[]; totals: JournalTotals }> {
  const asOf = q.asOf ?? new Date();
  const today = asOf.toISOString().slice(0, 10);
  const rows = await readPredictions(engine, q);
  const horizons = await readTakeHorizons(engine, rows.map(r => Number(r.id)));
  const recorded = await recordedPredictionTimes(engine, rows);
  const grades = new Map((await engine.executeRaw<{ take_id: number | string; verdict: string; confidence: number | string; applied: boolean; judge_model_id: string; prompt_version: string; graded_at: string | Date }>(
    `SELECT DISTINCT ON (take_id) take_id, verdict, confidence, applied, judge_model_id, prompt_version, graded_at
       FROM take_grade_cache WHERE take_id = ANY($1::bigint[]) ORDER BY take_id, graded_at DESC`,
    [rows.map(r => Number(r.id))])).map(g => [Number(g.take_id), {
      verdict: g.verdict, confidence: Number(g.confidence), applied: g.applied, judge_model_id: g.judge_model_id,
      judge_measured: gradeJudgeMeasured(g.judge_model_id, g.prompt_version), graded_at: iso(g.graded_at)!,
    }]));
  const all: JournalEntry[] = rows.map(r => {
    const h = horizons.get(Number(r.id));
    const horizon = h?.current ? { date: h.horizon_date, basis: h.basis, current: true as const } : null;
    const state: JournalState = r.resolved_quality ? 'resolved' : !horizon ? 'undated' : horizon.date < today ? 'overdue' : 'open';
    const prediction = recorded.get(`${r.source_id}\u0000${r.page_slug}\u0000${takeClaimHash(r.claim, r.kind, r.holder)}`)
      ?? { at: r.since_date, basis: r.since_date ? 'since_date' as const : null, certified: false };
    return {
      take_id: Number(r.id), source_id: r.source_id, page_slug: r.page_slug, row_num: r.row_num, claim: r.claim, kind: r.kind, holder: r.holder,
      weight: r.weight == null ? null : Number(r.weight), prediction, horizon, state,
      resolution: r.resolved_quality ? { quality: r.resolved_quality, resolved_at: iso(r.resolved_at), resolved_by: r.resolved_by, resolved_source: r.resolved_source } : null,
      grade: grades.get(Number(r.id)) ?? null,
    };
  });
  const count = (s: JournalState) => all.filter(e => e.state === s).length;
  const totals: JournalTotals = {
    predictions: all.length, resolved: count('resolved'), unresolved: all.length - count('resolved'),
    open: count('open'), overdue: count('overdue'), undated: count('undated'),
    certified_prediction_times: all.filter(e => e.prediction.certified).length,
  };
  const filtered = q.state ? all.filter(e => e.state === q.state) : all;
  return { as_of: asOf.toISOString(), entries: q.limit === undefined ? filtered : filtered.slice(0, q.limit), totals };
}

function wilson95(k: number, n: number): [number, number] | null {
  if (n === 0) return null;
  const z = 1.959963984540054, p = k / n, d = 1 + z * z / n;
  const centre = (p + z * z / (2 * n)) / d;
  const half = z * Math.sqrt(p * (1 - p) / n + z * z / (4 * n * n)) / d;
  return [Math.max(0, centre - half), Math.min(1, centre + half)];
}

/**
 * Per holder or source, over the journal's predictions: accuracy on
 * correct/incorrect resolutions, shrunk toward the pooled accuracy with a
 * Beta prior of SCORECARD_PRIOR_STRENGTH pseudo-observations, its Wilson 95%
 * interval, recency-weighted accuracy, Brier and Brier skill (against the
 * group's own base rate), citations (resolutions naming a source), the open,
 * overdue and undated counts, and low_n when n < minN.
 */
export async function scorecardGroups(engine: BrainEngine, q: JournalQuery & { groupBy: 'holder' | 'source'; minN?: number }): Promise<ScorecardGroup[]> {
  const asOf = q.asOf ?? new Date();
  const { entries } = await readForecastJournal(engine, { ...q, state: undefined, limit: undefined, asOf });
  const byKey = new Map<string, JournalEntry[]>();
  for (const e of entries) {
    const key = q.groupBy === 'holder' ? e.holder : e.source_id;
    byKey.set(key, [...(byKey.get(key) ?? []), e]);
  }
  const scored = (e: JournalEntry) => e.resolution?.quality === 'correct' || e.resolution?.quality === 'incorrect';
  const pooledN = entries.filter(scored).length;
  const pooled = pooledN ? entries.filter(e => e.resolution?.quality === 'correct').length / pooledN : 0.5;
  const minN = q.minN ?? SCORECARD_DEFAULT_MIN_N;
  return [...byKey.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([key, group]) => {
    const resolved = group.filter(scored);
    const outcome = (e: JournalEntry) => e.resolution!.quality === 'correct' ? 1 : 0;
    const n = resolved.length;
    const correct = resolved.filter(e => outcome(e) === 1).length;
    const weighted = resolved.filter(e => e.weight != null);
    const brier = weighted.length ? weighted.reduce((s, e) => s + (e.weight! - outcome(e)) ** 2, 0) / weighted.length : null;
    const base = weighted.length ? weighted.filter(e => outcome(e) === 1).length / weighted.length : null;
    const ref = base === null ? 0 : base * (1 - base);
    const dated = resolved.filter(e => e.resolution!.resolved_at);
    const decay = (e: JournalEntry) => 0.5 ** (Math.max(0, asOf.getTime() - Date.parse(e.resolution!.resolved_at!)) / 86_400_000 / SCORECARD_RECENCY_HALF_LIFE_DAYS);
    const wSum = dated.reduce((s, e) => s + decay(e), 0);
    const count = (s: JournalState) => group.filter(e => e.state === s).length;
    return {
      key, n, correct, incorrect: n - correct,
      partial: group.filter(e => e.resolution?.quality === 'partial').length,
      unresolvable: group.filter(e => e.resolution?.quality === 'unresolvable').length,
      accuracy: n ? correct / n : null,
      accuracy_shrunk: (correct + SCORECARD_PRIOR_STRENGTH * pooled) / (n + SCORECARD_PRIOR_STRENGTH),
      accuracy_wilson95: wilson95(correct, n),
      accuracy_recent: wSum > 0 ? dated.reduce((s, e) => s + decay(e) * outcome(e), 0) / wSum : null,
      brier, brier_skill: brier !== null && ref > 0 ? 1 - brier / ref : null,
      citations: resolved.filter(e => e.resolution!.resolved_source?.trim()).length,
      open: count('open'), overdue: count('overdue'), undated: count('undated'), unresolved: group.length - count('resolved'),
      low_n: n < minN,
    };
  });
}