Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.7 KB
Raw Blame
/**
 * Use-attributed retrieval feedback storage and rating on PGLite: moving-average
 * math, per-element idempotency, receipts, revision handling, authority and
 * the rate_answer refusals.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import type { OperationContext } from '../src/core/ops/contract.ts';
import { insertRetrievalEvents, readWeights } from '../src/core/feedback/store.ts';
import { rateAnswer } from '../src/core/feedback/rate.ts';
import {
  _resetFeedbackRecordingForTests, answerIdTime, canTeachSource, drainFeedbackQueue, droppedFeedbackEvents, isAnswerPending,
  mintAnswerId, QUEUE_CAP, recordAnswer,
} from '../src/core/feedback/record.ts';
import { __listDrainerNamesForTest } from '../src/core/background-work.ts';
import { _resetFeedbackSettingsCacheForTests } from '../src/core/feedback/settings.ts';
import { retrievalFeedbackUpgradeNotice } from '../src/core/feedback/upgrade-notice.ts';

let engine: PGLiteEngine;

function localCtx(): OperationContext {
  return { engine, config: {} as OperationContext['config'], remote: false, logger: console } as unknown as OperationContext;
}

async function hashOf(slug: string, source = 'default'): Promise<string> {
  const rows = await engine.executeRaw<{ content_hash: string }>(
    'SELECT content_hash FROM pages WHERE slug = $1 AND source_id = $2', [slug, source]);
  return rows[0]!.content_hash;
}

async function seedEvent(id: string, slugs: string[], opts: { op?: 'query' | 'think'; cited?: string[]; clientId?: string } = {}) {
  const pages = [];
  for (const [rank, slug] of slugs.entries()) {
    pages.push({ source_id: 'default', slug, content_hash: await hashOf(slug), rank, cited: (opts.cited ?? []).includes(slug) });
  }
  await insertRetrievalEvents(engine, [{
    id, client_id: opts.clientId ?? 'local', op: opts.op ?? 'query', pages,
    links: [{ source_id: 'default', edge_key: 'people/alice-example|works_at|companies/acme-example', to_slug: 'companies/acme-example' }],
  }]);
}

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  for (const slug of ['people/alice-example', 'companies/acme-example', 'notes/meeting-example']) {
    await engine.putPage(slug, { type: slug.startsWith('people') ? 'person' : 'note', title: slug, compiled_truth: `about ${slug}` });
  }
}, 60_000);

afterAll(async () => {
  await engine.disconnect();
});

beforeEach(async () => {
  await engine.executeRaw('DELETE FROM retrieval_weights');
  await engine.executeRaw('DELETE FROM retrieval_events');
  await engine.executeRaw('DELETE FROM config WHERE key LIKE $1', ['feedback.%']);
  await engine.setConfig('feedback.enabled', 'true');
  await engine.setConfig('feedback.implicit', 'true');
  _resetFeedbackSettingsCacheForTests();
  _resetFeedbackRecordingForTests();
});

describe('defaults', () => {
  test('a brain that never set feedback.enabled records nothing and returns no answer id', async () => {
    await engine.executeRaw('DELETE FROM config WHERE key LIKE $1', ['feedback.%']);
    _resetFeedbackSettingsCacheForTests();
    expect(await recordAnswer(localCtx(), { op: 'query', pages: [{ slug: 'people/alice-example' }] })).toBeNull();
    const notice = await retrievalFeedbackUpgradeNotice(engine);
    expect(notice?.join('\n')).toContain('gbrain config set feedback.enabled true');
    await engine.setConfig('feedback.enabled', 'false');
    expect(await retrievalFeedbackUpgradeNotice(engine)).toBeNull();
  });
});

describe('answer ids', () => {
  test('mint and decode the timestamp', () => {
    const now = 1_790_000_000_000;
    const id = mintAnswerId(now);
    expect(id).toMatch(/^ans_[0-9A-HJKMNP-TV-Z]{26}$/);
    expect(answerIdTime(id)).toBe(now);
    expect(answerIdTime('ans_nope')).toBeNull();
  });
});

describe('rate_answer', () => {
  test('whole-answer rating moves each page and the path edge by the moving average', async () => {
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['people/alice-example', 'companies/acme-example']);
    const receipt = await rateAnswer(localCtx(), { answer_id: id, rating: 5 });
    expect(receipt.applied.map(a => a.ref).sort()).toEqual([
      'default:companies/acme-example', 'default:people/alice-example',
      'default:people/alice-example|works_at|companies/acme-example',
    ]);
    for (const a of receipt.applied) {
      expect(a.weight_before).toBe(0.5);
      expect(a.weight_after).toBeCloseTo(0.55, 4);
      expect(a.multiplier).toBeCloseTo(1.01, 4);
    }
    const weights = await readWeights(engine, 'page', [{ source_id: 'default', key: 'people/alice-example' }]);
    expect(weights.get('default:people/alice-example')).toBeCloseTo(0.55, 4);
  });

  test('identical retry returns the same receipt and does not compound', async () => {
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['people/alice-example']);
    const first = await rateAnswer(localCtx(), { answer_id: id, rating: 1 });
    const second = await rateAnswer(localCtx(), { answer_id: id, rating: 1 });
    expect(second.applied).toEqual(first.applied);
    const weights = await readWeights(engine, 'page', [{ source_id: 'default', key: 'people/alice-example' }]);
    expect(weights.get('default:people/alice-example')).toBeCloseTo(0.45, 4);
  });

  test('targeted ratings touch only the named pages and can differ per page', async () => {
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['people/alice-example', 'companies/acme-example']);
    const receipt = await rateAnswer(localCtx(), {
      answer_id: id,
      pages: [{ ref: 'default:people/alice-example', rating: 1 }, { ref: 'companies/acme-example', rating: 5 }],
    });
    expect(receipt.applied.map(a => [a.ref, a.weight_after])).toEqual(expect.arrayContaining([
      ['default:people/alice-example', 0.45], ['default:companies/acme-example', 0.55],
    ]));
    expect(receipt.applied.some(a => a.kind === 'link')).toBe(false);
    const again = await rateAnswer(localCtx(), { answer_id: id, pages: [{ ref: 'default:people/alice-example', rating: 5 }] });
    expect(again.applied).toEqual([]);
    expect(again.skipped).toEqual([{ kind: 'page', ref: 'default:people/alice-example', reason: 'already_rated', rating: 1 }]);
  });

  test('a think answer rates its cited pages only', async () => {
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['people/alice-example', 'notes/meeting-example'], { op: 'think', cited: ['notes/meeting-example'] });
    const receipt = await rateAnswer(localCtx(), { answer_id: id, rating: 5 });
    expect(receipt.applied.filter(a => a.kind === 'page').map(a => a.ref)).toEqual(['default:notes/meeting-example']);
  });

  test('a page edited after the answer is skipped as stale_revision', async () => {
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['notes/meeting-example']);
    await engine.putPage('notes/meeting-example', { type: 'note', title: 'notes/meeting-example', compiled_truth: 'corrected text' });
    const receipt = await rateAnswer(localCtx(), { answer_id: id, rating: 1 });
    expect(receipt.applied).toEqual([]);
    expect(receipt.skipped[0]).toEqual({ kind: 'page', ref: 'default:notes/meeting-example', reason: 'stale_revision' });
  });

  test('an edited page reads at half its learned deviation', async () => {
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['companies/acme-example']);
    await rateAnswer(localCtx(), { answer_id: id, rating: 5 });
    await engine.putPage('companies/acme-example', { type: 'note', title: 'companies/acme-example', compiled_truth: 'rewritten' });
    const weights = await readWeights(engine, 'page', [{ source_id: 'default', key: 'companies/acme-example' }]);
    expect(weights.get('default:companies/acme-example')).toBeCloseTo(0.525, 4);
  });

  test('refusals carry their codes', async () => {
    const ctx = localCtx();
    const id = mintAnswerId(Date.now() - 120_000);
    await seedEvent(id, ['people/alice-example', 'companies/acme-example']);
    await expect(rateAnswer(ctx, { answer_id: id, rating: 9 })).rejects.toMatchObject({ code: 'invalid_rating' });
    await expect(rateAnswer(ctx, { answer_id: id, pages: [{ ref: 'default:nope', rating: 3 }] })).rejects.toMatchObject({ code: 'ref_not_in_answer' });
    await expect(rateAnswer(ctx, { answer_id: mintAnswerId(Date.now() - 120_000), rating: 3 })).rejects.toMatchObject({ code: 'answer_unavailable' });
    await expect(rateAnswer(ctx, { answer_id: mintAnswerId(), rating: 3 })).rejects.toMatchObject({ code: 'answer_pending' });
    const other = mintAnswerId(Date.now() - 120_000);
    await seedEvent(other, ['people/alice-example'], { clientId: 'client-b' });
    await expect(rateAnswer(ctx, { answer_id: other, rating: 3 })).rejects.toMatchObject({ code: 'answer_not_yours' });
    await engine.setConfig('feedback.enabled', 'false');
    _resetFeedbackSettingsCacheForTests();
    await expect(rateAnswer(ctx, { answer_id: id, rating: 3 })).rejects.toMatchObject({ code: 'feedback_disabled' });
  });
});

describe('authority', () => {
  const base = { remote: true, transport: 'http' as const, sourceId: 'default' };
  test('owner CLI and stdio may teach; grants must be unrestricted write on that source', () => {
    expect(canTeachSource({ remote: false, sourceId: 'default' }, 'default')).toBe(true);
    expect(canTeachSource({ remote: true, transport: 'stdio', sourceId: 'default' }, 'default')).toBe(true);
    expect(canTeachSource({ ...base, auth: { clientId: 'c', scopes: ['read'] } as never }, 'default')).toBe(false);
    expect(canTeachSource({ ...base, auth: { clientId: 'c', scopes: ['write'] } as never }, 'default')).toBe(true);
    expect(canTeachSource({ ...base, auth: { clientId: 'c', scopes: ['write'] } as never }, 'other')).toBe(false);
    expect(canTeachSource({ ...base, auth: { clientId: 'c', scopes: ['write'], boundSlugPrefixes: ['notes/'] } as never }, 'default')).toBe(false);
    expect(canTeachSource({ ...base, auth: { clientId: 'c', scopes: ['write'], allowedOperations: ['query'] } as never }, 'default')).toBe(false);
    expect(canTeachSource({ ...base, viaSubagent: true, auth: { clientId: 'c', scopes: ['write'] } as never }, 'default')).toBe(false);
  });
});

describe('recordAnswer', () => {
  test('records through the queue, applies the citation signal, and returns a rateable id', async () => {
    const meta = await recordAnswer(localCtx(), {
      op: 'think',
      pages: [
        { source_id: 'default', slug: 'notes/meeting-example', content_hash: await hashOf('notes/meeting-example'), cited: true },
        { source_id: 'default', slug: 'people/alice-example', content_hash: await hashOf('people/alice-example') },
      ],
    });
    expect(meta?.feedback.rateable).toBe(true);
    expect(meta?.answer_id).toMatch(/^ans_/);
    expect(meta?.feedback.how_to_rate).toContain('rate_answer');
    await drainFeedbackQueue(5000);
    const weights = await readWeights(engine, 'page', [
      { source_id: 'default', key: 'notes/meeting-example' }, { source_id: 'default', key: 'people/alice-example' },
    ]);
    expect(weights.get('default:notes/meeting-example')).toBeCloseTo(0.5125, 4);
    expect(weights.has('default:people/alice-example')).toBe(false);
  });

  test('readers and disabled brains record nothing', async () => {
    const reader = { ...localCtx(), remote: true, transport: 'http', auth: { clientId: 'r', scopes: ['read'] } } as unknown as OperationContext;
    const meta = await recordAnswer(reader, { op: 'query', pages: [{ source_id: 'default', slug: 'people/alice-example' }] });
    expect(meta).toEqual({ feedback: { rateable: false, reason: 'not_authorized' } });
    expect(await recordAnswer(localCtx(), { op: 'query', pages: [] })).toEqual({ feedback: { rateable: false, reason: 'empty' } });
    await engine.setConfig('feedback.enabled', 'false');
    _resetFeedbackSettingsCacheForTests();
    expect(await recordAnswer(localCtx(), { op: 'query', pages: [{ slug: 'people/alice-example' }] })).toBeNull();
    await drainFeedbackQueue(5000);
    const rows = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM retrieval_events');
    expect(rows[0]!.n).toBe(0);
  });
});

describe('write-behind queue', () => {
  async function withBlockedWrites(run: (release: () => void) => Promise<void>): Promise<void> {
    const original = engine.executeRaw.bind(engine);
    let release!: () => void;
    const gate = new Promise<void>(resolve => { release = resolve; });
    (engine as unknown as { executeRaw: typeof original }).executeRaw = (async (sql: string, params?: unknown[]) => {
      if (sql.includes('INSERT INTO retrieval_events')) await gate;
      return original(sql, params);
    }) as typeof original;
    try {
      await run(release);
    } finally {
      release();
      (engine as unknown as { executeRaw: typeof original }).executeRaw = original;
    }
  }

  test('a rating that races its answer gets answer_pending, then succeeds once the write lands', async () => {
    await withBlockedWrites(async (release) => {
      const meta = await recordAnswer(localCtx(), { op: 'query', pages: [{ source_id: 'default', slug: 'people/alice-example', content_hash: await hashOf('people/alice-example') }] });
      const id = meta!.answer_id!;
      expect(isAnswerPending(id)).toBe(true);
      await expect(rateAnswer(localCtx(), { answer_id: id, rating: 5 })).rejects.toMatchObject({ code: 'answer_pending' });
      expect(await drainFeedbackQueue(50)).toEqual({ unfinished: 1 });
      release();
      expect(await drainFeedbackQueue(5000)).toEqual({ unfinished: 0 });
      expect(isAnswerPending(id)).toBe(false);
      const receipt = await rateAnswer(localCtx(), { answer_id: id, rating: 5 });
      expect(receipt).toBeTruthy();
    });
  });

  test('overflow drops the oldest answers, counts them, and never blocks the caller', async () => {
    await withBlockedWrites(async (release) => {
      const ids: string[] = [];
      for (let i = 0; i < QUEUE_CAP + 5; i++) {
        const meta = await recordAnswer(localCtx(), { op: 'query', pages: [{ source_id: 'default', slug: 'notes/meeting-example' }] });
        ids.push(meta!.answer_id!);
      }
      expect(droppedFeedbackEvents()).toBe(4);
      expect(isAnswerPending(ids[1]!)).toBe(false);
      expect(isAnswerPending(ids.at(-1)!)).toBe(true);
      expect((await drainFeedbackQueue(50)).unfinished).toBe(QUEUE_CAP + 1);
      release();
      expect(await drainFeedbackQueue(10_000)).toEqual({ unfinished: 0 });
    });
    const rows = await engine.executeRaw<{ n: number }>('SELECT count(*)::int AS n FROM retrieval_events');
    expect(rows[0]!.n).toBe(QUEUE_CAP + 1);
  });

  test('the queue drains on CLI exit and before disconnect', () => {
    expect(__listDrainerNamesForTest()).toContain('retrieval-feedback');
  });
});