Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
14.2 KB
Raw Blame
/**
 * #6696 (A4): pushed capacity warnings read a per-engine snapshot (at most
 * one light forecast a minute); `write_capacity_low` fires at 80% or under the
 * warn days, at most once an hour per process per scope on receipts and every
 * time unthrottled; the pause notice fires when a scope passes the background
 * pause line and again only after it clears. Both backends.
 */
import { afterAll, beforeAll, beforeEach, describe, expect, test } from 'bun:test';
import { mkdtempSync, rmSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { __resetCapacityNoticesForTests, cachedCapacitySnapshot, CAPACITY_NOTICE_INTERVAL_MS, CAPACITY_SNAPSHOT_TTL_MS,
  capacityNotices } from '../src/core/persistence/capacity-snapshot.ts';
import { declarePersistenceProtocol } from '../src/core/persistence/protocol.ts';
import { renderCapacityMetrics } from '../src/commands/serve-http-metrics.ts';
import { NOTICE_CODES } from '../src/core/error-registry.ts';
import { renderNotice } from '../src/core/agent-output.ts';
import { readCapacityForecast } from '../src/core/persistence/capacity-forecast.ts';
import { oneYearCapacity, recommendedLimit } from '../src/core/persistence/limits.ts';
import { editRequest, freshBrainFactory, requestFixture } from './helpers/persistence-request-fixture.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';

const databaseUrl = process.env.DATABASE_URL;
const setIds = (engine: BrainEngine, used: number) => engine.transaction(async tx => {
  await declarePersistenceProtocol(tx);
  await tx.executeRaw("UPDATE persistence_counters SET lifetime_ids=$1 WHERE key LIKE 'principal:%'", [used]);
});
const codes = (notices: Array<{ code: string }>) => notices.map(n => n.code);

test('the notice kinds follow the mute convention: low is muteable info, the pause is degraded', () => {
  expect(NOTICE_CODES.write_capacity_low.kind).toBe('info');
  expect(NOTICE_CODES.write_capacity_backpressure.kind).toBe('degraded');
});

for (const kind of testBackends()) {
  describe(`capacity notices ${kind}`, () => {
    let factory: Awaited<ReturnType<typeof freshBrainFactory>>;
    let scratch: string;
    beforeAll(async () => {
      scratch = mkdtempSync(join(tmpdir(), 'gbrain-capacity-notices-'));
      factory = await freshBrainFactory(kind, databaseUrl);
    }, 120_000);
    afterAll(async () => {
      await factory?.dispose();
      if (scratch) rmSync(scratch, { recursive: true, force: true });
    });
    beforeEach(() => __resetCapacityNoticesForTests());
    const check = (name: string, fn: (engine: BrainEngine) => Promise<void>) => test(name, async () => {
      const engine = await factory.fresh();
      await withEnv({ GBRAIN_HOME: scratch, DATABASE_URL: undefined, GBRAIN_DATABASE_URL: undefined }, () => fn(engine));
    }, 120_000);

    check('a fresh brain carries no capacity notice', async engine => {
      await requestFixture(engine).then(r => r.admit('one'));
      expect(await capacityNotices(engine, { throttle: false })).toEqual([]);
    });

    check('at 80% the receipt notice shows once an hour per scope; unthrottled outputs show it every time', async engine => {
      await requestFixture(engine).then(r => r.admit('one'));
      await engine.setConfig('persistence.limits.principal_lifetime_ids', '1000');
      await engine.setConfig('persistence.capacity.background_pause_pct', '1');
      await setIds(engine, 810);
      const t0 = Date.now() + 100 * CAPACITY_SNAPSHOT_TTL_MS; // past any snapshot an earlier case left on a reused engine
      const first = await capacityNotices(engine, { throttle: true, now: t0 });
      expect(codes(first)).toEqual(['write_capacity_low']);
      // GST-2107: the notice rides a write that succeeded, so its last lines name a read-only check, never a cap raise for the user.
      expect(first[0]).toMatchObject({ kind: 'info', fix: { actor: 'agent', argv: ['gbrain', 'doctor', '--only', 'persistence_capacity', '--json'] } });
      expect(first[0]!.why).toContain('81%');
      expect(first[0]!.why).toContain('This notice is not a refusal');
      expect(first[0]!.why).not.toContain('config set');
      // A remote agent reads the rendered notice: over MCP its fix is the read-scope get_brain_health call, never a step to relay.
      for (const transport of ['http', 'stdio'] as const) {
        const fix = renderNotice(first[0]!, { transport, isCallable: tool => tool === 'get_brain_health', preapproved: () => true }).fix;
        expect(fix).toMatchObject({ actor: 'agent', next: 'run', mcp: { tool: 'get_brain_health' } });
      }
      expect(await capacityNotices(engine, { throttle: true, now: t0 + 1000 })).toEqual([]);
      expect(codes(await capacityNotices(engine, { throttle: false, now: t0 + 1000 }))).toEqual(['write_capacity_low']);
      expect(codes(await capacityNotices(engine, { throttle: true, now: t0 + CAPACITY_NOTICE_INTERVAL_MS }))).toEqual(['write_capacity_low']);
    });

    check('the pause notice fires when the pause starts, and again only after it clears', async engine => {
      await requestFixture(engine).then(r => r.admit('one'));
      await engine.setConfig('persistence.limits.principal_lifetime_ids', '1000');
      await engine.setConfig('persistence.capacity.warn_days', '0'); await engine.setConfig('persistence.capacity.fail_days', '0');
      let now = 2_000_000_000_000;
      const step = () => (now += CAPACITY_SNAPSHOT_TTL_MS);
      await setIds(engine, 500);
      expect(await capacityNotices(engine, { throttle: true, now: step() })).toEqual([]);
      await setIds(engine, 820);
      const started = await capacityNotices(engine, { throttle: true, now: step() });
      expect(codes(started)).toEqual(['write_capacity_low', 'write_capacity_backpressure']);
      expect(started[1]).toMatchObject({ kind: 'degraded' });
      expect(started[1]!.why).toContain('80% pause line');
      expect(codes(await capacityNotices(engine, { throttle: false, now: step() }))).toEqual(['write_capacity_low']);
      await setIds(engine, 500);
      expect(await capacityNotices(engine, { throttle: false, now: step() })).toEqual([]);
      await setIds(engine, 820);
      expect(codes(await capacityNotices(engine, { throttle: false, now: step() }))).toContain('write_capacity_backpressure');
    });

    check('a remote caller sees only the brain and its own principal, and a hidden scope is not marked shown', async engine => {
      await requestFixture(engine).then(r => r.admit('one'));
      await engine.setConfig('persistence.limits.principal_lifetime_ids', '1000');
      await setIds(engine, 900);
      const now = 2_500_000_000_000; // between the cases around it, so neither reads the other's snapshot
      expect(await capacityNotices(engine, { throttle: true, now, scopes: ['brain', 'principal:oauth_client:other-example'] })).toEqual([]);
      const own = await capacityNotices(engine, { throttle: true, now });
      expect(codes(own)).toEqual(['write_capacity_low', 'write_capacity_backpressure']);
      expect(own[0]!.why).toMatch(/principal:local_cli:/);
    });

    check('the one-year value follows the 7-day mean, not a burst in the last day (GST-2107)', async engine => {
      const fixture = await requestFixture(engine);
      const ids: string[] = [];
      for (let i = 0; i < 71; i++) ids.push((await fixture.admit(`burst-${i}`)).id);
      // One admission before the window (so it spans a full 7 days), 42 spread over the six days before the last one and 28 in
      // the last hour: a mean of 10 a day, 28 in the last day.
      const ageDays = (i: number) => i === 0 ? 10 : 6.5 - Math.floor((i - 1) / 7);
      for (let i = 0; i < 43; i++) await editRequest(engine, ids[i]!, "created_at=now()-($2::double precision*interval '1 day')", [ageDays(i)]);
      await engine.setConfig('persistence.limits.principal_lifetime_ids', '100');
      await engine.setConfig('persistence.limits.principal_terminal_bytes', '1');
      const forecast = await readCapacityForecast(engine);
      const row = (resource: string) => forecast.rows.find(r => r.scope.startsWith('principal:') && r.resource === resource)!;
      expect(row('lifetime_ids').per_day_7d).toBe(28);
      expect(row('lifetime_ids').recommended_limit).toBe(71 + 10 * 365);
      const [{ gross }] = await engine.executeRaw<{ gross: string }>('SELECT SUM(terminal_reservation)::text AS gross FROM persistence_requests');
      const used = row('terminal_bytes').used;
      const ceiling = recommendedLimit('terminal_bytes', used, 1, { perDay: Number(gross) / 7, netPerDay: Number(gross) / 7 }, forecast.retention_days);
      expect(row('terminal_bytes').recommended_limit).toBeLessThanOrEqual(ceiling);
    });

    check('after a burst the admission refusal names the same one-year value as doctor (GST-2107)', async engine => {
      await engine.setConfig('persistence.limits.principal_outstanding', '5000');
      await engine.setConfig('persistence.limits.brain_outstanding', '5000');
      const fixture = await requestFixture(engine);
      const ids: string[] = [];
      for (let i = 0; i < 1101; i++) ids.push((await fixture.admit(`dense-${i}`)).id);
      // One admission before the window, 100 over the week before the last day, then 1,000 in the last hour: the latest
      // 1,000 admissions are all burst, while the 7-day mean is 1,100 / 7 a day.
      await engine.transaction(async tx => {
        await declarePersistenceProtocol(tx);
        await tx.executeRaw("UPDATE persistence_requests SET created_at=now()-interval '10 days' WHERE id=$1::uuid", [ids[0]]);
        await tx.executeRaw("UPDATE persistence_requests SET created_at=now()-interval '2 days' WHERE id=ANY($1::uuid[])", [ids.slice(1, 101)]);
      });
      await engine.setConfig('persistence.limits.principal_lifetime_ids', '1101');
      await engine.setConfig('persistence.limits.principal_terminal_bytes', String(16 * 1024 ** 3));
      // The forecast a receipt, doctor or /metrics read in the last minute records the rates the refusal uses under its locks.
      const forecast = await readCapacityForecast(engine);
      const refusal = await fixture.admit('one-more').then(() => null, (error: { code?: string; fix?: { argv?: string[] } }) => error);
      expect(refusal).toMatchObject({ code: 'write_quota_exhausted', reason: 'lifetime_ids' });
      const doctor = forecast.rows.find(r => r.scope.startsWith('principal:') && r.resource === 'lifetime_ids')!;
      expect(doctor.recommended_limit).toBe(Math.ceil(1101 + (1100 / 7) * 365));
      expect(Number(refusal!.fix!.argv!.at(-1))).toBe(doctor.recommended_limit);
      const principal = forecast.rows.find(r => r.scope.startsWith('principal:'))!.scope;
      const terminal = forecast.rows.find(r => r.scope === principal && r.resource === 'terminal_bytes')!;
      expect(await oneYearCapacity(engine, principal, 'TerminalBytes', terminal.used, terminal.limit)).toBe(terminal.recommended_limit);
    });

    check('the one-year value counts the week\'s lifecycle changes, as doctor does (GST-2107)', async engine => {
      const fixture = await requestFixture(engine);
      for (let i = 0; i < 14; i++) await fixture.admit(`week-${i}`);
      await engine.transaction(async tx => {
        await declarePersistenceProtocol(tx);
        await tx.executeRaw(`INSERT INTO persistence_topology_changes(principal_id,request_id,digest,operation,source_id,state,terminal_bytes)
          SELECT gen_random_uuid(),gen_random_uuid(),'d','source_add','source-a','committed',2048 FROM generate_series(1,56)`);
      });
      await engine.setConfig('persistence.limits.brain_lifetime_ids', '10');
      await engine.setConfig('persistence.limits.brain_terminal_bytes', '1');
      const forecast = await readCapacityForecast(engine);
      for (const [resource, setting] of [['lifetime_ids', 'LifetimeIds'], ['terminal_bytes', 'TerminalBytes']] as const) {
        const doctor = forecast.rows.find(r => r.scope === 'brain' && r.resource === resource)!;
        expect(await oneYearCapacity(engine, 'brain', setting, doctor.used, doctor.limit)).toBe(doctor.recommended_limit);
      }
    });

    check('a refusal under the counter locks reads at most the latest 1,000 admissions, and none after a forecast recorded the rates', async engine => {
      const fixture = await requestFixture(engine);
      for (let i = 0; i < 3; i++) await fixture.admit(`rate-${i}`);
      await engine.setConfig('persistence.limits.brain_lifetime_ids', '10');
      const reads: string[] = [];
      const counted = { ...engine, executeRaw: (sql: string, params?: unknown[]) => {
        if (/FROM persistence_requests/.test(sql)) reads.push(sql);
        return engine.executeRaw(sql, params);
      } } as unknown as BrainEngine;
      await oneYearCapacity(counted, 'brain', 'LifetimeIds', 3, 10);
      expect(reads.some(sql => sql.includes('LIMIT 1000'))).toBe(true);
      expect(reads.some(sql => /LIMIT (?!1\b|1000\b)\d+/.test(sql))).toBe(false);
      const doctor = (await readCapacityForecast(counted)).rows.find(r => r.scope === 'brain' && r.resource === 'lifetime_ids')!;
      reads.length = 0;
      expect(await oneYearCapacity(counted, 'brain', 'LifetimeIds', doctor.used, doctor.limit)).toBe(doctor.recommended_limit);
      expect(reads.some(sql => sql.includes('LIMIT 1000'))).toBe(false);
    });

    check('writes within a minute share one snapshot; the next minute reads again', async engine => {
      await requestFixture(engine).then(r => r.admit('one'));
      await engine.setConfig('persistence.limits.principal_lifetime_ids', '1000');
      const t0 = 3_000_000_000_000;
      const before = await cachedCapacitySnapshot(engine, t0);
      expect(before.low).toEqual([]);
      await setIds(engine, 900);
      expect(await cachedCapacitySnapshot(engine, t0 + CAPACITY_SNAPSHOT_TTL_MS - 1)).toBe(before);
      const after = await cachedCapacitySnapshot(engine, t0 + CAPACITY_SNAPSHOT_TTL_MS);
      expect(after.low.map(row => row.resource)).toEqual(['lifetime_ids']);
      const metrics = renderCapacityMetrics(after);
      expect(metrics).toContain('# TYPE gbrain_write_capacity_used gauge');
      expect(metrics).toMatch(/gbrain_write_capacity_used\{scope="principal:[^"]+",resource="lifetime_ids"\} 900/);
      expect(metrics).toMatch(/gbrain_write_capacity_limit\{scope="principal:[^"]+",resource="lifetime_ids"\} 1000/);
    });
  });
}