Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
12.5 KB
Raw Blame
/**
 * #6278 (B7, defect 2): a managed drain whose next admission is refused for
 * write capacity (`queue_capacity`, the writer's outstanding-request cap held
 * by another writer's queued requests on the same principal) waits instead of
 * dying. Before this change the refusal escaped `performManagedSync`, the run
 * recorded a managed-sync failure that named `--retry-failed`, and the CLI
 * exited 1 with no drain summary and no `next` (the Phase 4.1b lock run). Now
 * the drain retries with backoff while the other writer's requests settle,
 * prints `waiting for write capacity (N outstanding)`, and after its
 * no-progress window ends `blocked` / `write_capacity` with `next` naming
 * writer status; a cumulative cap (permanent request IDs, receipt bytes) is
 * still a refusal that needs a config change, never a wait. The forced probe
 * fills the sync writer's own outstanding counter to the cap, so the pass is
 * refused exactly where the bench died. Synthetic content only.
 */
import { afterAll, beforeAll, describe, expect, spyOn, test } from 'bun:test';
import { execFileSync } from 'node:child_process';
import { randomUUID } from 'node:crypto';
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import type { BrainEngine } from '../src/core/engine.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { OperationError } from '../src/core/ops/contract.ts';
import { claimWorktree } from '../src/core/persistence/ownership.ts';
import { disposePersistenceConsumer } from '../src/core/persistence/service.ts';
import { capacityError } from '../src/core/persistence/journal.ts';
import { isWriteCapacityWait } from '../src/core/persistence/admission-retry.ts';
import { drainManagedSync, drainJsonFields, drainNext, formatDrainSummary, runDrain } from '../src/core/persistence/sync-drain.ts';
import { ERROR_CATALOGUE } from '../src/core/error-catalogue.ts';
import type { SyncResult } from '../src/commands/sync.ts';
import { isolatedPersistencePostgres } from './helpers/persistence-postgres.ts';
import { testBackends } from './helpers/test-backends.ts';
import { withEnv } from './helpers/with-env.ts';

const base: SyncResult = { status: 'synced', fromCommit: 'a', toCommit: 'b', added: 0, modified: 0, deleted: 0, renamed: 0, chunksCreated: 0, embedded: 0, pagesAffected: [] };
const RESUME = 'gbrain sync --source s --no-pull';
const refused = () => capacityError('principal outstanding requests', { used: 99, limit: 100 });

function captureStderr(): { lines: string[]; waiting: () => string[]; restore(): void } {
  const lines: string[] = [];
  const spy = spyOn(console, 'error').mockImplementation((...args: unknown[]) => { lines.push(args.map(String).join(' ')); });
  return { lines, waiting: () => lines.filter(line => line.includes('waiting for write capacity')), restore: () => spy.mockRestore() };
}
const until = async (check: () => boolean, ms: number) => { const end = Date.now() + ms; while (!check() && Date.now() < end) await new Promise(r => setTimeout(r, 10)); return check(); };

describe('write-capacity refusals at admission', () => {
  test('the outstanding-request refusal carries its counts and is a wait; cumulative caps and other codes are not', () => {
    const error = refused();
    expect(error).toMatchObject({ code: 'queue_capacity', message: 'Write capacity exhausted: principal outstanding requests.', detail: 'outstanding=99 limit=100' });
    expect(isWriteCapacityWait(error)).toBe(true);
    expect(isWriteCapacityWait(capacityError('brain outstanding requests', { used: 1000, limit: 1000 }))).toBe(true);
    expect(isWriteCapacityWait(capacityError('principal intent bytes'))).toBe(false);
    const cumulative = new OperationError('queue_capacity', 'Write capacity exhausted: principal permanent request IDs (5 used of 5).', 'raise it');
    cumulative.detail = 'principalLifetimeIds';
    expect(isWriteCapacityWait(cumulative)).toBe(false);
    expect(isWriteCapacityWait(new OperationError('storage_error', 'x', 'y'))).toBe(false);
    expect(isWriteCapacityWait(new Error('queue_capacity'))).toBe(false);
  });

  test('a refused pass is retried with backoff until capacity returns, printing the wait; the drain then finishes synced', async () => {
    let passes = 0;
    const err = captureStderr();
    try {
      const result = await runDrain({ announce: true, progressMs: 20, backoffMs: 5, stallMs: 5000, pass: async () => { if (++passes <= 4) throw refused(); return { ...base, managedCursor: { index: 3, total: 3 } }; } });
      expect(result.drain).toMatchObject({ outcome: 'synced', passes: 5 });
      expect(result.drain!.stop_reason).toBeUndefined();
      expect(err.waiting().length).toBeGreaterThanOrEqual(1);
      expect(err.waiting()[0]).toMatch(/^\[sync\] \d+\/\? processed · waiting for write capacity \(99 outstanding of 100\)$/);
    } finally { err.restore(); }
  });

  test('capacity still full after the no-progress window: blocked / write_capacity with the wait on the report, writer status as next, and docs', async () => {
    let passes = 0;
    const result = await runDrain({ backoffMs: 5, stallMs: 150, pass: async () => { passes++; throw refused(); } });
    expect(passes).toBeGreaterThan(1);
    expect(result.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'write_capacity', capacity: { outstanding: 99, limit: 100, scope: 'principal' } });
    expect(result.drain!.capacity!.waited_seconds).toBeGreaterThanOrEqual(0);
    const next = drainNext(result, RESUME, 's')!;
    expect(next).toMatchObject({ command: 'gbrain sources writer status --source s --json', safe_to_loop: false, docs: ERROR_CATALOGUE.sync_drain_write_capacity.docs });
    expect(next.why).toContain('99 of the write principal\'s 100 outstanding-request slots');
    expect(next.why).toContain(RESUME);
    expect(next.why).toContain('persistence.limits.principal_outstanding');
    expect(ERROR_CATALOGUE.sync_drain_write_capacity).toEqual({ code: 'queue_capacity', docs: 'docs/guides/write-refusals.md#drain-write-capacity' });
    expect(drainJsonFields(result, RESUME, 's')).toMatchObject({ outcome: 'blocked', next: { command: 'gbrain sources writer status --source s --json' } });
    const summary = formatDrainSummary(result, RESUME, 's');
    expect(summary[0]).toMatch(/^Managed sync blocked: 0 entries this run/);
    expect(summary.join('\n')).toContain('Waited');
    expect(summary.join('\n')).toContain('Next: gbrain sources writer status --source s --json');
  });

  test('a pass that succeeded before the refusal keeps its result on the blocked report; a cumulative cap still ends the drain with the error', async () => {
    let passes = 0;
    const pendingResult: SyncResult = { ...base, status: 'partial', reason: 'writer_yield', added: 7, managedCursor: { index: 7, total: 20 } };
    const result = await runDrain({ backoffMs: 5, stallMs: 100, pass: async () => { if (++passes === 1) return pendingResult; throw refused(); } });
    expect(result).toMatchObject({ added: 7, drain: { outcome: 'blocked', stop_reason: 'write_capacity', remaining: 13 } });
    const cumulative = new OperationError('queue_capacity', 'Write capacity exhausted: principal permanent request IDs (5 used of 5).', 'raise it');
    cumulative.detail = 'principalLifetimeIds';
    await expect(runDrain({ backoffMs: 5, stallMs: 100, pass: async () => { throw cumulative; } })).rejects.toBe(cumulative);
  });
});

describe('forced probe: a real managed sync refused for write capacity', () => {
  const engines: BrainEngine[] = [];
  let closePostgres: (() => Promise<void>) | undefined;
  const home = mkdtempSync(join(tmpdir(), 'gbrain-drain-capacity-'));
  beforeAll(async () => {
    if (testBackends().includes('pglite')) { const lite = new PGLiteEngine(); await lite.connect({}); await lite.initSchema(); engines.push(lite); }
    if (testBackends().includes('postgres')) { const pg = await isolatedPersistencePostgres(process.env.DATABASE_URL!); engines.push(pg.engine); closePostgres = pg.close; }
  }, 120_000);
  afterAll(async () => {
    for (const engine of engines) { await disposePersistenceConsumer(engine); await engine.disconnect(); }
    await closePostgres?.(); rmSync(home, { recursive: true, force: true });
  });

  test('the cap held by other requests: the drain waits and prints, proceeds when they settle; held past the window it ends blocked with next, nothing in the failure ledger', () => withEnv({ GBRAIN_HOME: home, GBRAIN_SYNC_FAILURES_DIR: home }, async () => {
    for (const engine of engines) {
      const id = `cap-${randomUUID().replace(/-/g, '').slice(0, 20)}`, root = join(home, id);
      const git = (...args: string[]) => execFileSync('git', ['-C', root, ...args], { stdio: ['ignore', 'pipe', 'pipe'] });
      const write = (name: string) => writeFileSync(join(root, 'notes', `${name}.md`), `---\ntitle: ${name}\n---\nA synthetic observation about ${name}.\n`);
      const commit = (message: string) => { git('add', '-A'); git('-c', 'user.name=Example', '-c', '[email protected]', 'commit', '-qm', message); };
      mkdirSync(join(root, 'notes'), { recursive: true });
      git('init', '-q'); write('a'); commit('fixture');
      await engine.executeRaw('UPDATE persistence_brain SET enabled=false WHERE singleton=1');
      await engine.executeRaw("INSERT INTO sources(id,name,local_path,config) VALUES($1,$1,$2,'{}')", [id, root]);
      await claimWorktree(engine, id, root);
      await engine.executeRaw('UPDATE persistence_brain SET enabled=true WHERE singleton=1');
      const opts = { sourceId: id, noPull: true, noEmbed: true, noExtract: true };
      const ledger = () => engine.executeRaw("SELECT 1 FROM op_checkpoints WHERE op='managed-sync-failure' AND completed_keys::text LIKE $1", [`%${id}%`]);
      const outstanding = (n: number) => engine.executeRaw("UPDATE persistence_counters SET outstanding_count=$1 WHERE key LIKE 'principal:local_cli:%'", [n]);
      const err = captureStderr();
      try {
        // A first sync registers the writer's counter row.
        expect((await drainManagedSync(engine, opts, false)).drain).toMatchObject({ outcome: 'synced', written: 1 });
        const [principal] = await engine.executeRaw<{ key: string }>("SELECT key FROM persistence_counters WHERE key LIKE 'principal:local_cli:%'");
        expect(principal).toBeDefined();
        // Another writer on the same principal holds the whole cap (the bench: 89 queued adoption writes and one running against 100).
        await engine.setConfig('persistence.limits.principal_outstanding', '4');
        await outstanding(4);
        write('b'); write('c'); commit('two more');
        const drain = drainManagedSync(engine, { ...opts, drainStartedAt: Date.now() }, true);
        expect(await until(() => err.waiting().length > 0, 20_000)).toBe(true);
        expect(err.waiting()[0]).toMatch(/waiting for write capacity \(4 outstanding of 4\)$/);
        // The other writer's requests settle: the same drain proceeds.
        await outstanding(0);
        const result = await drain;
        expect(result.drain).toMatchObject({ outcome: 'synced', written: 2, remaining: 0 });
        expect(result.drain!.passes).toBeGreaterThan(1);
        expect(await engine.getPage('notes/c', { sourceId: id })).not.toBeNull();
        expect(await ledger()).toEqual([]);
        // Held past the window: one blocked stop with next, no uncaught error, no failure-ledger row, and the cursor intact for the rerun.
        await outstanding(4);
        write('d'); commit('one more');
        const blocked = await runDrain({ announce: false, stallMs: 300, backoffMs: 10, pass: (signal, onProgress) => import('../src/core/persistence/sync-run.ts')
          .then(m => m.performManagedSync(engine, { ...opts, signal, onProgress, drainStartedAt: Date.now() })) });
        expect(blocked.drain).toMatchObject({ outcome: 'blocked', stop_reason: 'write_capacity', capacity: { outstanding: 4, limit: 4, scope: 'principal' } });
        expect(drainNext(blocked, `gbrain sync --source ${id} --no-pull`, id)!.command).toBe(`gbrain sources writer status --source ${id} --json`);
        expect(await ledger()).toEqual([]);
        expect(await engine.getPage('notes/d', { sourceId: id })).toBeNull();
        await outstanding(0);
        expect((await drainManagedSync(engine, opts, false)).drain).toMatchObject({ outcome: 'synced', written: 1 });
        expect(await engine.getPage('notes/d', { sourceId: id })).not.toBeNull();
      } finally { err.restore(); await engine.unsetConfig('persistence.limits.principal_outstanding'); await disposePersistenceConsumer(engine); }
    }
  }), 180_000);
});