Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
8.6 KB
Raw Blame
/**
 * The resident consumer's stderr line is the only trace an operator gets when
 * a phase fails or overruns. #5233: the line carried only an error code, so a
 * pooler's "max clients reached" was invisible. It now carries a redacted,
 * one-line, length-capped message, a fix command and a docs anchor. #5801: it
 * also says whether the phase ever obtained a connection (first_conn_ms, or
 * checkout=not_observed with the time spent waiting) and the event-loop lag.
 *
 * Regression that fails it: dropping the message (or the redaction) from the
 * line, reporting a checkout that happened after the deadline or in another
 * caller's async chain, or omitting the timing fields. Existing coverage
 * asserts consumer state, not this line.
 */
import { afterEach, describe, expect, spyOn, test } from 'bun:test';
import { PersistenceConsumer } from '../src/core/persistence/consumer.ts';
import type { BrainEngine } from '../src/core/engine.ts';

const stderr: string[] = [];
let spy: ReturnType<typeof spyOn> | undefined;
afterEach(() => { spy?.mockRestore(); spy = undefined; stderr.length = 0; });

function captureStderr() {
  spy = spyOn(process.stderr, 'write').mockImplementation(((chunk: string | Uint8Array) => {
    stderr.push(String(chunk));
    return true;
  }) as typeof process.stderr.write);
}

function failingEngine(error: Error): BrainEngine {
  return new Proxy({ kind: 'pglite' } as Record<string, unknown>, {
    get(target, prop) {
      if (prop in target) return target[prop as string];
      if (prop === 'then' || prop === 'onCheckout') return undefined;
      return async () => { throw error; };
    },
  }) as unknown as BrainEngine;
}

describe('persistence consumer stderr line', () => {
  test('#5233: carries the SQLSTATE and a redacted, capped, one-line message', async () => {
    const error = Object.assign(new Error(
      '(EMAXCONNSESSION) max clients reached in session mode\n - pool_size: 15 for postgresql://writer:[email protected]:5432/postgres '
        + 'x'.repeat(400)), { code: '53300' });
    captureStderr();
    const consumer = new PersistenceConsumer(failingEngine(error), { engine: 'pglite' }, async () => { throw error; }, { hostId: crypto.randomUUID() });
    try { await consumer.tick(); } finally { await consumer.stop(); }
    const line = stderr.find(entry => entry.startsWith('[persistence] phase='));
    expect(line).toBeDefined();
    expect(line).toContain('reason=53300');
    // GBRA-133: the pooler's client cap names itself and the pool-sizing fix ahead of the writer-status pointer.
    expect(line).toContain('reason=53300 cause=pool_exhausted ');
    expect(line).toContain('fix: gbrain sources writer status --json; pool: every gbrain process on this database shares the pooler');
    expect(line).toContain('(docs/ENGINES.md#pool-sizing); docs: docs/ENGINES.md#persistence-consumer-log');
    expect(line).toContain('message="(EMAXCONNSESSION) max clients reached in session mode - pool_size: 15 for ');
    expect(line).not.toContain('secret-pw');
    expect(line).toContain('fix: gbrain sources writer status --json');
    expect(line).toContain('docs: docs/ENGINES.md#persistence-consumer-log');
    const message = /message="([^"]*)"/.exec(line!)![1];
    expect(message.length).toBeLessThanOrEqual(200);
    expect(line!.trimEnd().includes('\n')).toBe(false);
  });
});

function delayedCheckoutEngine(checkoutAfterMs: number, failAfterMs: number, error: Error) {
  const listeners = new Set<() => void>();
  const call = async () => {
    await Bun.sleep(checkoutAfterMs);
    for (const listener of listeners) listener();
    await Bun.sleep(failAfterMs);
    throw error;
  };
  const target: Record<string, unknown> = {
    kind: 'postgres',
    onCheckout: (listener: () => void) => { listeners.add(listener); return () => { listeners.delete(listener); }; },
  };
  const engine = new Proxy(target, {
    get(t, prop) {
      if (prop in t) return t[prop as string];
      if (prop === 'then') return undefined;
      return call;
    },
  }) as unknown as BrainEngine;
  return { engine, unrelatedCheckout: () => { for (const listener of listeners) listener(); } };
}

describe('#5801 connection-wait evidence on the consumer line', () => {
  const failure = () => Object.assign(new Error('synthetic storage failure'), { code: '08006' });

  test('a checkout that arrives after the deadline is reported as not_observed with the time waited', async () => {
    const { engine } = delayedCheckoutEngine(200, 0, failure());
    captureStderr();
    const consumer = new PersistenceConsumer(engine, { engine: 'postgres' }, async () => { throw failure(); }, { hostId: crypto.randomUUID(), phaseMs: 100 });
    try { await consumer.tick(); } finally { await consumer.stop(); }
    const line = stderr.find(entry => entry.includes('reason=deadline_exceeded'));
    expect(line).toBeDefined();
    expect(line).toContain('checkout=not_observed');
    expect(line).not.toContain('first_conn_ms=');
    expect(Number(/conn_wait_ms=(\d+)/.exec(line!)![1])).toBeGreaterThanOrEqual(100);
    expect(line).toMatch(/loop_lag_ms=\d+/);
  });

  test('a checkout inside the phase reports its real acquisition time', async () => {
    const { engine } = delayedCheckoutEngine(150, 50, failure());
    captureStderr();
    const consumer = new PersistenceConsumer(engine, { engine: 'postgres' }, async () => { throw failure(); }, { hostId: crypto.randomUUID(), phaseMs: 2_000 });
    try { await consumer.tick(); } finally { await consumer.stop(); }
    const line = stderr.find(entry => entry.includes('reason=08006'));
    expect(line).toBeDefined();
    const firstConn = Number(/first_conn_ms=(\d+)/.exec(line!)![1]);
    expect(firstConn).toBeGreaterThanOrEqual(140);
    expect(firstConn).toBeLessThan(2_000);
    expect(line).not.toContain('checkout=not_observed');
  });

  test('an unrelated checkout outside the phase cannot satisfy its observation', async () => {
    const { engine, unrelatedCheckout } = delayedCheckoutEngine(300, 0, failure());
    captureStderr();
    const consumer = new PersistenceConsumer(engine, { engine: 'postgres' }, async () => { throw failure(); }, { hostId: crypto.randomUUID(), phaseMs: 100 });
    const ticking = consumer.tick();
    await Bun.sleep(20);
    unrelatedCheckout();
    try { await ticking; } finally { await consumer.stop(); }
    const line = stderr.find(entry => entry.includes('reason=deadline_exceeded'));
    expect(line).toContain('checkout=not_observed');
    expect(line).not.toContain('first_conn_ms=');
  });
});

/** #6278: every statement parks until its signal aborts, then (as the engine's client-side discard does) rejects CONNECTION_DESTROYED. */
function clientReadWedgeEngine(settleAfterAbortMs: number) {
  let statements = 0;
  const call = (..._args: unknown[]) => {
    const signal = _args.find((arg): arg is { signal?: AbortSignal } => !!arg && typeof arg === 'object' && 'signal' in (arg as object))?.signal;
    statements++;
    return new Promise<never>((_, reject) => {
      const end = () => setTimeout(() => reject(Object.assign(new Error('write CONNECTION_DESTROYED'), { code: 'CONNECTION_DESTROYED' })), settleAfterAbortMs);
      if (signal?.aborted) end(); else signal?.addEventListener('abort', end, { once: true });
    });
  };
  const target: Record<string, unknown> = { kind: 'postgres', onCheckout: () => () => undefined };
  const engine = new Proxy(target, {
    get(t, prop) {
      if (prop in t) return t[prop as string];
      if (prop === 'then') return undefined;
      return call;
    },
  }) as unknown as BrainEngine;
  return { engine, statements: () => statements };
}

describe('#6278 a round-trip the pooler never completes ends as the phase deadline, and the consumer keeps ticking', () => {
  test('the phase logs deadline_exceeded once, the discarded connection is not reported as storage_error, and the next tick runs', async () => {
    const { engine, statements } = clientReadWedgeEngine(30);
    captureStderr();
    const consumer = new PersistenceConsumer(engine, { engine: 'postgres' }, async () => { throw new Error('unused'); }, { hostId: crypto.randomUUID(), phaseMs: 100 });
    try {
      await consumer.tick();
      const first = statements();
      expect(first).toBeGreaterThan(0);
      expect(consumer.status().last_error).toMatchObject({ code: 'deadline_exceeded' });
      expect(stderr.filter(entry => entry.includes('reason=deadline_exceeded')).length).toBe(1);
      expect(stderr.filter(entry => entry.includes('CONNECTION_DESTROYED') || entry.includes('reason=storage_error'))).toEqual([]);
      // The consumer is not parked: a later tick issues statements again.
      await consumer.tick();
      expect(statements()).toBeGreaterThan(first);
    } finally { await consumer.stop(); }
  });
});