Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.3 KB
Raw Blame
/**
 * Five-issue fix wave — doctor's NEW queue_health surfaces (queue-type
 * divergence + waiting-TTL cancellation visibility) and the
 * malformed_path_pages discovery check.
 *
 * computeQueueHealthCheck is Postgres-only (short-circuits to ok on PGLite),
 * so its grouped SQL runs on a real PGLite engine behind a `kind: 'postgres'`
 * stub — the same harness as test/doctor-wedged-queue.test.ts.
 * malformed_path_pages has no exported compute function; it is driven through
 * buildChecks on the real PGLite engine (the seam
 * test/doctor-graph-coverage-soft-deleted.test.ts already uses).
 */

import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { resetPgliteState } from './helpers/reset-pglite.ts';
import { withEnv } from './helpers/with-env.ts';
import { computeQueueHealthCheck, buildChecks } from '../src/commands/doctor.ts';
import type { BrainEngine } from '../src/core/engine.ts';

let base: PGLiteEngine;
let pgLike: BrainEngine;

beforeAll(async () => {
  base = new PGLiteEngine();
  await base.connect({});
  await base.initSchema();
  // computeQueueHealthCheck only reads .kind + .executeRaw.
  pgLike = {
    kind: 'postgres',
    executeRaw: base.executeRaw.bind(base),
  } as unknown as BrainEngine;
});

afterAll(async () => {
  await base.disconnect();
});

beforeEach(async () => {
  await resetPgliteState(base);
});

async function seed(
  queue: string,
  name: string,
  status: string,
  extra: { createdAtSql?: string; finishedAtSql?: string; errorText?: string } = {},
): Promise<void> {
  await base.executeRaw(
    `INSERT INTO minion_jobs (submission_authority, name, queue, status, error_text, created_at, updated_at, finished_at)
     VALUES ('{"version":1,"kind":"application"}'::jsonb, $1, $2, $3, $4, ${extra.createdAtSql ?? 'now()'}, now(), ${extra.finishedAtSql ?? 'NULL'})`,
    [name, queue, status, extra.errorText ?? null],
  );
}

describe('queue_health — DIVERGENT queue type detection', () => {
  it('flags a type whose intake structurally exceeds drain while a real backlog waits', async () => {
    // Lower the min-waiting knob (default 50) so a small seed triggers.
    await withEnv({ GBRAIN_QUEUE_DIVERGENCE_MIN_WAITING: '5' }, async () => {
      // 6 waiting rows created within 24h: intake=6, completed=0, waiting=6.
      // waiting 6 > minWaiting 5 AND intake 6 > ratio 2 × max(0, 1) → DIVERGENT.
      // Stays under GBRAIN_QUEUE_WAITING_THRESHOLD (10) so the depth problem
      // can't fire and mask which surface produced the warn.
      for (let i = 0; i < 6; i++) {
        await seed('default', 'ingest-batch', 'waiting', {
          createdAtSql: "now() - interval '1 hour'",
        });
      }
      const check = await computeQueueHealthCheck(pgLike, {
        readWorkers: () => [{ queue: 'default' }],
      });
      expect(check.status).toBe('warn');
      expect(check.message).toContain("DIVERGENT queue type 'ingest-batch'");
      expect(check.message).toContain('intake 6/24h vs 0 completed/24h');
      // The quota-config admission hint must be paste-ready.
      expect(check.message).toContain('minions.quota_max_waiting.ingest-batch');
    });
  });
});

describe('queue_health — waiting-TTL cancellation visibility', () => {
  it('surfaces waiting_ttl_expired cancellations from the last 24h with the TTL tuning hint', async () => {
    for (const finishedAtSql of ["now() - interval '1 hour'", "now() - interval '2 hours'"]) {
      await seed('default', 'ingest-batch', 'cancelled', {
        errorText: 'waiting_ttl_expired: test',
        finishedAtSql,
      });
    }
    const check = await computeQueueHealthCheck(pgLike, { readWorkers: () => [] });
    expect(check.status).toBe('warn');
    expect(check.message).toContain("waiting-TTL cancelled 2 'ingest-batch' job(s)");
    expect(check.message).toContain('minions.ttl_waiting_hours.ingest-batch');
  });

  it('ignores TTL cancellations older than the 24h window', async () => {
    await seed('default', 'ingest-batch', 'cancelled', {
      errorText: 'waiting_ttl_expired: test',
      finishedAtSql: "now() - interval '48 hours'",
    });
    const check = await computeQueueHealthCheck(pgLike, { readWorkers: () => [] });
    expect(check.status).toBe('ok');
    expect(check.message).not.toContain('waiting-TTL cancelled');
  });
});

describe('queue_health — healthy queue stays quiet', () => {
  it('neither the DIVERGENT nor the waiting-TTL string appears on a healthy seed', async () => {
    await seed('default', 'ingest-batch', 'waiting');
    await seed('default', 'ingest-batch', 'completed', { finishedAtSql: 'now()' });
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => [{ queue: 'default' }],
    });
    expect(check.status).toBe('ok');
    expect(check.message).not.toContain('DIVERGENT');
    expect(check.message).not.toContain('waiting-TTL cancelled');
  });
});

describe('queue_health — unreadable worker registry', () => {
  it('reports liveness unknown instead of erroring or advising a worker start', async () => {
    await seed('default', 'embed-backfill', 'waiting', { createdAtSql: "now() - interval '3 hours'" });
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => { throw new Error('EACCES: permission denied'); },
    });
    expect(check.status).toBe('warn');
    expect(check.details?.worker_alive).toBeNull();
    expect(check.message).toContain('worker liveness is unknown');
    expect(check.message).not.toContain('no live worker is registered');
    expect(check.fix).toBeUndefined();
  });

  it('carries the registry errno and path in details and the message', async () => {
    await seed('default', 'embed-backfill', 'waiting', { createdAtSql: "now() - interval '3 hours'" });
    const { WorkerRegistryUnreadableError } = await import('../src/core/minions/worker-registry.ts');
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => { throw new WorkerRegistryUnreadableError('/home/alice-example/.gbrain/workers', 'EACCES', null); },
    });
    expect(check.details).toMatchObject({
      worker_alive: null,
      worker_registry_unreadable: true,
      worker_registry_errno: 'EACCES',
      worker_registry_path: '/home/alice-example/.gbrain/workers',
    });
    expect(check.message).toContain('unreadable (EACCES at /home/alice-example/.gbrain/workers)');
  });

  it('withholds the registry path from a remote caller; the local caller keeps it', async () => {
    await seed('default', 'embed-backfill', 'waiting', { createdAtSql: "now() - interval '3 hours'" });
    const { WorkerRegistryUnreadableError } = await import('../src/core/minions/worker-registry.ts');
    const readWorkers = () => { throw new WorkerRegistryUnreadableError('/home/alice-example/.gbrain/workers', 'EACCES', null); };
    const remote = await computeQueueHealthCheck(pgLike, { readWorkers, remote: true });
    expect(JSON.stringify(remote)).not.toContain('/home/alice-example');
    expect(remote.details).toMatchObject({ worker_alive: null, worker_registry_unreadable: true, worker_registry_errno: 'EACCES', worker_registry_path_redacted: true });
    expect(remote.details).not.toHaveProperty('worker_registry_path');
    expect(remote.message).toContain('unreadable (EACCES; path withheld from remote callers');
    const local = await computeQueueHealthCheck(pgLike, { readWorkers, remote: false });
    expect(local.details?.worker_registry_path).toBe('/home/alice-example/.gbrain/workers');
    expect(local.message).toContain('unreadable (EACCES at /home/alice-example/.gbrain/workers)');
  });
});

describe('malformed_path_pages discovery check (buildChecks seam)', () => {
  it('warns naming the count + slug when a page is backed by a bracketed filename', async () => {
    await base.executeRaw(
      `INSERT INTO pages (source_id, slug, source_path, type, title, compiled_truth, timeline, frontmatter, content_hash)
       VALUES ('default', 'junk-1', '[foo.md](https-x).md', 'note', 'Junk', 'b', '', '{}'::jsonb, 'mp1')`,
    );
    const checks = await buildChecks(base, [], null);
    const check = checks.find((c) => c.name === 'malformed_path_pages');
    expect(check, 'malformed_path_pages check must be present').toBeDefined();
    expect(check!.status).toBe('warn');
    expect(check!.message).toContain('1 page(s) backed by malformed filenames');
    expect(check!.message).toContain('junk-1');
  });

  it('clean DB → the check does not fire (absence is the ok state)', async () => {
    await base.executeRaw(
      `INSERT INTO pages (source_id, slug, source_path, type, title, compiled_truth, timeline, frontmatter, content_hash)
       VALUES ('default', 'clean-1', 'notes/clean-1.md', 'note', 'Clean', 'b', '', '{}'::jsonb, 'cl1')`,
    );
    const checks = await buildChecks(base, [], null);
    expect(checks.find((c) => c.name === 'malformed_path_pages')).toBeUndefined();
  });
});

describe('queue_health — prompt_too_long deaths are attributed per dream phase (#6303)', () => {
  async function seedDead(key: string | null): Promise<void> {
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, error_text, idempotency_key, created_at, updated_at, finished_at)
       VALUES ('{"version":1,"kind":"application"}'::jsonb, 'subagent', 'default', 'dead', 'prompt_too_long: 300000 tokens', $1, now(), now(), now())`,
      [key],
    );
  }

  it('patterns deaths get patterns advice, never the synthesize budget knob', async () => {
    await seedDead('dream:patterns:abc');
    await seedDead('dream:patterns:def');
    const check = await computeQueueHealthCheck(pgLike);
    expect(check.message).toContain('2 dream patterns subagent job(s)');
    expect(check.message).toContain('models.dream.patterns');
    expect(check.message).toContain('dream.patterns.lookback_days');
    expect(check.message).not.toContain('dream.synthesize.max_prompt_tokens');
  });

  it('synthesize and other deaths keep their own lines', async () => {
    await seedDead('dream:synth-v2:default:filename:a.md:0123456789abcdef');
    await seedDead(null);
    const check = await computeQueueHealthCheck(pgLike);
    expect(check.message).toContain('1 dream synthesize subagent job(s)');
    expect(check.message).toContain('dream.synthesize.max_prompt_tokens');
    expect(check.message).toContain('1 subagent job(s) dead-lettered with prompt_too_long');
    expect(check.message).not.toContain('models.dream.patterns');
  });
});