Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
31.9 KB
Raw Blame
/**
 * issue #1801 fix #3 — doctor surfaces the wedged-queue signature as a health
 * ERROR (and the latent `state`→`status` regression in the remote queue_health
 * check is gone).
 *
 * computeWedgedQueueCheck is Postgres-only (short-circuits to ok on PGLite), so
 * we run its grouped SQL on a real PGLite engine behind a `kind: 'postgres'`
 * stub. Seeds verify: wedged → fail; live-lock → ok; null-completions →
 * conservative ok (the supervisor's startup-grace-aware watchdog owns that case);
 * per-queue grouping so a healthy queue doesn't mask a wedged one (Codex #15).
 */

import { describe, it, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { doctorSource } from './helpers/doctor-source.ts';
import { withEnv } from './helpers/with-env.ts';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import {
  computeOrphanedPrivateQueueCheck,
  computeQueueHealthCheck,
  computeWedgedQueueCheck,
} from '../src/commands/doctor.ts';
import type { BrainEngine } from '../src/core/engine.ts';

let base: PGLiteEngine;
let pgLike: BrainEngine;

beforeAll(async () => {
  base = new PGLiteEngine();
  await base.connect({});
  await base.initSchema();
  // computeWedgedQueueCheck only reads .kind + .executeRaw.
  pgLike = {
    kind: 'postgres',
    executeRaw: base.executeRaw.bind(base),
  } as unknown as BrainEngine;
});

afterAll(async () => {
  await base.disconnect();
});

beforeEach(async () => {
  // Wipe only minion_jobs (preserve config/version that ensureSchema reads).
  await base.executeRaw('DELETE FROM minion_jobs');
});

async function seed(
  queue: string,
  name: string,
  status: string,
  extra: { lockUntilSql?: string; updatedAtSql?: string; createdAtSql?: string } = {},
): Promise<void> {
  await base.executeRaw(
    `INSERT INTO minion_jobs (submission_authority, name, queue, status, lock_until, updated_at, created_at)
     VALUES ('{"version":1,"kind":"application"}'::jsonb, $1, $2, $3, ${extra.lockUntilSql ?? 'NULL'}, ${extra.updatedAtSql ?? 'now()'}, ${extra.createdAtSql ?? 'now()'})`,
    [name, queue, status],
  );
}

describe('issue #2557 — queue_health catches deferred embed with no worker', () => {
  it('warns when old embed-backfill jobs have no live worker for their queue', async () => {
    await seed('default', 'embed-backfill', 'waiting', {
      createdAtSql: "now() - interval '3 hours'",
    });
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => [],
      oldWaitingHours: 1,
    });
    expect(check.status).toBe('warn');
    expect(check.message).toContain('embed-backfill');
    expect(check.message).toContain('no live worker');
    expect(check.message).toContain('gbrain jobs work --queue default');
  });

  it('does not warn for old embed-backfill jobs when a worker is live on that queue', async () => {
    await seed('default', 'embed-backfill', 'waiting', {
      createdAtSql: "now() - interval '3 hours'",
    });
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => [{ queue: 'default' }],
      oldWaitingHours: 1,
    });
    expect(check.status).toBe('ok');
    expect(check.message).toContain('no old embed-backfill jobs without a worker');
  });
});

describe('issue #1801 fix #3 — computeWedgedQueueCheck', () => {
  it('flags a wedged queue (waiting, 0 active_healthy, stale completion) as fail', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain("'default'");
  });

  // #3063: the check must not claim "worker alive" when no worker is
  // registered for the wedged queue, and must keep the original "worker
  // alive but stuck" wording when one genuinely is.
  it('says "No worker subscribed" (not "worker alive") when no live worker is registered for the wedged queue', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike, { readWorkers: () => [] });
    expect(check.status).toBe('fail');
    expect(check.message).toContain('No worker subscribed');
    expect(check.message).not.toContain('worker alive but not claiming work');
  });

  it('keeps "worker alive but not claiming work" when a live worker IS registered for the wedged queue', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike, { readWorkers: () => [{ queue: 'default' }] });
    expect(check.status).toBe('fail');
    expect(check.message).toContain('worker alive but not claiming work');
    expect(check.message).not.toContain('No worker subscribed');
  });

  it('a throwing registry read → still fail, but liveness-unknown wording (no fabricated verdict)', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike, {
      readWorkers: () => { throw new Error('registry dir unreadable'); },
    });
    expect(check.status).toBe('fail');
    expect(check.message).toContain('worker registry unreadable');
    expect(check.message).not.toContain('worker alive but not claiming work');
    expect(check.message).not.toContain('No worker subscribed');
    expect(check.details?.worker_registry_unreadable).toBe(true);
    expect(check.details?.worker_registry_errno).toBe('Error');
    expect(check.details?.worker_registry_path).toBeNull();
  });

  it('a remote caller never sees the registry path; the local caller does', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const { WorkerRegistryUnreadableError } = await import('../src/core/minions/worker-registry.ts');
    const readWorkers = () => { throw new WorkerRegistryUnreadableError('/home/alice-example/.gbrain/workers', 'EACCES', null); };
    const remote = await computeWedgedQueueCheck(pgLike, { readWorkers, remote: true });
    expect(remote.status).toBe('fail');
    expect(JSON.stringify(remote)).not.toContain('/home/alice-example');
    expect(remote.details).toMatchObject({ worker_registry_unreadable: true, worker_registry_errno: 'EACCES', worker_registry_path_redacted: true });
    expect(remote.details).not.toHaveProperty('worker_registry_path');
    const local = await computeWedgedQueueCheck(pgLike, { readWorkers });
    expect(local.details?.worker_registry_path).toBe('/home/alice-example/.gbrain/workers');
    expect(local.message).toContain('EACCES at /home/alice-example/.gbrain/workers');
  });

  it('mixed stuck + no-worker queues → both messages, details count each subset', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    await seed('q-orphan', 'cycle', 'waiting');
    await seed('q-orphan', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike, { readWorkers: () => [{ queue: 'default' }] });
    expect(check.status).toBe('fail');
    expect(check.message).toContain('worker alive but not claiming work');
    expect(check.message).toContain("'default'");
    expect(check.message).toContain('No worker subscribed');
    expect(check.message).toContain("'q-orphan'");
    expect(check.details?.stuck_worker_queues).toBe(1);
    expect(check.details?.no_worker_queues).toBe(1);
    expect(check.details?.wedged_queues).toBe(2);
  });

  it('does NOT flag when a job holds a live lock (active_healthy > 0)', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" });
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike);
    expect(check.status).toBe('ok');
  });

  it('does NOT flag an expired-lock active row as healthy — still wedged (Codex #6)', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'active', { lockUntilSql: "now() - interval '2 min'" }); // expired
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '20 min'" });
    const check = await computeWedgedQueueCheck(pgLike);
    expect(check.status).toBe('fail'); // expired lock does not count as active_healthy
  });

  it('is conservative on never-completed queues (null mins → ok)', async () => {
    await seed('default', 'cycle', 'waiting'); // no completed row → mins null
    const check = await computeWedgedQueueCheck(pgLike);
    expect(check.status).toBe('ok');
  });

  it('groups by queue — a healthy queue does not mask a wedged one (Codex #15)', async () => {
    // healthy queue
    await seed('q-healthy', 'cycle', 'active', { lockUntilSql: "now() + interval '5 min'" });
    await seed('q-healthy', 'cycle', 'completed', { updatedAtSql: 'now()' });
    // wedged queue
    await seed('q-wedged', 'cycle', 'waiting');
    await seed('q-wedged', 'cycle', 'completed', { updatedAtSql: "now() - interval '30 min'" });
    const check = await computeWedgedQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain("'q-wedged'");
    expect(check.message).not.toContain("'q-healthy'");
  });

  it('shell-escapes an embedded single quote in the restart hint (producer-controlled queue names)', async () => {
    // The hint is copy-pasted by operators AND remediation agents, so a
    // queue named q-wedge'd must arrive POSIX-escaped, never raw.
    // A live worker must be registered for the queue, or the #3063
    // liveness split routes this to the "no worker" branch instead,
    // which never builds the restart-hint string at all.
    await seed("q-wedge'd", 'cycle', 'waiting');
    await seed("q-wedge'd", 'cycle', 'completed', { updatedAtSql: "now() - interval '30 min'" });
    const check = await computeWedgedQueueCheck(pgLike, { readWorkers: () => [{ queue: "q-wedge'd" }] });
    expect(check.status).toBe('fail');
    expect(check.message).toContain(String.raw`--queue 'q-wedge'\''d'`);
  });

  it('hint union: wedged default AND non-default queues → bare hint plus --queue variant, comma-joined', async () => {
    await seed('default', 'cycle', 'waiting');
    await seed('default', 'cycle', 'completed', { updatedAtSql: "now() - interval '30 min'" });
    await seed('q-side', 'cycle', 'waiting');
    await seed('q-side', 'cycle', 'completed', { updatedAtSql: "now() - interval '30 min'" });
    // Both queues need a live worker registered so the #3063 liveness
    // split routes them into the "stuck" branch that builds this
    // restart-hint union, not the "no worker" branch.
    const check = await computeWedgedQueueCheck(pgLike, {
      readWorkers: () => [{ queue: 'default' }, { queue: 'q-side' }],
    });
    expect(check.status).toBe('fail');
    // The bare pair (for 'default') terminates with a backtick — no --queue —
    // and the comma-join carries the per-queue variant right behind it.
    expect(check.message).toContain('`gbrain jobs supervisor stop && gbrain jobs supervisor start`, `');
    expect(check.message).toContain("--queue 'q-side'");
  });

  it('does NOT misclassify parent-owned dream queues as worker wedges', async () => {
    await seed('dream-inline-dead-parent', 'subagent', 'waiting', {
      createdAtSql: "now() - interval '2 hours'",
    });
    await seed('dream-inline-dead-parent', 'subagent', 'completed', {
      updatedAtSql: "now() - interval '2 hours'",
    });
    const check = await computeWedgedQueueCheck(pgLike);
    expect(check.status).toBe('ok');
  });

  it('returns ok on PGLite (no multi-process worker surface)', async () => {
    const check = await computeWedgedQueueCheck(base as unknown as BrainEngine);
    expect(check.status).toBe('ok');
    expect(check.message).toContain('PGLite');
  });
});

describe('orphaned private dream queues', () => {
  it('flags an old private queue with waiting jobs and no live parent', async () => {
    await seed('dream-inline-dead-parent', 'subagent', 'waiting', {
      createdAtSql: "now() - interval '2 hours'",
    });
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain('supervisor restart cannot consume');
    // No owner metadata seeded → the legacy bucket, whose ONLY advertised
    // remediation is the retriage preview (never a worker restart).
    expect(check.message).toContain('gbrain dream retriage');
    expect(check.details).toMatchObject({
      orphaned_private_queues: 1,
      waiting_jobs: 1,
      legacy_unowned_queues: 1,
      recoverable_queues: 0,
    });
  });

  it('does not flag a private queue whose parent still holds a live job lock', async () => {
    await seed('dream-inline-live-parent', 'subagent', 'waiting', {
      createdAtSql: "now() - interval '2 hours'",
    });
    await seed('dream-inline-live-parent', 'subagent', 'active', {
      lockUntilSql: "now() + interval '5 min'",
    });
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
  });

  it('allows a startup grace period for a new private queue', async () => {
    await seed('dream-inline-new-parent', 'subagent', 'waiting', {
      createdAtSql: "now() - interval '10 min'",
    });
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
  });

  it('runs on PGLite too (no engine short-circuit): a dead-parent queue is flagged', async () => {
    // PGLite mints the same dream-inline-* queues (children are inlined
    // precisely because no worker process can run) — the old short-circuit
    // hid exactly the brains with the fewest recovery lanes.
    await seed('dream-inline-pglite-dead', 'subagent', 'waiting', {
      createdAtSql: "now() - interval '2 hours'",
    });
    const check = await computeOrphanedPrivateQueueCheck(base as unknown as BrainEngine);
    expect(check.status).toBe('fail');
    // Engine-aware remediation: on PGLite the auto-recovery trigger is the
    // next dream run, never a supervisor command.
    expect(check.message).not.toContain('supervisor start`');
  });

  it('buckets a metadata-backed orphan as recoverable with the auto-recovery remediation', async () => {
    // Terminal owner + expired lease → the classifier verdict is 'orphan':
    // auto-recovery cancels it at the next worker spawn / cycle start.
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at) VALUES ('{"version":1,"kind":"application"}'::jsonb, 'parent', 'cycle', 'completed', now() - interval '3 hours')`,
    );
    const owner = await base.executeRaw<{ id: number }>(`SELECT max(id)::int AS id FROM minion_jobs`);
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_owner_job_id, private_queue_owner_token, private_queue_lease_until)
       VALUES ('{"version":1,"kind":"application"}'::jsonb, 'child', 'dream-inline-owned-dead', 'waiting', now() - interval '2 hours', now() - interval '2 hours', $1, 'tok', now() - interval '1 hour')`,
      [owner[0].id],
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain('Auto-recovery cancels the metadata-backed queue(s)');
    expect(check.details).toMatchObject({ recoverable_queues: 1, legacy_unowned_queues: 0 });
  });

  it('suppresses a queue whose owner lease is still in the future (live, never flagged)', async () => {
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, private_queue_owner_token, private_queue_lease_until)
       VALUES ('{"version":1,"kind":"application"}'::jsonb, 'child', 'dream-inline-leased', 'waiting', now() - interval '2 hours', 'tok', now() + interval '30 min')`,
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
    expect(check.details).toMatchObject({ suppressed_by_live_lease: 1 });
  });
});

describe('orphaned private dream queues — cycle-lock liveness + ownership correlation (#4250)', () => {
  async function seedWithData(
    queue: string,
    status: string,
    dataJson: Record<string, unknown>,
    extra: { createdAtSql?: string } = {},
  ): Promise<void> {
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, data, created_at)
       VALUES ('{"version":1,"kind":"application"}'::jsonb, $1, $2, $3, $4::text::jsonb, ${extra.createdAtSql ?? 'now()'})`,
      ['subagent', queue, status, JSON.stringify(dataJson)],
    );
  }

  async function seedLock(id: string, acquiredAtSql: string): Promise<void> {
    await base.executeRaw(
      `INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at)
       VALUES ($1, 1234, 'test-host', ${acquiredAtSql}, now() + interval '10 min', now())
       ON CONFLICT (id) DO UPDATE SET acquired_at = ${acquiredAtSql}, ttl_expires_at = now() + interval '10 min'`,
      [id],
    );
  }

  function inlineQueueName(bornMsAgo: number): string {
    return `dream-inline-${Date.now() - bornMsAgo}-abcdef01`;
  }

  beforeEach(async () => {
    await base.executeRaw('DELETE FROM gbrain_cycle_locks');
  });

  it('a live per-source lock acquired BEFORE the queue was born suppresses it (possibly owned)', async () => {
    const q = inlineQueueName(2 * 3600_000); // born 2h ago
    await seedWithData(q, 'waiting', { source_id: 'repo-a' }, { createdAtSql: "now() - interval '2 hours'" });
    await seedLock('gbrain-cycle:repo-a', "now() - interval '3 hours'"); // acquired before birth
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
    expect((check.details as any)?.suppressed_by_live_lock).toBe(1);
  });

  it('a live lock acquired AFTER the queue was born cannot own it — still flagged', async () => {
    const q = inlineQueueName(2 * 3600_000); // born 2h ago
    await seedWithData(q, 'waiting', { source_id: 'repo-a' }, { createdAtSql: "now() - interval '2 hours'" });
    await seedLock('gbrain-cycle:repo-a', "now() - interval '30 min'"); // new cycle, acquired after birth
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain(q);
  });

  it("a live lock for a DIFFERENT source does not suppress another source's dead queue", async () => {
    const q = inlineQueueName(2 * 3600_000);
    await seedWithData(q, 'waiting', { source_id: 'repo-a' }, { createdAtSql: "now() - interval '2 hours'" });
    await seedLock('gbrain-cycle:repo-b', "now() - interval '3 hours'");
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
  });

  it('the bare global lock suppresses only queues it could own (born at/after acquisition)', async () => {
    const oldQ = inlineQueueName(4 * 3600_000); // predates the lock → flagged
    const newQ = inlineQueueName(90 * 60_000);  // born after acquisition, >60m old → suppressed
    await seedWithData(oldQ, 'waiting', {}, { createdAtSql: "now() - interval '4 hours'" });
    await seedWithData(newQ, 'waiting', {}, { createdAtSql: "now() - interval '90 min'" });
    await seedLock('gbrain-cycle', "now() - interval '2 hours'");
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain(oldQ);
    expect(check.message).not.toContain(newQ);
    expect((check.details as any)?.suppressed_by_live_lock).toBe(1);
  });

  it('delayed rows count toward the waiting-class (retriage repairs waiting|delayed)', async () => {
    await seedWithData('dream-inline-delayed-only', 'delayed', {}, { createdAtSql: "now() - interval '2 hours'" });
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect((check.details as any)?.waiting_jobs).toBe(1);
  });

  it('paused rows never fail the check (no advertised repair path) but surface in details', async () => {
    await seedWithData('dream-inline-paused-only', 'paused', {}, { createdAtSql: "now() - interval '2 hours'" });
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
    expect((check.details as any)?.paused_jobs).toBe(1);
  });

  it('GBRAIN_ORPHANED_PRIVATE_QUEUE_MINUTES overrides the age threshold', async () => {
    await seedWithData('dream-inline-fresh-orphan', 'waiting', {}, { createdAtSql: "now() - interval '10 min'" });
    await withEnv({ GBRAIN_ORPHANED_PRIVATE_QUEUE_MINUTES: '5' }, async () => {
      const check = await computeOrphanedPrivateQueueCheck(pgLike);
      expect(check.status).toBe('fail'); // 10min > 5min override; default 60 would pass
    });
  });

  it('a TTL-lapsed lock refreshed within the steal grace still counts as live (starved-but-alive holder)', async () => {
    const q = inlineQueueName(2 * 3600_000);
    await seedWithData(q, 'waiting', { source_id: 'repo-slow' }, { createdAtSql: "now() - interval '2 hours'" });
    // TTL lapsed 1 min ago, but the holder refreshed 30s ago and acquired
    // BEFORE the queue was born — db-lock's steal path would not kill this
    // holder, so doctor must not point operators at cancelling its queue.
    await base.executeRaw(
      `INSERT INTO gbrain_cycle_locks (id, holder_pid, holder_host, acquired_at, ttl_expires_at, last_refreshed_at)
       VALUES ('gbrain-cycle:repo-slow', 1234, 'test-host', now() - interval '3 hours', now() - interval '1 min', now() - interval '30 sec')`,
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
    expect((check.details as any)?.suppressed_by_live_lock).toBe(1);
  });

  it('clock-skew tolerance: a lock acquired seconds AFTER the queue birth still owns it', async () => {
    // Host Date.now (queue name) vs DB NOW() (acquired_at) can skew by
    // seconds on remote engines; the maintenance lane mints its queue right
    // after acquiring the lock. A 30s gap must suppress, not flag.
    const q = inlineQueueName(2 * 3600_000);
    await seedWithData(q, 'waiting', { source_id: 'repo-skew' }, { createdAtSql: "now() - interval '2 hours'" });
    await seedLock('gbrain-cycle:repo-skew', "now() - interval '2 hours' + interval '30 sec'");
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('ok');
    expect((check.details as any)?.suppressed_by_live_lock).toBe(1);
  });
});

describe('orphaned private dream queues — classify cap, buckets, flagged-only counts (749a7dcb)', () => {
  beforeEach(async () => {
    // Leftover live cycle locks from earlier describes must not suppress
    // these candidates via the ownership correlation.
    await base.executeRaw('DELETE FROM gbrain_cycle_locks');
  });

  it('caps classification at 100 candidates and reports the remainder as unclassified (fail path)', async () => {
    // 101 metadata-backed orphan candidates (aged, waiting, expired lease —
    // classifier verdict 'orphan'), one row per queue for speed.
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_lease_until)
       SELECT '{"version":1,"kind":"application"}'::jsonb, 'subagent', 'dream-inline-cap-' || i, 'waiting',
              now() - interval '2 hours', now() - interval '2 hours', now() - interval '1 hour'
         FROM generate_series(1, 101) AS i`,
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect((check.details as any).orphaned_private_queues).toBeLessThanOrEqual(100);
    expect((check.details as any).orphaned_private_queues).toBe(100);
    expect((check.details as any).unclassified_candidates).toBe(1);
  });

  it('ok-path truncation honesty: 100 live + 1 never-classified is NOT a clean bill of health', async () => {
    // First 100 candidates classify 'live' (future lease + fresh updated_at);
    // the 101st never gets classified. #7145: that is health unknown (warn),
    // and the message and details still surface the unclassified remainder.
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_lease_until)
       SELECT '{"version":1,"kind":"application"}'::jsonb, 'subagent', 'dream-inline-live-' || i, 'waiting',
              now() - interval '2 hours', now(), now() + interval '30 minutes'
         FROM generate_series(1, 101) AS i`,
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('warn');
    expect(check.details).toMatchObject({ health: 'unknown', reason: 'bound', bound: 'rows', limit: 100 });
    expect(check.message).toContain('unclassified');
    expect((check.details as any).unclassified_candidates).toBe(1);
    expect((check.details as any).suppressed_by_live_lease).toBe(100);
  });

  it('buckets a NON-terminal unclaimed owner as owner_pending with the inspect-first remediation', async () => {
    // Owner job still waiting (not claimed, no lock) → verdict not_orphan:
    // doctor must say "inspect the owner", never advertise cancellation.
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at) VALUES ('{"version":1,"kind":"application"}'::jsonb, 'dream-cycle', 'default', 'waiting', now() - interval '3 hours')`,
    );
    const owner = await base.executeRaw<{ id: number }>(`SELECT max(id)::int AS id FROM minion_jobs`);
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_owner_job_id, private_queue_owner_token)
       VALUES ('{"version":1,"kind":"application"}'::jsonb, 'subagent', 'dream-inline-owner-pending', 'waiting', now() - interval '2 hours', now() - interval '2 hours', $1, 'tok')`,
      [owner[0].id],
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.message).toContain('inspect it with');
    expect(check.message).toContain('gbrain jobs get');
    expect(check.details).toMatchObject({
      owner_pending_queues: 1,
      recoverable_queues: 0,
      legacy_unowned_queues: 0,
    });
  });

  it('PGLite recoverable fixture advertises `gbrain dream` as the trigger, never the supervisor', async () => {
    // Terminal owner + aged + expired lease → recoverable bucket. On PGLite
    // the engine-aware remediation must name the dream trigger (a supervisor
    // command is impossible advice — no worker process can ever run there).
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at) VALUES ('{"version":1,"kind":"application"}'::jsonb, 'dream-cycle', 'default', 'completed', now() - interval '3 hours')`,
    );
    const owner = await base.executeRaw<{ id: number }>(`SELECT max(id)::int AS id FROM minion_jobs`);
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_owner_job_id, private_queue_owner_token, private_queue_lease_until)
       VALUES ('{"version":1,"kind":"application"}'::jsonb, 'subagent', 'dream-inline-pglite-recoverable', 'waiting', now() - interval '2 hours', now() - interval '2 hours', $1, 'tok', now() - interval '1 hour')`,
      [owner[0].id],
    );
    const check = await computeOrphanedPrivateQueueCheck(base as unknown as BrainEngine);
    expect(check.status).toBe('fail');
    expect(check.message).toContain('Auto-recovery cancels the metadata-backed queue(s)');
    expect(check.message).toContain('trigger now: `gbrain dream`');
    expect(check.message).not.toContain('supervisor start`');
  });

  it('waiting_jobs counts FLAGGED queues only — live-lease-suppressed waiting rows are excluded (749a7dcb)', async () => {
    // Recoverable orphan with 3 waiting rows (terminal owner, expired lease).
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at) VALUES ('{"version":1,"kind":"application"}'::jsonb, 'dream-cycle', 'default', 'completed', now() - interval '3 hours')`,
    );
    const owner = await base.executeRaw<{ id: number }>(`SELECT max(id)::int AS id FROM minion_jobs`);
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_owner_job_id, private_queue_owner_token, private_queue_lease_until)
       SELECT '{"version":1,"kind":"application"}'::jsonb, 'subagent', 'dream-inline-orphan-n', 'waiting',
              now() - interval '2 hours', now() - interval '2 hours', $1, 'tok', now() - interval '1 hour'
         FROM generate_series(1, 3)`,
      [owner[0].id],
    );
    // Aged candidate suppressed by a live lease, carrying 2 waiting rows that
    // must NOT leak into waiting_jobs (pre-fix it summed N+M during the scan).
    await base.executeRaw(
      `INSERT INTO minion_jobs (submission_authority, name, queue, status, created_at, updated_at, private_queue_lease_until)
       SELECT '{"version":1,"kind":"application"}'::jsonb, 'subagent', 'dream-inline-lively-m', 'waiting',
              now() - interval '2 hours', now() - interval '2 hours', now() + interval '30 minutes'
         FROM generate_series(1, 2)`,
    );
    const check = await computeOrphanedPrivateQueueCheck(pgLike);
    expect(check.status).toBe('fail');
    expect(check.details).toMatchObject({
      orphaned_private_queues: 1,
      waiting_jobs: 3,
      suppressed_by_live_lease: 1,
    });
  });
});

describe('Minions-visibility wave — computeQueueHealthCheck structured details', () => {
  it('ok path carries {depth, oldest_age_seconds, worker_alive} (messages unchanged)', async () => {
    const check = await computeQueueHealthCheck(pgLike, { readWorkers: () => [] });
    expect(check.status).toBe('ok');
    // Message text is pinned elsewhere by prose consumers — unchanged.
    expect(check.message).toContain('No stalled-forever jobs');
    // Empty queue: zero depth, null oldest age, vacuously worker_alive.
    expect(check.details).toEqual({ depth: 0, oldest_age_seconds: null, worker_alive: true });
  });

  it('warn path reports total waiting depth + oldest age + worker_alive=false when a waiting queue has no worker', async () => {
    await seed('default', 'embed-backfill', 'waiting', {
      createdAtSql: "now() - interval '3 hours'",
    });
    await seed('default', 'embed', 'waiting', {
      createdAtSql: "now() - interval '10 minutes'",
    });
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => [],
      oldWaitingHours: 1,
    });
    expect(check.status).toBe('warn');
    const d = check.details as { depth: number; oldest_age_seconds: number; worker_alive: boolean };
    expect(d.depth).toBe(2);
    expect(d.oldest_age_seconds).toBeGreaterThanOrEqual(3 * 3600 - 60);
    expect(d.worker_alive).toBe(false);
  });

  it('worker_alive=true when every waiting queue has a live registered worker', async () => {
    await seed('default', 'embed', 'waiting');
    const check = await computeQueueHealthCheck(pgLike, {
      readWorkers: () => [{ queue: 'default' }],
    });
    const d = check.details as { worker_alive: boolean };
    expect(d.worker_alive).toBe(true);
  });
});

describe('issue #1801 fix #3 — remote queue_health state→status regression', () => {
  it('doctor.ts no longer queries the non-existent `state` column', () => {
    const src = doctorSource();
    // The column is `status`; the pre-fix `WHERE state = 'active'` errored every
    // run and the catch silently returned "No queue activity".
    expect(src).not.toContain("state = 'active'");
  });
});