Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
19.6 KB
Raw Blame
/**
 * Queue admission control (five-issue fix wave, commit 5): param-coalescing,
 * waiting-TTL sweep, and the name-global quota. Covers the four
 * outside-voice-hardened invariants:
 *   - owner lanes never cross (hash INCLUDES __owner_client_id),
 *   - parented submits never coalesce (aggregator bookkeeping),
 *   - coalescing is age-bounded to ttl/2 (a fresh submit must not coalesce
 *     onto a nearly-expired row and die an hour later),
 *   - TTL cancellation flows through cancelJobs (descendants + child_done +
 *     parent resolution) and writes the reason error_text every alerting
 *     surface keys on.
 */

import { describe, test, expect, beforeAll, afterAll, beforeEach } from 'bun:test';
import { PGLiteEngine } from '../src/core/pglite-engine.ts';
import { withEnv } from './helpers/with-env.ts';
import { MinionQueue } from '../src/core/minions/queue.ts';
import {
  computeParamHash,
  QueueQuotaExceededError,
  runWaitingTtlTick,
  TTL_NOTICE_SHOWN_KEY,
  _resetAdmissionCacheForTest,
} from '../src/core/minions/admission.ts';

let engine: PGLiteEngine;
let queue: MinionQueue;

const SUB = { allowProtectedSubmit: true } as const;

beforeAll(async () => {
  engine = new PGLiteEngine();
  await engine.connect({});
  await engine.initSchema();
  queue = new MinionQueue(engine);
}, 60_000);

afterAll(async () => {
  await engine.disconnect();
}, 60_000);

beforeEach(async () => {
  // Targeted reset (same posture as the other minion suites): wipe jobs +
  // inbox + this suite's config keys, but keep the schema-version row that
  // ensureSchema() checks.
  await engine.executeRaw(`DELETE FROM minion_inbox`, []);
  await engine.executeRaw(`DELETE FROM minion_jobs`, []);
  await engine.executeRaw(`DELETE FROM config WHERE key LIKE 'minions.%'`, []);
  _resetAdmissionCacheForTest();
});

async function backdate(id: number, hours: number): Promise<void> {
  // Both timestamps: the TTL sweep + coalesce age-bound key on updated_at
  // (a requeue refreshes the TTL window); getStats oldest-wait uses created_at.
  await engine.executeRaw(
    `UPDATE minion_jobs SET created_at = now() - ($2 * interval '1 hour'),
                            updated_at = now() - ($2 * interval '1 hour') WHERE id = $1`,
    [id, hours],
  );
}

describe('computeParamHash', () => {
  test('stable across key order, excludes only __param_hash', () => {
    const a = computeParamHash({ prompt: 'p', model: 'm', nested: { x: 1, y: 2 } });
    const b = computeParamHash({ model: 'm', nested: { y: 2, x: 1 }, prompt: 'p' });
    expect(a).toBe(b);
    expect(computeParamHash({ prompt: 'p', __param_hash: 'stale' })).toBe(computeParamHash({ prompt: 'p' }));
  });

  test('__owner_client_id is INCLUDED — owner lanes hash differently', () => {
    const a = computeParamHash({ prompt: 'p', __owner_client_id: 'client-a' });
    const b = computeParamHash({ prompt: 'p', __owner_client_id: 'client-b' });
    expect(a).not.toBe(b);
  });
});

describe('param-coalescing (subagent default-on)', () => {
  test('identical parentless submit coalesces onto the waiting row', async () => {
    const first = await queue.add('subagent', { prompt: 'daily digest', model: 'anthropic:claude-sonnet-5' }, {}, SUB);
    expect(first.coalesced).not.toBe(true);
    expect((first.data as Record<string, unknown>).__param_hash).toBeDefined();

    const second = await queue.add('subagent', { prompt: 'daily digest', model: 'anthropic:claude-sonnet-5' }, {}, SUB);
    expect(second.id).toBe(first.id);
    expect(second.coalesced).toBe(true);

    const rows = await engine.executeRaw<{ count: string }>(
      `SELECT count(*)::text AS count FROM minion_jobs WHERE name = 'subagent'`,
    );
    expect(parseInt(rows[0].count, 10)).toBe(1);
  });

  test('different prompts insert separate rows', async () => {
    const a = await queue.add('subagent', { prompt: 'task A' }, {}, SUB);
    const b = await queue.add('subagent', { prompt: 'task B' }, {}, SUB);
    expect(b.id).not.toBe(a.id);
    expect(b.coalesced).not.toBe(true);
  });

  test('identical prompts from different owner lanes never coalesce', async () => {
    const a = await queue.add('subagent', { prompt: 'same', __owner_client_id: 'client-a' }, {}, SUB);
    const b = await queue.add('subagent', { prompt: 'same', __owner_client_id: 'client-b' }, {}, SUB);
    expect(b.id).not.toBe(a.id);
  });

  test('a RUNNING identical job does not suppress a re-run (waiting-only)', async () => {
    const first = await queue.add('subagent', { prompt: 'rerun me' }, {}, SUB);
    await engine.executeRaw(
      `UPDATE minion_jobs SET status = 'active', claim_generation = claim_generation + 1, lock_until = now() + interval '5 minutes' WHERE id = $1`,
      [first.id],
    );
    const second = await queue.add('subagent', { prompt: 'rerun me' }, {}, SUB);
    expect(second.id).not.toBe(first.id);
  });

  test('EMPTY payloads never coalesce (no dedupe signal — scaffolding/placeholder jobs stay distinct)', async () => {
    const a = await queue.add('subagent', {}, {}, SUB);
    const b = await queue.add('subagent', {}, {}, SUB);
    expect(b.id).not.toBe(a.id);
    expect((a.data as Record<string, unknown>).__param_hash).toBeUndefined();
  });

  test('parented submits never coalesce (fanout children belong to their parent)', async () => {
    const parent = await queue.add('subagent_aggregator', { kind: 'agg' }, {}, SUB);
    const c1 = await queue.add('subagent', { prompt: 'chunk' }, { parent_job_id: parent.id }, SUB);
    const c2 = await queue.add('subagent', { prompt: 'chunk' }, { parent_job_id: parent.id }, SUB);
    expect(c2.id).not.toBe(c1.id);
  });

  test('age-bounded: a near-TTL waiting row is not a coalesce target', async () => {
    const first = await queue.add('subagent', { prompt: 'old intent' }, {}, SUB);
    await backdate(first.id, 47); // TTL default 48h → bound is 24h
    const second = await queue.add('subagent', { prompt: 'old intent' }, {}, SUB);
    expect(second.id).not.toBe(first.id);
  });

  test('kill-switch GBRAIN_MINIONS_ADMISSION=0 disables coalescing', async () => {
    await withEnv({ GBRAIN_MINIONS_ADMISSION: '0' }, async () => {
      _resetAdmissionCacheForTest();
      const a = await queue.add('subagent', { prompt: 'killed' }, {}, SUB);
      const b = await queue.add('subagent', { prompt: 'killed' }, {}, SUB);
      expect(b.id).not.toBe(a.id);
    });
    _resetAdmissionCacheForTest(); // don't leak the killed-policy cache to later tests
  });

  test('config off-switch minions.coalesce_params.subagent=false disables it', async () => {
    await engine.setConfig('minions.coalesce_params.subagent', 'false');
    _resetAdmissionCacheForTest();
    const a = await queue.add('subagent', { prompt: 'cfg-off' }, {}, SUB);
    const b = await queue.add('subagent', { prompt: 'cfg-off' }, {}, SUB);
    expect(b.id).not.toBe(a.id);
  });

  test('opt-in for a non-default name via coalesce_params submit option', async () => {
    const a = await queue.add('shell', { command: 'echo hi' }, { coalesce_params: true }, SUB);
    const b = await queue.add('shell', { command: 'echo hi' }, { coalesce_params: true }, SUB);
    expect(b.id).toBe(a.id);
    expect(b.coalesced).toBe(true);
  });
});

describe('name-global quota (config-only, no shipped default)', () => {
  test('no quota configured → unlimited (D2C: default protection is the scream, not the cap)', async () => {
    for (let i = 0; i < 5; i++) {
      await queue.add('subagent', { prompt: `distinct ${i}` }, {}, SUB);
    }
    const rows = await engine.executeRaw<{ count: string }>(
      `SELECT count(*)::text AS count FROM minion_jobs WHERE name = 'subagent' AND status = 'waiting'`,
    );
    expect(parseInt(rows[0].count, 10)).toBe(5);
  });

  test('configured quota rejects with a typed, actionable error', async () => {
    await engine.setConfig('minions.quota_max_waiting.subagent', '2');
    _resetAdmissionCacheForTest();
    await queue.add('subagent', { prompt: 'q1' }, {}, SUB);
    await queue.add('subagent', { prompt: 'q2' }, {}, SUB);
    let threw: unknown = null;
    try {
      await queue.add('subagent', { prompt: 'q3' }, {}, SUB);
    } catch (e) {
      threw = e;
    }
    expect(threw).toBeInstanceOf(QueueQuotaExceededError);
    expect((threw as Error).message).toContain('minions.quota_max_waiting.subagent');
    expect((threw as QueueQuotaExceededError).code).toBe('quota_exceeded');
  });

  test('quota counts the name across ALL queues (private fanout queues cannot dodge it)', async () => {
    await engine.setConfig('minions.quota_max_waiting.subagent', '2');
    _resetAdmissionCacheForTest();
    await queue.add('subagent', { prompt: 'a' }, { queue: 'dream-inline-1' }, SUB);
    await queue.add('subagent', { prompt: 'b' }, { queue: 'dream-inline-2' }, SUB);
    await expect(
      queue.add('subagent', { prompt: 'c' }, { queue: 'dream-inline-3' }, SUB),
    ).rejects.toThrow('quota');
  });

  test('an identical submit coalesces BEFORE the quota check (dedupe is not rejection)', async () => {
    await engine.setConfig('minions.quota_max_waiting.subagent', '2');
    _resetAdmissionCacheForTest();
    const a = await queue.add('subagent', { prompt: 'same one' }, {}, SUB);
    await queue.add('subagent', { prompt: 'other' }, {}, SUB);
    // At quota. An IDENTICAL re-submit must return the existing row, not throw.
    const again = await queue.add('subagent', { prompt: 'same one' }, {}, SUB);
    expect(again.id).toBe(a.id);
    expect(again.coalesced).toBe(true);
  });
});

describe('waiting-TTL sweep (handleWaitingTTL via cancelJobs)', () => {
  test('cancels waiting rows past TTL with the reason error_text; young rows untouched', async () => {
    const old = await queue.add('subagent', { prompt: 'ancient' }, {}, SUB);
    const young = await queue.add('subagent', { prompt: 'fresh' }, {}, SUB);
    await backdate(old.id, 49);

    const { cancelled, by_name } = await queue.handleWaitingTTL();
    expect(cancelled).toBe(1);
    expect(by_name.subagent).toBe(1);

    const oldRow = await queue.getJob(old.id);
    expect(oldRow?.status).toBe('cancelled');
    expect(oldRow?.error_text).toContain('waiting_ttl_expired');
    expect(oldRow?.finished_at).toBeTruthy();
    const youngRow = await queue.getJob(young.id);
    expect(youngRow?.status).toBe('waiting');
  });

  test('active and delayed rows are never TTL-cancelled', async () => {
    const active = await queue.add('subagent', { prompt: 'running' }, {}, SUB);
    await engine.executeRaw(
      `UPDATE minion_jobs SET status = 'active', claim_generation = claim_generation + 1, lock_until = now() + interval '5 minutes',
        created_at = now() - interval '100 hours' WHERE id = $1`,
      [active.id],
    );
    const { cancelled } = await queue.handleWaitingTTL();
    expect(cancelled).toBe(0);
    expect((await queue.getJob(active.id))?.status).toBe('active');
  });

  test('CRITICAL: TTL-cancelling a fanout child unblocks its waiting-children parent', async () => {
    const parent = await queue.add('subagent_aggregator', { kind: 'agg' }, {}, SUB);
    const child = await queue.add(
      'subagent',
      { prompt: 'doomed child' },
      { parent_job_id: parent.id, on_child_fail: 'continue' },
      SUB,
    );
    expect((await queue.getJob(parent.id))?.status).toBe('waiting-children');
    await backdate(child.id, 49);

    const { cancelled } = await queue.handleWaitingTTL();
    expect(cancelled).toBe(1);
    expect((await queue.getJob(child.id))?.status).toBe('cancelled');
    // The bookkeeping cancelJobs carries: parent resolves out of
    // waiting-children instead of wedging forever (the v128-raw-UPDATE trap).
    expect((await queue.getJob(parent.id))?.status).toBe('waiting');
    // And the child_done inbox message landed for the parent.
    const inbox = await engine.executeRaw<{ count: string }>(
      `SELECT count(*)::text AS count FROM minion_inbox WHERE job_id = $1`,
      [parent.id],
    );
    expect(parseInt(inbox[0].count, 10)).toBe(1);
  });

  test('per-tick cap bounds one sweep; the backlog drains across ticks oldest-first', async () => {
    const ids: number[] = [];
    for (let i = 0; i < 5; i++) {
      const j = await queue.add('subagent', { prompt: `stale ${i}` }, {}, SUB);
      await backdate(j.id, 50 + i);
      ids.push(j.id);
    }
    const tick1 = await queue.handleWaitingTTL({ maxPerTick: 2 });
    expect(tick1.cancelled).toBe(2);
    const tick2 = await queue.handleWaitingTTL({ maxPerTick: 2 });
    expect(tick2.cancelled).toBe(2);
    const tick3 = await queue.handleWaitingTTL({ maxPerTick: 2 });
    expect(tick3.cancelled).toBe(1);
    for (const id of ids) {
      expect((await queue.getJob(id))?.status).toBe('cancelled');
    }
  });

  test('minions.ttl_waiting_hours.subagent=0 disables the sweep for that name', async () => {
    await engine.setConfig('minions.ttl_waiting_hours.subagent', '0');
    const j = await queue.add('subagent', { prompt: 'protected' }, {}, SUB);
    await backdate(j.id, 500);
    const { cancelled } = await queue.handleWaitingTTL();
    expect(cancelled).toBe(0);
    expect((await queue.getJob(j.id))?.status).toBe('waiting');
  });

  test('config override listConfigKeys path: TTL applies to a non-default name', async () => {
    await engine.setConfig('minions.ttl_waiting_hours.shell', '1');
    const j = await queue.add('shell', { command: 'echo stale' }, {}, SUB);
    await backdate(j.id, 2);
    const { by_name } = await queue.handleWaitingTTL();
    expect(by_name.shell).toBe(1);
    expect((await queue.getJob(j.id))?.status).toBe('cancelled');
  });

  test('a TTL-cancelled row frees its idempotency key for a fresh submit', async () => {
    const first = await queue.add('subagent', { prompt: 'keyed' }, { idempotency_key: 'ttl-key-1' }, SUB);
    await backdate(first.id, 49);
    await queue.handleWaitingTTL();
    expect((await queue.getJob(first.id))?.status).toBe('cancelled');
    const fresh = await queue.add('subagent', { prompt: 'keyed' }, { idempotency_key: 'ttl-key-1' }, SUB);
    expect(fresh.id).not.toBe(first.id);
    expect(fresh.status).toBe('waiting');
  });
});

describe('getStats drain/depth extensions', () => {
  test('by_type carries drained_* split, waiting_now, and oldest_waiting_minutes', async () => {
    const done = await queue.add('subagent', { prompt: 'done one' }, {}, SUB);
    await engine.executeRaw(
      `UPDATE minion_jobs SET status = 'completed', started_at = now() - interval '2 minutes',
        finished_at = now() - interval '1 minute' WHERE id = $1`,
      [done.id],
    );
    const ttlVictim = await queue.add('subagent', { prompt: 'shredded' }, {}, SUB);
    await backdate(ttlVictim.id, 49);
    await queue.handleWaitingTTL();
    const waiting = await queue.add('subagent', { prompt: 'still here' }, {}, SUB);
    await backdate(waiting.id, 3);

    const stats = await queue.getStats();
    const sub = stats.by_type.find(t => t.name === 'subagent');
    expect(sub).toBeDefined();
    expect(sub!.drained_completed).toBe(1);
    expect(sub!.drained_cancelled).toBe(1);
    expect(sub!.waiting_now).toBe(1);
    expect(sub!.oldest_waiting_minutes).toBeGreaterThanOrEqual(170); // ~3h
  });
});

describe('idempotency_key precedence over param-coalescing', () => {
  test('a keyed submit never param-coalesces onto an unkeyed identical row', async () => {
    // Producer-owned idempotency is the STRONGER contract: a param-coalesce
    // hit would return a row the key was never registered against, so a
    // later same-key submit would insert fresh and run the work twice.
    const unkeyed = await queue.add('subagent', { prompt: 'same work' }, {}, SUB);
    const keyed = await queue.add('subagent', { prompt: 'same work' }, { idempotency_key: 'key-precedence-1' }, SUB);
    expect(keyed.id).not.toBe(unkeyed.id);
    expect(keyed.coalesced).not.toBe(true);
    // And the keyed row carries no param hash — coalescing was disabled, not deferred.
    expect((keyed.data as Record<string, unknown>).__param_hash).toBeUndefined();
  });
});

describe('cancelJobs reason + rootStatuses (direct)', () => {
  test('reason sets error_text; no-reason cancel preserves pre-existing error_text', async () => {
    const a = await queue.add('subagent', { prompt: 'reason target' }, {}, SUB);
    const b = await queue.add('subagent', { prompt: 'prior-text holder' }, {}, SUB);
    await engine.executeRaw(`UPDATE minion_jobs SET error_text = 'prior failure detail' WHERE id = $1`, [b.id]);

    await queue.cancelJobs([a.id], { reason: 'explicit cancel reason' });
    await queue.cancelJobs([b.id]); // no reason → COALESCE keeps what was there

    const aRow = await queue.getJob(a.id);
    const bRow = await queue.getJob(b.id);
    expect(aRow?.status).toBe('cancelled');
    expect(aRow?.error_text).toBe('explicit cancel reason');
    expect(bRow?.status).toBe('cancelled');
    expect(bRow?.error_text).toBe('prior failure detail');
  });

  test('rootStatuses re-check: an active row in the batch is left running', async () => {
    const w = await queue.add('subagent', { prompt: 'still waiting' }, {}, SUB);
    const act = await queue.add('subagent', { prompt: 'claimed meanwhile' }, {}, SUB);
    await engine.executeRaw(
      `UPDATE minion_jobs SET status = 'active', claim_generation = claim_generation + 1, lock_until = now() + interval '5 minutes' WHERE id = $1`,
      [act.id],
    );
    const swept = await queue.cancelJobs([w.id, act.id], { reason: 'sweep', rootStatuses: ['waiting'] });
    expect(swept.some(j => j.id === w.id)).toBe(true);
    expect(swept.some(j => j.id === act.id)).toBe(false);
    expect((await queue.getJob(w.id))?.status).toBe('cancelled');
    expect((await queue.getJob(act.id))?.status).toBe('active');
  });
});

describe('runWaitingTtlTick warn-before-act gate (D1A)', () => {
  test('tick 1 = notice (counts affected, stamps ISO flag, cancels NOTHING); grace holds; sweep after grace', async () => {
    const victim = await queue.add('subagent', { prompt: 'expired long ago' }, {}, SUB);
    await backdate(victim.id, 49);

    // Tick 1: notice. Nothing cancelled, flag stamped with a timestamp.
    const t1 = await runWaitingTtlTick(engine, queue);
    expect(t1.phase).toBe('notice');
    expect(t1.affected).toBe(1);
    expect(t1.cancelled).toBe(0);
    expect((await queue.getJob(victim.id))?.status).toBe('waiting');
    const flag = await engine.getConfig(TTL_NOTICE_SHOWN_KEY);
    expect(Number.isFinite(Date.parse(flag ?? ''))).toBe(true);

    // Tick 2 inside the grace window: still nothing cancelled.
    const t2 = await runWaitingTtlTick(engine, queue);
    expect(t2.phase).toBe('grace');
    expect((await queue.getJob(victim.id))?.status).toBe('waiting');

    // Grace elapsed (env seam = 0ms): the sweep runs.
    await withEnv({ GBRAIN_MINIONS_TTL_NOTICE_GRACE_MS: '0' }, async () => {
      const t3 = await runWaitingTtlTick(engine, queue);
      expect(t3.phase).toBe('swept');
      expect(t3.cancelled).toBe(1);
    });
    const after = await queue.getJob(victim.id);
    expect(after?.status).toBe('cancelled');
    expect(after?.error_text).toStartWith('waiting_ttl_expired');
  });

  test("legacy flag value 'true' (pre-grace releases) sweeps immediately", async () => {
    const victim = await queue.add('subagent', { prompt: 'legacy-flag victim' }, {}, SUB);
    await backdate(victim.id, 49);
    await engine.setConfig(TTL_NOTICE_SHOWN_KEY, 'true');
    const tick = await runWaitingTtlTick(engine, queue);
    expect(tick.phase).toBe('swept');
    expect((await queue.getJob(victim.id))?.status).toBe('cancelled');
  });

  test('kill-switch: GBRAIN_MINIONS_ADMISSION=0 does nothing, not even the notice', async () => {
    await withEnv({ GBRAIN_MINIONS_ADMISSION: '0' }, async () => {
      const tick = await runWaitingTtlTick(engine, queue);
      expect(tick.phase).toBe('killed');
    });
    expect(await engine.getConfig(TTL_NOTICE_SHOWN_KEY)).toBeNull();
  });
});