Skip to content

Garry's Opinionated OpenClaw/Hermes Agent Brain

TypeScript
10.5 KB
Raw Blame
/**
 * #4310 — provider-identity-keyed global-LLM-halt cooldown at the queued-job
 * boundary (facts-absorb).
 *
 * Pre-fix: a globally-broken provider (revoked key, exhausted quota, hard
 * rate limit) made every queued facts-absorb job burn its own attempts on
 * doomed LLM calls and dead-letter independently. The wrapper arms one
 * provider-keyed cooldown on a halt-class error, DEFERS matching jobs without
 * burning attempts (RateLeaseUnavailableError + retryInMs → worker lease-full
 * requeue), and admits exactly one bounded probe on expiry.
 */

import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
import {
  withGlobalLlmHaltCooldown,
  providerIdentityForJob,
  _resetLlmHaltCooldownsForTests,
  llmHaltCooldownRemainingMs,
  HALT_COOLDOWN_BASE_MS,
} from '../src/core/minions/llm-halt-cooldown.ts';
import { RateLeaseUnavailableError } from '../src/core/minions/rate-leases.ts';
import { RATE_LIMIT_HALT_STREAK } from '../src/core/ai/errors.ts';
import type { MinionJobContext } from '../src/core/minions/types.ts';
import { withEnv } from './helpers/with-env.ts';

const job = (data: Record<string, unknown> = {}): MinionJobContext =>
  ({
    id: 1,
    name: 'facts-absorb',
    data,
    attempts_made: 0,
    signal: new AbortController().signal,
    deadlineAtMs: null,
    updateProgress: async () => {},
  }) as unknown as MinionJobContext;

const authError = () => Object.assign(new Error('provider rejected'), { status: 401 });
const billingError = () => new Error('insufficient_quota: monthly spend limit reached');
const rateLimitError = () => Object.assign(new Error('slow down'), { status: 429 });

beforeEach(() => _resetLlmHaltCooldownsForTests());
afterEach(() => {
  _resetLlmHaltCooldownsForTests();
});

function makeWrapped(opts: {
  key?: string;
  fail?: () => Error | null; // null = succeed
  now?: () => number;
  calls?: { count: number };
}) {
  const calls = opts.calls ?? { count: 0 };
  const handler = async () => {
    calls.count++;
    const err = opts.fail?.();
    if (err) throw err;
    return { ok: true };
  };
  return {
    calls,
    run: withGlobalLlmHaltCooldown(() => opts.key ?? 'prov-a', handler, { now: opts.now }),
  };
}

describe('auth class', () => {
  test('first failure propagates AND arms the cooldown; next job defers without a handler call', async () => {
    const { run, calls } = makeWrapped({ fail: authError });
    await expect(run(job())).rejects.toThrow('provider rejected'); // visible failure
    expect(llmHaltCooldownRemainingMs('prov-a')).toBeGreaterThan(0);

    await expect(run(job())).rejects.toBeInstanceOf(RateLeaseUnavailableError);
    expect(calls.count).toBe(1); // deferred job never reached the provider
  });

  test('deferral carries retryInMs ≈ remaining cooldown (worker requeues delayed, no attempt burned)', async () => {
    const { run } = makeWrapped({ fail: authError });
    await run(job()).catch(() => {});
    try {
      await run(job());
      throw new Error('expected deferral');
    } catch (e) {
      const lease = e as RateLeaseUnavailableError;
      expect(lease).toBeInstanceOf(RateLeaseUnavailableError);
      expect(lease.key).toBe('global-llm-halt:auth:prov-a');
      expect(lease.retryInMs).toBeGreaterThanOrEqual(1000);
      expect(lease.retryInMs!).toBeLessThanOrEqual(HALT_COOLDOWN_BASE_MS.auth + 6000);
    }
  });
});

describe('billing class', () => {
  test('first billing failure arms the cooldown', async () => {
    const { run, calls } = makeWrapped({ fail: billingError });
    await expect(run(job())).rejects.toThrow('insufficient_quota');
    await expect(run(job())).rejects.toBeInstanceOf(RateLeaseUnavailableError);
    expect(calls.count).toBe(1);
  });
});

describe('rate_limit class', () => {
  test(`arms only after ${RATE_LIMIT_HALT_STREAK} consecutive 429s (streak shared ACROSS jobs)`, async () => {
    const { run, calls } = makeWrapped({ fail: rateLimitError });
    for (let i = 0; i < RATE_LIMIT_HALT_STREAK - 1; i++) {
      await expect(run(job())).rejects.toThrow('slow down'); // still per-item
      expect(llmHaltCooldownRemainingMs('prov-a')).toBe(0);
    }
    await expect(run(job())).rejects.toThrow('slow down'); // streak crosses threshold
    expect(llmHaltCooldownRemainingMs('prov-a')).toBeGreaterThan(0);
    await expect(run(job())).rejects.toBeInstanceOf(RateLeaseUnavailableError);
    expect(calls.count).toBe(RATE_LIMIT_HALT_STREAK);
  });

  test('a success between 429s clears the shared streak', async () => {
    let n = 0;
    const { run } = makeWrapped({ fail: () => (++n === 2 ? null : rateLimitError()) });
    await run(job()).catch(() => {});
    await run(job()); // success → streak reset
    await run(job()).catch(() => {});
    await run(job()).catch(() => {});
    expect(llmHaltCooldownRemainingMs('prov-a')).toBe(0); // 2 consecutive < threshold
  });
});

describe('probe on expiry', () => {
  test('one probe admitted; success clears the cooldown', async () => {
    let t = 1_000_000;
    let failNow = true;
    const { run, calls } = makeWrapped({ fail: () => (failNow ? authError() : null), now: () => t });
    await run(job()).catch(() => {});
    expect(llmHaltCooldownRemainingMs('prov-a', t)).toBe(HALT_COOLDOWN_BASE_MS.auth);

    t += HALT_COOLDOWN_BASE_MS.auth + 1; // cooldown lapsed
    failNow = false; // provider recovered
    const result = await run(job()); // the bounded probe
    expect(result).toEqual({ ok: true });
    expect(llmHaltCooldownRemainingMs('prov-a', t)).toBe(0);
    await run(job()); // steady state restored
    expect(calls.count).toBe(3);
  });

  test('probe failure re-arms with DOUBLED backoff', async () => {
    let t = 1_000_000;
    const { run } = makeWrapped({ fail: authError, now: () => t });
    await run(job()).catch(() => {});
    t += HALT_COOLDOWN_BASE_MS.auth + 1;
    await expect(run(job())).rejects.toThrow('provider rejected'); // probe fails
    const remaining = llmHaltCooldownRemainingMs('prov-a', t);
    expect(remaining).toBe(HALT_COOLDOWN_BASE_MS.auth * 2);
  });

  test('while the probe is in flight, other jobs defer on a short retry', async () => {
    let t = 1_000_000;
    let release: () => void = () => {};
    const gate = new Promise<void>((r) => { release = r; });
    let armed = true;
    const handler = async () => {
      if (armed) { armed = false; throw authError(); }
      await gate;
      return { ok: true };
    };
    const run = withGlobalLlmHaltCooldown(() => 'prov-a', handler, { now: () => t });
    await run(job()).catch(() => {});
    t += HALT_COOLDOWN_BASE_MS.auth + 1;
    const probe = run(job()); // in flight, parked on the gate
    await expect(run(job())).rejects.toBeInstanceOf(RateLeaseUnavailableError);
    release();
    expect(await probe).toEqual({ ok: true });
  });

  test('a nested deferral from the probe releases the probe slot (no permanent wedge)', async () => {
    let t = 1_000_000;
    let mode: 'halt' | 'nested-defer' | 'ok' = 'halt';
    const handler = async () => {
      if (mode === 'halt') throw authError();
      if (mode === 'nested-defer') throw new RateLeaseUnavailableError('model-lease', 1, 1, 500);
      return { ok: true };
    };
    const run = withGlobalLlmHaltCooldown(() => 'prov-a', handler, { now: () => t });
    await run(job()).catch(() => {}); // arms the cooldown
    t += HALT_COOLDOWN_BASE_MS.auth + 1;
    mode = 'nested-defer';
    // The admitted probe defers via a nested lease wrapper — passes through.
    await expect(run(job())).rejects.toBeInstanceOf(RateLeaseUnavailableError);
    // Pre-fix: probeInFlight stayed latched forever, so this job (and the
    // requeued probe itself) deferred on the 15s cadence until worker restart.
    mode = 'ok';
    expect(await run(job())).toEqual({ ok: true });
  });
});

describe('strikes escalate per probe, not per concurrent failure', () => {
  test('a burst of concurrent halt-class failures arms ONE strike (base cooldown, not escalated)', async () => {
    let t = 1_000_000;
    let release: () => void = () => {};
    const gate = new Promise<void>((r) => { release = r; });
    const handler = async () => { await gate; throw authError(); };
    const run = withGlobalLlmHaltCooldown(() => 'prov-a', handler, { now: () => t });
    // Three jobs in flight BEFORE any cooldown exists — all fail together.
    const a = run(job()).catch(() => {});
    const b = run(job()).catch(() => {});
    const c = run(job()).catch(() => {});
    release();
    await Promise.all([a, b, c]);
    // Pre-fix: each concurrent failure incremented strikes (3 → 20min).
    // The documented policy is consecutive halted PROBES → exponential re-arm.
    expect(llmHaltCooldownRemainingMs('prov-a', t)).toBe(HALT_COOLDOWN_BASE_MS.auth);
  });
});

describe('isolation + hatches', () => {
  test('cooldown is provider-keyed: a different provider is unaffected', async () => {
    const a = makeWrapped({ key: 'prov-a', fail: authError });
    const b = makeWrapped({ key: 'prov-b', fail: () => null });
    await a.run(job()).catch(() => {});
    expect(await b.run(job())).toEqual({ ok: true });
  });

  test('non-halt errors stay per-item (no cooldown)', async () => {
    const { run } = makeWrapped({ fail: () => new Error('context length exceeded for this page') });
    await run(job()).catch(() => {});
    expect(llmHaltCooldownRemainingMs('prov-a')).toBe(0);
    await expect(run(job())).rejects.toThrow('context length'); // handler ran again
  });

  test('GBRAIN_LLM_HALT_COOLDOWN=0 disables the mechanism', async () => {
    await withEnv({ GBRAIN_LLM_HALT_COOLDOWN: '0' }, async () => {
      const { run, calls } = makeWrapped({ fail: authError });
      await run(job()).catch(() => {});
      await run(job()).catch(() => {});
      expect(calls.count).toBe(2); // no deferral — pre-#4310 behavior
    });
  });
});

describe('providerIdentityForJob', () => {
  test('job model override wins; provider half is the key', () => {
    expect(providerIdentityForJob(job({ model: 'anthropic:claude-x' }), () => 'openai:gpt-4o')).toBe('anthropic');
  });
  test('falls back to the configured chat model', () => {
    expect(providerIdentityForJob(job(), () => 'openai:gpt-4o')).toBe('openai');
  });
  test('bare model ids key on the whole string', () => {
    expect(providerIdentityForJob(job({ model: 'gpt-4o' }), () => null)).toBe('gpt-4o');
  });
});

describe('RateLeaseUnavailableError.retryInMs (worker contract)', () => {
  test('optional field carried for the worker lease-full path', () => {
    expect(new RateLeaseUnavailableError('k', 1, 1, 5000).retryInMs).toBe(5000);
    expect(new RateLeaseUnavailableError('k', 1, 1).retryInMs).toBeUndefined();
  });
});